Store failed commands in traces

This commit is contained in:
Andy Leiserson authored and Teodor Tanasoaia committed 2026-01-09 15:55:33 +01:00
1 parent cb02f74aa4
commit e412592b23
4 files changed
+382 -249

No files matched your search

+17
View File
@@ -417,6 +417,23 @@ impl Player {
let buffer = wgc::command::CommandBuffer::from_trace(device, resolved_commands);
queue.submit(&[buffer]).unwrap();
}
Action::FailedCommands {
commands,
failed_at_submit,
error,
} => {
let action = if failed_at_submit.is_some() {
"submitting"
} else {
"encoding"
};
if let Some(commands) = commands {
log::trace!(
"Trace recorded an error {action} the following commands: {commands:#?}"
);
}
panic!("Error recorded in trace: {error}");
}
Action::CreateBlas { id, desc, sizes } => {
let blas = device.create_blas(&desc, sizes).expect("create_blas error");
self.blas_s.insert(id, blas);
+306 -239
View File
@@ -98,6 +98,31 @@ pub type CopyExternalImageDestInfo = ffi::CopyExternalImageDestInfo;
const IMMEDIATES_CLEAR_ARRAY: &[u32] = &[0_u32; 64];
pub(crate) struct EncoderErrorState {
error: CommandEncoderError,
#[cfg(feature = "trace")]
trace_commands: Option<Vec<Command<PointerReferences>>>,
}
/// Construct an `EncoderErrorState` with only a `CommandEncoderError` (without
/// any traced commands).
///
/// This is used in cases where pass begin/end were mismatched, if the same
/// encoder was finished multiple times, or in the status of a command buffer
/// (in which case the commands were already saved to the trace). In some of
/// these cases there may be commands that could be saved to the trace, but if
/// the application is that confused about using encoders, it's not clear
/// whether it's worth the effort to try and preserve the commands.
fn make_error_state<E: Into<CommandEncoderError>>(error: E) -> CommandEncoderStatus {
CommandEncoderStatus::Error(EncoderErrorState {
error: error.into(),
#[cfg(feature = "trace")]
trace_commands: None,
})
}
/// The current state of a command or pass encoder.
///
/// In the WebGPU spec, the state of an encoder (open, locked, or ended) is
@@ -145,7 +170,7 @@ pub(crate) enum CommandEncoderStatus {
///
/// The error that caused the invalidation is stored here, and will
/// be raised by `CommandEncoder.finish()`.
Error(CommandEncoderError),
Error(EncoderErrorState),
/// Temporary state used internally by methods on `CommandEncoderStatus`.
/// Encoder should never be left in this state.
@@ -333,7 +358,7 @@ impl CommandEncoderStatus {
Err(EncoderStateError::Ended)
}
Self::Recording(_) => {
*self = Self::Error(EncoderStateError::Unlocked.into());
*self = make_error_state(EncoderStateError::Unlocked);
Err(EncoderStateError::Unlocked)
}
st @ Self::Consumed => {
@@ -362,22 +387,39 @@ impl CommandEncoderStatus {
}
Self::Finished(inner)
}
Self::Consumed | Self::Finished(_) => Self::Error(EncoderStateError::Ended.into()),
Self::Locked(_) => Self::Error(EncoderStateError::Locked.into()),
Self::Consumed | Self::Finished(_) => make_error_state(EncoderStateError::Ended),
Self::Locked(_) => make_error_state(EncoderStateError::Locked),
st @ Self::Error(_) => st,
Self::Transitioning => unreachable!(),
}
}
// Invalidate the command encoder and store the error `err` causing the
// invalidation for diagnostic purposes.
//
// Since we do not track the state of an invalid encoder, it is not
// necessary to unlock an encoder that has been invalidated.
/// Invalidate the command encoder due to an error.
///
/// The error `err` is stored so that it can be reported when the encoder is
/// finished. If tracing is enabled, the traced commands are also stored.
///
/// Since we do not track the state of an invalid encoder, it is not
/// necessary to unlock an encoder that has been invalidated.
fn invalidate<E: Clone + Into<CommandEncoderError>>(&mut self, err: E) -> E {
#[cfg(feature = "trace")]
let trace_commands = match self {
Self::Recording(cmd_buf_data) => Some(
mem::take(&mut cmd_buf_data.commands)
.into_iter()
.map(crate::device::trace::IntoTrace::into_trace)
.collect(),
),
_ => None,
};
let enc_err = err.clone().into();
api_log!("Invalidating command encoder: {enc_err:?}");
*self = Self::Error(enc_err);
*self = Self::Error(EncoderErrorState {
error: enc_err,
#[cfg(feature = "trace")]
trace_commands,
});
err
}
}
@@ -861,7 +903,7 @@ impl CommandEncoder {
CommandEncoder {
device: device.clone(),
label: label.to_string(),
data: Mutex::new(rank::COMMAND_BUFFER_DATA, CommandEncoderStatus::Error(err)),
data: Mutex::new(rank::COMMAND_BUFFER_DATA, make_error_state(err)),
}
}
@@ -941,6 +983,220 @@ impl CommandEncoder {
}
}
fn encode_commands(
device: &Arc<Device>,
cmd_buf_data: &mut CommandBufferMutable,
) -> Result<(), CommandEncoderError> {
device.check_is_valid()?;
let snatch_guard = device.snatchable_lock.read();
let mut debug_scope_depth = 0;
if cmd_buf_data.encoder.api == EncodingApi::Raw {
// Should have panicked on the first call that switched APIs,
// but lets be sure.
assert!(cmd_buf_data.commands.is_empty());
}
let commands = mem::take(&mut cmd_buf_data.commands);
#[cfg(feature = "trace")]
if device.trace.lock().is_some() {
cmd_buf_data.trace_commands = Some(
commands
.iter()
.map(crate::device::trace::IntoTrace::to_trace)
.collect(),
);
}
for command in commands {
if matches!(
command,
ArcCommand::RunRenderPass { .. } | ArcCommand::RunComputePass { .. }
) {
// Compute passes and render passes can accept either an
// open or closed encoder. This state object holds an
// `InnerCommandEncoder`. See the documentation of
// [`EncodingState`].
let mut state = EncodingState {
device,
raw_encoder: &mut cmd_buf_data.encoder,
tracker: &mut cmd_buf_data.trackers,
buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions,
texture_memory_actions: &mut cmd_buf_data.texture_memory_actions,
as_actions: &mut cmd_buf_data.as_actions,
temp_resources: &mut cmd_buf_data.temp_resources,
indirect_draw_validation_resources: &mut cmd_buf_data
.indirect_draw_validation_resources,
snatch_guard: &snatch_guard,
debug_scope_depth: &mut debug_scope_depth,
};
match command {
ArcCommand::RunRenderPass {
pass,
color_attachments,
depth_stencil_attachment,
timestamp_writes,
occlusion_query_set,
multiview_mask,
} => {
api_log!(
"Begin encoding render pass with '{}' label",
pass.label.as_deref().unwrap_or("")
);
let res = encode_render_pass(
&mut state,
pass,
color_attachments,
depth_stencil_attachment,
timestamp_writes,
occlusion_query_set,
multiview_mask,
);
match res.as_ref() {
Err(err) => {
api_log!("Finished encoding render pass ({err:?})")
}
Ok(_) => {
api_log!("Finished encoding render pass (success)")
}
}
res?;
}
ArcCommand::RunComputePass {
pass,
timestamp_writes,
} => {
api_log!(
"Begin encoding compute pass with '{}' label",
pass.label.as_deref().unwrap_or("")
);
let res = encode_compute_pass(&mut state, pass, timestamp_writes);
match res.as_ref() {
Err(err) => {
api_log!("Finished encoding compute pass ({err:?})")
}
Ok(_) => {
api_log!("Finished encoding compute pass (success)")
}
}
res?;
}
_ => unreachable!(),
}
} else {
// All the other non-pass encoding routines assume the
// encoder is open, so open it if necessary. This state
// object holds an `&mut dyn hal::DynCommandEncoder`. By
// convention, a bare HAL encoder reference in
// [`EncodingState`] must always be an open encoder.
let raw_encoder = cmd_buf_data.encoder.open_if_closed()?;
let mut state = EncodingState {
device,
raw_encoder,
tracker: &mut cmd_buf_data.trackers,
buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions,
texture_memory_actions: &mut cmd_buf_data.texture_memory_actions,
as_actions: &mut cmd_buf_data.as_actions,
temp_resources: &mut cmd_buf_data.temp_resources,
indirect_draw_validation_resources: &mut cmd_buf_data
.indirect_draw_validation_resources,
snatch_guard: &snatch_guard,
debug_scope_depth: &mut debug_scope_depth,
};
match command {
ArcCommand::CopyBufferToBuffer {
src,
src_offset,
dst,
dst_offset,
size,
} => {
copy_buffer_to_buffer(
&mut state, &src, src_offset, &dst, dst_offset, size,
)?;
}
ArcCommand::CopyBufferToTexture { src, dst, size } => {
copy_buffer_to_texture(&mut state, &src, &dst, &size)?;
}
ArcCommand::CopyTextureToBuffer { src, dst, size } => {
copy_texture_to_buffer(&mut state, &src, &dst, &size)?;
}
ArcCommand::CopyTextureToTexture { src, dst, size } => {
copy_texture_to_texture(&mut state, &src, &dst, &size)?;
}
ArcCommand::ClearBuffer { dst, offset, size } => {
clear_buffer(&mut state, dst, offset, size)?;
}
ArcCommand::ClearTexture {
dst,
subresource_range,
} => {
clear_texture_cmd(&mut state, dst, &subresource_range)?;
}
ArcCommand::WriteTimestamp {
query_set,
query_index,
} => {
write_timestamp(&mut state, query_set, query_index)?;
}
ArcCommand::ResolveQuerySet {
query_set,
start_query,
query_count,
destination,
destination_offset,
} => {
resolve_query_set(
&mut state,
query_set,
start_query,
query_count,
destination,
destination_offset,
)?;
}
ArcCommand::PushDebugGroup(label) => {
push_debug_group(&mut state, &label)?;
}
ArcCommand::PopDebugGroup => {
pop_debug_group(&mut state)?;
}
ArcCommand::InsertDebugMarker(label) => {
insert_debug_marker(&mut state, &label)?;
}
ArcCommand::BuildAccelerationStructures { blas, tlas } => {
build_acceleration_structures(&mut state, blas, tlas)?;
}
ArcCommand::TransitionResources {
buffer_transitions,
texture_transitions,
} => {
transition_resources(&mut state, buffer_transitions, texture_transitions)?;
}
ArcCommand::RunComputePass { .. } | ArcCommand::RunRenderPass { .. } => {
unreachable!()
}
}
}
}
if debug_scope_depth > 0 {
Err(CommandEncoderError::DebugGroupError(
DebugGroupError::MissingPop,
))?;
}
// Close the encoder, unless it was closed already by a render or compute pass.
cmd_buf_data.encoder.close_if_open()?;
// Note: if we want to stop tracking the swapchain texture view,
// this is the place to do it.
Ok(())
}
fn finish(
self: &Arc<Self>,
desc: &wgt::CommandBufferDescriptor<Label>,
@@ -948,239 +1204,46 @@ impl CommandEncoder {
let mut cmd_enc_status = self.data.lock();
let res = match cmd_enc_status.finish() {
CommandEncoderStatus::Finished(cmd_buf_data) => Ok(cmd_buf_data),
CommandEncoderStatus::Error(err) => Err(err),
CommandEncoderStatus::Finished(mut cmd_buf_data) => {
match Self::encode_commands(&self.device, &mut cmd_buf_data) {
Ok(()) => Ok(cmd_buf_data),
Err(error) => Err(EncoderErrorState {
error,
#[cfg(feature = "trace")]
trace_commands: mem::take(&mut cmd_buf_data.trace_commands),
}),
}
}
CommandEncoderStatus::Error(error_state) => Err(error_state),
_ => unreachable!(),
};
let res = res.and_then(|mut cmd_buf_data| {
self.device.check_is_valid()?;
let snatch_guard = self.device.snatchable_lock.read();
let mut debug_scope_depth = 0;
if cmd_buf_data.encoder.api == EncodingApi::Raw {
// Should have panicked on the first call that switched APIs,
// but lets be sure.
assert!(cmd_buf_data.commands.is_empty());
}
let mut commands = mem::take(&mut cmd_buf_data.commands);
#[cfg(not(feature = "trace"))]
let command_iter = commands.drain(..);
#[cfg(feature = "trace")]
let mut trace_commands = None;
#[cfg(feature = "trace")]
let command_iter = {
if self.device.trace.lock().is_some() {
trace_commands = Some(
cmd_buf_data
.trace_commands
.insert(Vec::with_capacity(commands.len())),
);
}
commands.drain(..).inspect(|cmd| {
use crate::device::trace::IntoTrace;
if let Some(ref mut trace) = trace_commands {
trace.push(cmd.clone().to_trace());
}
})
};
for command in command_iter {
if matches!(
command,
ArcCommand::RunRenderPass { .. } | ArcCommand::RunComputePass { .. }
) {
// Compute passes and render passes can accept either an
// open or closed encoder. This state object holds an
// `InnerCommandEncoder`. See the documentation of
// [`EncodingState`].
let mut state = EncodingState {
device: &self.device,
raw_encoder: &mut cmd_buf_data.encoder,
tracker: &mut cmd_buf_data.trackers,
buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions,
texture_memory_actions: &mut cmd_buf_data.texture_memory_actions,
as_actions: &mut cmd_buf_data.as_actions,
temp_resources: &mut cmd_buf_data.temp_resources,
indirect_draw_validation_resources: &mut cmd_buf_data
.indirect_draw_validation_resources,
snatch_guard: &snatch_guard,
debug_scope_depth: &mut debug_scope_depth,
};
match command {
ArcCommand::RunRenderPass {
pass,
color_attachments,
depth_stencil_attachment,
timestamp_writes,
occlusion_query_set,
multiview_mask,
} => {
api_log!(
"Begin encoding render pass with '{}' label",
pass.label.as_deref().unwrap_or("")
);
let res = encode_render_pass(
&mut state,
pass,
color_attachments,
depth_stencil_attachment,
timestamp_writes,
occlusion_query_set,
multiview_mask,
);
match res.as_ref() {
Err(err) => api_log!("Finished encoding render pass ({err:?})"),
Ok(_) => api_log!("Finished encoding render pass (success)"),
}
res?;
}
ArcCommand::RunComputePass {
pass,
timestamp_writes,
} => {
api_log!(
"Begin encoding compute pass with '{}' label",
pass.label.as_deref().unwrap_or("")
);
let res = encode_compute_pass(&mut state, pass, timestamp_writes);
match res.as_ref() {
Err(err) => api_log!("Finished encoding compute pass ({err:?})"),
Ok(_) => api_log!("Finished encoding compute pass (success)"),
}
res?;
}
_ => unreachable!(),
}
} else {
// All the other non-pass encoding routines assume the
// encoder is open, so open it if necessary. This state
// object holds an `&mut dyn hal::DynCommandEncoder`. By
// convention, a bare HAL encoder reference in
// [`EncodingState`] must always be an open encoder.
let raw_encoder = cmd_buf_data.encoder.open_if_closed()?;
let mut state = EncodingState {
device: &self.device,
raw_encoder,
tracker: &mut cmd_buf_data.trackers,
buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions,
texture_memory_actions: &mut cmd_buf_data.texture_memory_actions,
as_actions: &mut cmd_buf_data.as_actions,
temp_resources: &mut cmd_buf_data.temp_resources,
indirect_draw_validation_resources: &mut cmd_buf_data
.indirect_draw_validation_resources,
snatch_guard: &snatch_guard,
debug_scope_depth: &mut debug_scope_depth,
};
match command {
ArcCommand::CopyBufferToBuffer {
src,
src_offset,
dst,
dst_offset,
size,
} => {
copy_buffer_to_buffer(
&mut state, &src, src_offset, &dst, dst_offset, size,
)?;
}
ArcCommand::CopyBufferToTexture { src, dst, size } => {
copy_buffer_to_texture(&mut state, &src, &dst, &size)?;
}
ArcCommand::CopyTextureToBuffer { src, dst, size } => {
copy_texture_to_buffer(&mut state, &src, &dst, &size)?;
}
ArcCommand::CopyTextureToTexture { src, dst, size } => {
copy_texture_to_texture(&mut state, &src, &dst, &size)?;
}
ArcCommand::ClearBuffer { dst, offset, size } => {
clear_buffer(&mut state, dst, offset, size)?;
}
ArcCommand::ClearTexture {
dst,
subresource_range,
} => {
clear_texture_cmd(&mut state, dst, &subresource_range)?;
}
ArcCommand::WriteTimestamp {
query_set,
query_index,
} => {
write_timestamp(&mut state, query_set, query_index)?;
}
ArcCommand::ResolveQuerySet {
query_set,
start_query,
query_count,
destination,
destination_offset,
} => {
resolve_query_set(
&mut state,
query_set,
start_query,
query_count,
destination,
destination_offset,
)?;
}
ArcCommand::PushDebugGroup(label) => {
push_debug_group(&mut state, &label)?;
}
ArcCommand::PopDebugGroup => {
pop_debug_group(&mut state)?;
}
ArcCommand::InsertDebugMarker(label) => {
insert_debug_marker(&mut state, &label)?;
}
ArcCommand::BuildAccelerationStructures { blas, tlas } => {
build_acceleration_structures(&mut state, blas, tlas)?;
}
ArcCommand::TransitionResources {
buffer_transitions,
texture_transitions,
} => {
transition_resources(
&mut state,
buffer_transitions,
texture_transitions,
)?;
}
ArcCommand::RunComputePass { .. } | ArcCommand::RunRenderPass { .. } => {
unreachable!()
}
}
}
}
if debug_scope_depth > 0 {
Err(CommandEncoderError::DebugGroupError(
DebugGroupError::MissingPop,
))?;
}
// Close the encoder, unless it was closed already by a render or compute pass.
cmd_buf_data.encoder.close_if_open()?;
// Note: if we want to stop tracking the swapchain texture view,
// this is the place to do it.
Ok(cmd_buf_data)
});
let (data, error) = match res {
Err(e) => {
if e.is_destroyed_error() {
Err(EncoderErrorState {
error,
#[cfg(feature = "trace")]
trace_commands,
}) => {
// Normally, commands are added to the trace when submitted, but
// since this command buffer won't be submitted, add it to the
// trace now.
#[cfg(feature = "trace")]
if let Some(trace) = self.device.trace.lock().as_mut() {
use alloc::string::ToString;
trace.add(crate::device::trace::Action::FailedCommands {
commands: trace_commands,
failed_at_submit: None,
error: error.to_string(),
});
}
if error.is_destroyed_error() {
// Errors related to destroyed resources are not reported until the
// command buffer is submitted.
(CommandEncoderStatus::Error(e.clone()), None)
(make_error_state(error), None)
} else {
(CommandEncoderStatus::Error(e.clone()), Some(e))
(make_error_state(error.clone()), Some(error))
}
}
@@ -1222,10 +1285,14 @@ impl CommandBuffer {
use CommandEncoderStatus as St;
match mem::replace(
&mut *self.data.lock(),
CommandEncoderStatus::Error(EncoderStateError::Submitted.into()),
make_error_state(EncoderStateError::Submitted),
) {
St::Finished(command_buffer_mutable) => Ok(command_buffer_mutable),
St::Error(err) => Err(err),
St::Error(EncoderErrorState {
#[cfg(feature = "trace")]
trace_commands: _,
error,
}) => Err(error),
St::Recording(_) | St::Locked(_) | St::Consumed | St::Transitioning => unreachable!(),
}
}
+51 -10
View File
@@ -1155,6 +1155,33 @@ impl Queue {
Ok(())
}
#[cfg(feature = "trace")]
fn trace_submission(
&self,
submit_index: SubmissionIndex,
commands: Vec<crate::command::Command<crate::command::PointerReferences>>,
) {
if let Some(ref mut trace) = *self.device.trace.lock() {
trace.add(Action::Submit(submit_index, commands));
}
}
#[cfg(feature = "trace")]
fn trace_failed_submission(
&self,
submit_index: SubmissionIndex,
commands: Option<Vec<crate::command::Command<crate::command::PointerReferences>>>,
error: alloc::string::String,
) {
if let Some(ref mut trace) = *self.device.trace.lock() {
trace.add(Action::FailedCommands {
commands,
failed_at_submit: Some(submit_index),
error,
});
}
}
pub fn submit(
&self,
command_buffers: &[Arc<CommandBuffer>],
@@ -1209,20 +1236,16 @@ impl Queue {
#[allow(unused_mut)]
let mut cmd_buf_data = command_buffer.take_finished();
#[cfg(feature = "trace")]
if let Some(ref mut trace) = *self.device.trace.lock() {
if let Ok(ref mut cmd_buf_data) = cmd_buf_data {
trace.add(Action::Submit(
submit_index,
cmd_buf_data.trace_commands.take().unwrap(),
));
}
}
if first_error.is_some() {
continue;
}
#[cfg(feature = "trace")]
let trace_commands = cmd_buf_data
.as_mut()
.ok()
.and_then(|data| mem::take(&mut data.trace_commands));
let mut baked = match cmd_buf_data {
Ok(cmd_buf_data) => {
let res = validate_command_buffer(
@@ -1235,12 +1258,30 @@ impl Queue {
&mut command_index_guard,
);
if let Err(err) = res {
#[cfg(feature = "trace")]
self.trace_failed_submission(
submit_index,
trace_commands,
err.to_string(),
);
first_error.get_or_insert(err);
continue;
}
#[cfg(feature = "trace")]
if let Some(commands) = trace_commands {
self.trace_submission(submit_index, commands);
}
cmd_buf_data.into_baked_commands()
}
Err(err) => {
#[cfg(feature = "trace")]
self.trace_failed_submission(
submit_index,
trace_commands,
err.to_string(),
);
first_error.get_or_insert(err.into());
continue;
}
+8
View File
@@ -130,6 +130,14 @@ pub enum Action<'a, R: ReferenceType> {
size: wgt::Extent3d,
},
Submit(crate::SubmissionIndex, Vec<Command<R>>),
FailedCommands {
commands: Option<Vec<Command<R>>>,
/// If `None`, then encoding failed due to a validation error (returned
/// from `CommandEncoder::finish`). If `Some`, submission failed due to
/// a resource having been destroyed.
failed_at_submit: Option<crate::SubmissionIndex>,
error: String,
},
CreateBlas {
id: R::Blas,
desc: crate::resource::BlasDescriptor<'a>,