From e412592b2368071e02c045e5de1b0410eb84e350 Mon Sep 17 00:00:00 2001 From: Andy Leiserson Date: Wed, 22 Oct 2025 18:19:15 -0700 Subject: [PATCH] Store failed commands in traces --- player/src/lib.rs | 17 ++ wgpu-core/src/command/mod.rs | 545 +++++++++++++++++++--------------- wgpu-core/src/device/queue.rs | 61 +++- wgpu-core/src/device/trace.rs | 8 + 4 files changed, 382 insertions(+), 249 deletions(-) diff --git a/player/src/lib.rs b/player/src/lib.rs index 369e34043..3602594eb 100644 --- a/player/src/lib.rs +++ b/player/src/lib.rs @@ -417,6 +417,23 @@ impl Player { let buffer = wgc::command::CommandBuffer::from_trace(device, resolved_commands); queue.submit(&[buffer]).unwrap(); } + Action::FailedCommands { + commands, + failed_at_submit, + error, + } => { + let action = if failed_at_submit.is_some() { + "submitting" + } else { + "encoding" + }; + if let Some(commands) = commands { + log::trace!( + "Trace recorded an error {action} the following commands: {commands:#?}" + ); + } + panic!("Error recorded in trace: {error}"); + } Action::CreateBlas { id, desc, sizes } => { let blas = device.create_blas(&desc, sizes).expect("create_blas error"); self.blas_s.insert(id, blas); diff --git a/wgpu-core/src/command/mod.rs b/wgpu-core/src/command/mod.rs index 84d12ce40..b4edd82c5 100644 --- a/wgpu-core/src/command/mod.rs +++ b/wgpu-core/src/command/mod.rs @@ -98,6 +98,31 @@ pub type CopyExternalImageDestInfo = ffi::CopyExternalImageDestInfo; const IMMEDIATES_CLEAR_ARRAY: &[u32] = &[0_u32; 64]; +pub(crate) struct EncoderErrorState { + error: CommandEncoderError, + + #[cfg(feature = "trace")] + trace_commands: Option>>, +} + +/// Construct an `EncoderErrorState` with only a `CommandEncoderError` (without +/// any traced commands). +/// +/// This is used in cases where pass begin/end were mismatched, if the same +/// encoder was finished multiple times, or in the status of a command buffer +/// (in which case the commands were already saved to the trace). In some of +/// these cases there may be commands that could be saved to the trace, but if +/// the application is that confused about using encoders, it's not clear +/// whether it's worth the effort to try and preserve the commands. +fn make_error_state>(error: E) -> CommandEncoderStatus { + CommandEncoderStatus::Error(EncoderErrorState { + error: error.into(), + + #[cfg(feature = "trace")] + trace_commands: None, + }) +} + /// The current state of a command or pass encoder. /// /// In the WebGPU spec, the state of an encoder (open, locked, or ended) is @@ -145,7 +170,7 @@ pub(crate) enum CommandEncoderStatus { /// /// The error that caused the invalidation is stored here, and will /// be raised by `CommandEncoder.finish()`. - Error(CommandEncoderError), + Error(EncoderErrorState), /// Temporary state used internally by methods on `CommandEncoderStatus`. /// Encoder should never be left in this state. @@ -333,7 +358,7 @@ impl CommandEncoderStatus { Err(EncoderStateError::Ended) } Self::Recording(_) => { - *self = Self::Error(EncoderStateError::Unlocked.into()); + *self = make_error_state(EncoderStateError::Unlocked); Err(EncoderStateError::Unlocked) } st @ Self::Consumed => { @@ -362,22 +387,39 @@ impl CommandEncoderStatus { } Self::Finished(inner) } - Self::Consumed | Self::Finished(_) => Self::Error(EncoderStateError::Ended.into()), - Self::Locked(_) => Self::Error(EncoderStateError::Locked.into()), + Self::Consumed | Self::Finished(_) => make_error_state(EncoderStateError::Ended), + Self::Locked(_) => make_error_state(EncoderStateError::Locked), st @ Self::Error(_) => st, Self::Transitioning => unreachable!(), } } - // Invalidate the command encoder and store the error `err` causing the - // invalidation for diagnostic purposes. - // - // Since we do not track the state of an invalid encoder, it is not - // necessary to unlock an encoder that has been invalidated. + /// Invalidate the command encoder due to an error. + /// + /// The error `err` is stored so that it can be reported when the encoder is + /// finished. If tracing is enabled, the traced commands are also stored. + /// + /// Since we do not track the state of an invalid encoder, it is not + /// necessary to unlock an encoder that has been invalidated. fn invalidate>(&mut self, err: E) -> E { + #[cfg(feature = "trace")] + let trace_commands = match self { + Self::Recording(cmd_buf_data) => Some( + mem::take(&mut cmd_buf_data.commands) + .into_iter() + .map(crate::device::trace::IntoTrace::into_trace) + .collect(), + ), + _ => None, + }; + let enc_err = err.clone().into(); api_log!("Invalidating command encoder: {enc_err:?}"); - *self = Self::Error(enc_err); + *self = Self::Error(EncoderErrorState { + error: enc_err, + #[cfg(feature = "trace")] + trace_commands, + }); err } } @@ -861,7 +903,7 @@ impl CommandEncoder { CommandEncoder { device: device.clone(), label: label.to_string(), - data: Mutex::new(rank::COMMAND_BUFFER_DATA, CommandEncoderStatus::Error(err)), + data: Mutex::new(rank::COMMAND_BUFFER_DATA, make_error_state(err)), } } @@ -941,6 +983,220 @@ impl CommandEncoder { } } + fn encode_commands( + device: &Arc, + cmd_buf_data: &mut CommandBufferMutable, + ) -> Result<(), CommandEncoderError> { + device.check_is_valid()?; + let snatch_guard = device.snatchable_lock.read(); + let mut debug_scope_depth = 0; + + if cmd_buf_data.encoder.api == EncodingApi::Raw { + // Should have panicked on the first call that switched APIs, + // but lets be sure. + assert!(cmd_buf_data.commands.is_empty()); + } + + let commands = mem::take(&mut cmd_buf_data.commands); + + #[cfg(feature = "trace")] + if device.trace.lock().is_some() { + cmd_buf_data.trace_commands = Some( + commands + .iter() + .map(crate::device::trace::IntoTrace::to_trace) + .collect(), + ); + } + + for command in commands { + if matches!( + command, + ArcCommand::RunRenderPass { .. } | ArcCommand::RunComputePass { .. } + ) { + // Compute passes and render passes can accept either an + // open or closed encoder. This state object holds an + // `InnerCommandEncoder`. See the documentation of + // [`EncodingState`]. + let mut state = EncodingState { + device, + raw_encoder: &mut cmd_buf_data.encoder, + tracker: &mut cmd_buf_data.trackers, + buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions, + texture_memory_actions: &mut cmd_buf_data.texture_memory_actions, + as_actions: &mut cmd_buf_data.as_actions, + temp_resources: &mut cmd_buf_data.temp_resources, + indirect_draw_validation_resources: &mut cmd_buf_data + .indirect_draw_validation_resources, + snatch_guard: &snatch_guard, + debug_scope_depth: &mut debug_scope_depth, + }; + + match command { + ArcCommand::RunRenderPass { + pass, + color_attachments, + depth_stencil_attachment, + timestamp_writes, + occlusion_query_set, + multiview_mask, + } => { + api_log!( + "Begin encoding render pass with '{}' label", + pass.label.as_deref().unwrap_or("") + ); + let res = encode_render_pass( + &mut state, + pass, + color_attachments, + depth_stencil_attachment, + timestamp_writes, + occlusion_query_set, + multiview_mask, + ); + match res.as_ref() { + Err(err) => { + api_log!("Finished encoding render pass ({err:?})") + } + Ok(_) => { + api_log!("Finished encoding render pass (success)") + } + } + res?; + } + ArcCommand::RunComputePass { + pass, + timestamp_writes, + } => { + api_log!( + "Begin encoding compute pass with '{}' label", + pass.label.as_deref().unwrap_or("") + ); + let res = encode_compute_pass(&mut state, pass, timestamp_writes); + match res.as_ref() { + Err(err) => { + api_log!("Finished encoding compute pass ({err:?})") + } + Ok(_) => { + api_log!("Finished encoding compute pass (success)") + } + } + res?; + } + _ => unreachable!(), + } + } else { + // All the other non-pass encoding routines assume the + // encoder is open, so open it if necessary. This state + // object holds an `&mut dyn hal::DynCommandEncoder`. By + // convention, a bare HAL encoder reference in + // [`EncodingState`] must always be an open encoder. + let raw_encoder = cmd_buf_data.encoder.open_if_closed()?; + let mut state = EncodingState { + device, + raw_encoder, + tracker: &mut cmd_buf_data.trackers, + buffer_memory_init_actions: &mut cmd_buf_data.buffer_memory_init_actions, + texture_memory_actions: &mut cmd_buf_data.texture_memory_actions, + as_actions: &mut cmd_buf_data.as_actions, + temp_resources: &mut cmd_buf_data.temp_resources, + indirect_draw_validation_resources: &mut cmd_buf_data + .indirect_draw_validation_resources, + snatch_guard: &snatch_guard, + debug_scope_depth: &mut debug_scope_depth, + }; + match command { + ArcCommand::CopyBufferToBuffer { + src, + src_offset, + dst, + dst_offset, + size, + } => { + copy_buffer_to_buffer( + &mut state, &src, src_offset, &dst, dst_offset, size, + )?; + } + ArcCommand::CopyBufferToTexture { src, dst, size } => { + copy_buffer_to_texture(&mut state, &src, &dst, &size)?; + } + ArcCommand::CopyTextureToBuffer { src, dst, size } => { + copy_texture_to_buffer(&mut state, &src, &dst, &size)?; + } + ArcCommand::CopyTextureToTexture { src, dst, size } => { + copy_texture_to_texture(&mut state, &src, &dst, &size)?; + } + ArcCommand::ClearBuffer { dst, offset, size } => { + clear_buffer(&mut state, dst, offset, size)?; + } + ArcCommand::ClearTexture { + dst, + subresource_range, + } => { + clear_texture_cmd(&mut state, dst, &subresource_range)?; + } + ArcCommand::WriteTimestamp { + query_set, + query_index, + } => { + write_timestamp(&mut state, query_set, query_index)?; + } + ArcCommand::ResolveQuerySet { + query_set, + start_query, + query_count, + destination, + destination_offset, + } => { + resolve_query_set( + &mut state, + query_set, + start_query, + query_count, + destination, + destination_offset, + )?; + } + ArcCommand::PushDebugGroup(label) => { + push_debug_group(&mut state, &label)?; + } + ArcCommand::PopDebugGroup => { + pop_debug_group(&mut state)?; + } + ArcCommand::InsertDebugMarker(label) => { + insert_debug_marker(&mut state, &label)?; + } + ArcCommand::BuildAccelerationStructures { blas, tlas } => { + build_acceleration_structures(&mut state, blas, tlas)?; + } + ArcCommand::TransitionResources { + buffer_transitions, + texture_transitions, + } => { + transition_resources(&mut state, buffer_transitions, texture_transitions)?; + } + ArcCommand::RunComputePass { .. } | ArcCommand::RunRenderPass { .. } => { + unreachable!() + } + } + } + } + + if debug_scope_depth > 0 { + Err(CommandEncoderError::DebugGroupError( + DebugGroupError::MissingPop, + ))?; + } + + // Close the encoder, unless it was closed already by a render or compute pass. + cmd_buf_data.encoder.close_if_open()?; + + // Note: if we want to stop tracking the swapchain texture view, + // this is the place to do it. + + Ok(()) + } + fn finish( self: &Arc, desc: &wgt::CommandBufferDescriptor