diff --git a/rustfmt.toml b/rustfmt.toml index ff1d7cf55..c1733530b 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,3 +1,4 @@ newline_style = "Native" +blank_lines_upper_bound = 2 spaces_around_ranges = true imports_layout = "HorizontalVertical" diff --git a/wgpu-native/src/command/compute.rs b/wgpu-native/src/command/compute.rs index a3f39cc7d..79f3ed5e7 100644 --- a/wgpu-native/src/command/compute.rs +++ b/wgpu-native/src/command/compute.rs @@ -1,6 +1,6 @@ use crate::{ command::bind::{Binder, LayoutChange}, - hub::HUB, + hub::{HUB, Token}, track::{Stitch, TrackerSet}, BindGroupId, BufferAddress, @@ -43,8 +43,9 @@ impl ComputePass { #[no_mangle] pub extern "C" fn wgpu_compute_pass_end_pass(pass_id: ComputePassId) -> CommandBufferId { - let mut cmb_guard = HUB.command_buffers.write(); - let pass = HUB.compute_passes.unregister(pass_id); + let mut token = Token::root(); + let (mut cmb_guard, mut token) = HUB.command_buffers.write(&mut token); + let (pass, _) = HUB.compute_passes.unregister(pass_id, &mut token); let cmb = &mut cmb_guard[pass.cmb_id.value]; // There are no transitions to be made: we've already been inserting barriers @@ -62,9 +63,11 @@ pub extern "C" fn wgpu_compute_pass_set_bind_group( offsets: *const BufferAddress, offsets_length: usize, ) { - let mut pass_guard = HUB.compute_passes.write(); + let mut token = Token::root(); + let (pipeline_layout_guard, mut token) = HUB.pipeline_layouts.read(&mut token); + let (bind_group_guard, mut token) = HUB.bind_groups.read(&mut token); + let (mut pass_guard, mut token) = HUB.compute_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; - let bind_group_guard = HUB.bind_groups.read(); let bind_group = pass .trackers @@ -93,21 +96,22 @@ pub extern "C" fn wgpu_compute_pass_set_bind_group( //Note: currently, WebGPU compute passes have synchronization defined // at a dispatch granularity, so we insert the necessary barriers here. + let (buffer_guard, mut token) = HUB.buffers.read(&mut token); + let (texture_guard, _) = HUB.textures.read(&mut token); CommandBuffer::insert_barriers( &mut pass.raw, &mut pass.trackers, &bind_group.used, Stitch::Last, - &*HUB.buffers.read(), - &*HUB.textures.read(), + &*buffer_guard, + &*texture_guard, ); if let Some((pipeline_layout_id, follow_up)) = pass.binder .provide_entry(index as usize, bind_group_id, bind_group, offsets) { - let pipeline_layout_guard = HUB.pipeline_layouts.read(); let bind_groups = iter::once(bind_group.raw.raw()).chain(follow_up.map(|bg_id| bind_group_guard[bg_id].raw.raw())); unsafe { @@ -143,8 +147,10 @@ pub extern "C" fn wgpu_compute_pass_insert_debug_marker( #[no_mangle] pub extern "C" fn wgpu_compute_pass_dispatch(pass_id: ComputePassId, x: u32, y: u32, z: u32) { + let mut token = Token::root(); + let (mut pass_guard, _) = HUB.compute_passes.write(&mut token); unsafe { - HUB.compute_passes.write()[pass_id].raw.dispatch([x, y, z]); + pass_guard[pass_id].raw.dispatch([x, y, z]); } } @@ -153,9 +159,12 @@ pub extern "C" fn wgpu_compute_pass_set_pipeline( pass_id: ComputePassId, pipeline_id: ComputePipelineId, ) { - let mut pass_guard = HUB.compute_passes.write(); + let mut token = Token::root(); + let (pipeline_layout_guard, mut token) = HUB.pipeline_layouts.read(&mut token); + let (bind_group_guard, mut token) = HUB.bind_groups.read(&mut token); + let (mut pass_guard, mut token) = HUB.compute_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; - let pipeline_guard = HUB.compute_pipelines.read(); + let (pipeline_guard, _) = HUB.compute_pipelines.read(&mut token); let pipeline = &pipeline_guard[pipeline_id]; unsafe { @@ -164,10 +173,7 @@ pub extern "C" fn wgpu_compute_pass_set_pipeline( // Rebind resources if pass.binder.pipeline_layout_id != Some(pipeline.layout_id.clone()) { - let pipeline_layout_guard = HUB.pipeline_layouts.read(); let pipeline_layout = &pipeline_layout_guard[pipeline.layout_id]; - let bind_group_guard = HUB.bind_groups.read(); - pass.binder.pipeline_layout_id = Some(pipeline.layout_id.clone()); pass.binder .reset_expectations(pipeline_layout.bind_group_layout_ids.len()); diff --git a/wgpu-native/src/command/mod.rs b/wgpu-native/src/command/mod.rs index ddaa398cc..863555998 100644 --- a/wgpu-native/src/command/mod.rs +++ b/wgpu-native/src/command/mod.rs @@ -18,7 +18,7 @@ use crate::{ RenderPassContext, RenderPassKey, }, - hub::{Storage, HUB}, + hub::{HUB, Storage, Token}, resource::TexturePlacement, swap_chain::{SwapChainLink, SwapImageEpoch}, track::{Stitch, TrackerSet}, @@ -162,8 +162,10 @@ pub struct CommandEncoderDescriptor { pub extern "C" fn wgpu_command_encoder_finish( command_encoder_id: CommandEncoderId, ) -> CommandBufferId { + let mut token = Token::root(); //TODO: actually close the last recorded command buffer - HUB.command_buffers.write()[command_encoder_id].is_recording = false; //TODO: check for the old value + let (mut comb_guard, _) = HUB.command_buffers.write(&mut token); + comb_guard[command_encoder_id].is_recording = false; //TODO: check for the old value command_encoder_id } @@ -171,12 +173,16 @@ pub fn command_encoder_begin_render_pass( command_encoder_id: CommandEncoderId, desc: &RenderPassDescriptor, ) -> RenderPass { - let device_guard = HUB.devices.read(); - let mut cmb_guard = HUB.command_buffers.write(); + let mut token = Token::root(); + let (adapter_guard, mut token) = HUB.adapters.read(&mut token); + let (device_guard, mut token) = HUB.devices.read(&mut token); + let (mut cmb_guard, mut token) = HUB.command_buffers.write(&mut token); let cmb = &mut cmb_guard[command_encoder_id]; let device = &device_guard[cmb.device_id.value]; - let texture_guard = HUB.textures.read(); - let view_guard = HUB.texture_views.read(); + + let (_, mut token) = HUB.buffers.read(&mut token); //skip token + let (texture_guard, mut token) = HUB.textures.read(&mut token); + let (view_guard, _) = HUB.texture_views.read(&mut token); let mut current_comb = device.com_allocator.extend(cmb); unsafe { @@ -188,7 +194,7 @@ pub fn command_encoder_begin_render_pass( let mut extent = None; let mut barriers = Vec::new(); - let limits = HUB.adapters.read()[device.adapter_id].physical_device.limits(); + let limits = adapter_guard[device.adapter_id].physical_device.limits(); let samples_count_limit = limits.framebuffer_color_samples_count; let color_attachments = @@ -565,13 +571,14 @@ pub extern "C" fn wgpu_command_encoder_begin_render_pass( desc: &RenderPassDescriptor, ) -> RenderPassId { let pass = command_encoder_begin_render_pass(command_encoder_id, desc); - HUB.render_passes.register_local(pass) + HUB.render_passes.register_local(pass, &mut Token::root()) } pub fn command_encoder_begin_compute_pass( command_encoder_id: CommandEncoderId, ) -> ComputePass { - let mut cmb_guard = HUB.command_buffers.write(); + let mut token = Token::root(); + let (mut cmb_guard, _) = HUB.command_buffers.write(&mut token); let cmb = &mut cmb_guard[command_encoder_id]; let raw = cmb.raw.pop().unwrap(); @@ -590,5 +597,5 @@ pub extern "C" fn wgpu_command_encoder_begin_compute_pass( command_encoder_id: CommandEncoderId, ) -> ComputePassId { let pass = command_encoder_begin_compute_pass(command_encoder_id); - HUB.compute_passes.register_local(pass) + HUB.compute_passes.register_local(pass, &mut Token::root()) } diff --git a/wgpu-native/src/command/render.rs b/wgpu-native/src/command/render.rs index 912227751..eb6a674b5 100644 --- a/wgpu-native/src/command/render.rs +++ b/wgpu-native/src/command/render.rs @@ -2,7 +2,7 @@ use crate::{ command::bind::{Binder, LayoutChange}, conv, device::{RenderPassContext, BIND_BUFFER_ALIGNMENT, MAX_VERTEX_BUFFERS}, - hub::HUB, + hub::{Token, HUB}, pipeline::{IndexFormat, InputStepMode, PipelineFlags}, resource::BufferUsage, track::{Stitch, TrackerSet}, @@ -174,13 +174,16 @@ impl RenderPass { #[no_mangle] pub extern "C" fn wgpu_render_pass_end_pass(pass_id: RenderPassId) -> CommandBufferId { - let mut cmb_guard = HUB.command_buffers.write(); - let mut pass = HUB.render_passes.unregister(pass_id); + let mut token = Token::root(); + let (mut cmb_guard, mut token) = HUB.command_buffers.write(&mut token); + let (mut pass, mut token) = HUB.render_passes.unregister(pass_id, &mut token); unsafe { pass.raw.end_render_pass(); } pass.trackers.optimize(); let cmb = &mut cmb_guard[pass.cmb_id.value]; + let (buffer_guard, mut token) = HUB.buffers.read(&mut token); + let (texture_guard, _) = HUB.textures.read(&mut token); match cmb.raw.last_mut() { Some(ref mut last) => { @@ -189,8 +192,8 @@ pub extern "C" fn wgpu_render_pass_end_pass(pass_id: RenderPassId) -> CommandBuf &mut cmb.trackers, &pass.trackers, Stitch::Last, - &*HUB.buffers.read(), - &*HUB.textures.read(), + &*buffer_guard, + &*texture_guard, ); unsafe { last.finish() }; } @@ -211,9 +214,12 @@ pub extern "C" fn wgpu_render_pass_set_bind_group( offsets: *const BufferAddress, offsets_length: usize, ) { - let mut pass_guard = HUB.render_passes.write(); + let mut token = Token::root(); + let (pipeline_layout_guard, mut token) = HUB.pipeline_layouts.read(&mut token); + let (bind_group_guard, mut token) = HUB.bind_groups.read(&mut token); + + let (mut pass_guard, _) = HUB.render_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; - let bind_group_guard = HUB.bind_groups.read(); let bind_group = pass .trackers @@ -246,7 +252,6 @@ pub extern "C" fn wgpu_render_pass_set_bind_group( pass.binder .provide_entry(index as usize, bind_group_id, bind_group, offsets) { - let pipeline_layout_guard = HUB.pipeline_layouts.read(); let bind_groups = iter::once(bind_group.raw.raw()).chain(follow_up.map(|bg_id| bind_group_guard[bg_id].raw.raw())); unsafe { @@ -283,8 +288,9 @@ pub extern "C" fn wgpu_render_pass_set_index_buffer( buffer_id: BufferId, offset: BufferAddress, ) { - let mut pass_guard = HUB.render_passes.write(); - let buffer_guard = HUB.buffers.read(); + let mut token = Token::root(); + let (mut pass_guard, mut token) = HUB.render_passes.write(&mut token); + let (buffer_guard, _) = HUB.buffers.read(&mut token); let pass = &mut pass_guard[pass_id]; let buffer = pass @@ -315,8 +321,9 @@ pub extern "C" fn wgpu_render_pass_set_vertex_buffers( offsets: *const BufferAddress, length: usize, ) { - let mut pass_guard = HUB.render_passes.write(); - let buffer_guard = HUB.buffers.read(); + let mut token = Token::root(); + let (mut pass_guard, mut token) = HUB.render_passes.write(&mut token); + let (buffer_guard, _) = HUB.buffers.read(&mut token); let buffers = unsafe { slice::from_raw_parts(buffers, length) }; let offsets = unsafe { slice::from_raw_parts(offsets, length) }; @@ -352,7 +359,8 @@ pub extern "C" fn wgpu_render_pass_draw( first_vertex: u32, first_instance: u32, ) { - let mut pass_guard = HUB.render_passes.write(); + let mut token = Token::root(); + let (mut pass_guard, _) = HUB.render_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; pass.is_ready().unwrap(); @@ -376,7 +384,8 @@ pub extern "C" fn wgpu_render_pass_draw_indexed( base_vertex: i32, first_instance: u32, ) { - let mut pass_guard = HUB.render_passes.write(); + let mut token = Token::root(); + let (mut pass_guard, _) = HUB.render_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; pass.is_ready().unwrap(); @@ -398,9 +407,12 @@ pub extern "C" fn wgpu_render_pass_set_pipeline( pass_id: RenderPassId, pipeline_id: RenderPipelineId, ) { - let mut pass_guard = HUB.render_passes.write(); + let mut token = Token::root(); + let (pipeline_layout_guard, mut token) = HUB.pipeline_layouts.read(&mut token); + let (bind_group_guard, mut token) = HUB.bind_groups.read(&mut token); + let (mut pass_guard, mut token) = HUB.render_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; - let pipeline_guard = HUB.render_pipelines.read(); + let (pipeline_guard, mut token) = HUB.render_pipelines.read(&mut token); let pipeline = &pipeline_guard[pipeline_id]; assert!( @@ -420,10 +432,7 @@ pub extern "C" fn wgpu_render_pass_set_pipeline( // Rebind resource if pass.binder.pipeline_layout_id != Some(pipeline.layout_id.clone()) { - let pipeline_layout_guard = HUB.pipeline_layouts.read(); let pipeline_layout = &pipeline_layout_guard[pipeline.layout_id]; - let bind_group_guard = HUB.bind_groups.read(); - pass.binder.pipeline_layout_id = Some(pipeline.layout_id.clone()); pass.binder .reset_expectations(pipeline_layout.bind_group_layout_ids.len()); @@ -455,7 +464,7 @@ pub extern "C" fn wgpu_render_pass_set_pipeline( pass.index_state.update_limit(); if let Some((buffer_id, ref range)) = pass.index_state.bound_buffer_view { - let buffer_guard = HUB.buffers.read(); + let (buffer_guard, _) = HUB.buffers.read(&mut token); let buffer = pass .trackers .buffers @@ -487,7 +496,8 @@ pub extern "C" fn wgpu_render_pass_set_pipeline( #[no_mangle] pub extern "C" fn wgpu_render_pass_set_blend_color(pass_id: RenderPassId, color: &Color) { - let mut pass_guard = HUB.render_passes.write(); + let mut token = Token::root(); + let (mut pass_guard, _) = HUB.render_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; pass.blend_color_status = OptionalState::Set; @@ -499,7 +509,8 @@ pub extern "C" fn wgpu_render_pass_set_blend_color(pass_id: RenderPassId, color: #[no_mangle] pub extern "C" fn wgpu_render_pass_set_stencil_reference(pass_id: RenderPassId, value: u32) { - let mut pass_guard = HUB.render_passes.write(); + let mut token = Token::root(); + let (mut pass_guard, _) = HUB.render_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; pass.stencil_reference_status = OptionalState::Set; @@ -519,7 +530,8 @@ pub extern "C" fn wgpu_render_pass_set_viewport( min_depth: f32, max_depth: f32, ) { - let mut pass_guard = HUB.render_passes.write(); + let mut token = Token::root(); + let (mut pass_guard, _) = HUB.render_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; unsafe { @@ -549,7 +561,8 @@ pub extern "C" fn wgpu_render_pass_set_scissor_rect( w: u32, h: u32, ) { - let mut pass_guard = HUB.render_passes.write(); + let mut token = Token::root(); + let (mut pass_guard, _) = HUB.render_passes.write(&mut token); let pass = &mut pass_guard[pass_id]; unsafe { diff --git a/wgpu-native/src/command/transfer.rs b/wgpu-native/src/command/transfer.rs index a6f686a8c..08a6d68fe 100644 --- a/wgpu-native/src/command/transfer.rs +++ b/wgpu-native/src/command/transfer.rs @@ -1,7 +1,7 @@ use crate::{ conv, device::{all_buffer_stages, all_image_stages}, - hub::HUB, + hub::{HUB, Token}, resource::TexturePlacement, swap_chain::SwapChainLink, BufferAddress, @@ -73,9 +73,10 @@ pub extern "C" fn wgpu_command_buffer_copy_buffer_to_buffer( dst_offset: BufferAddress, size: BufferAddress, ) { - let mut cmb_guard = HUB.command_buffers.write(); + let mut token = Token::root(); + let (mut cmb_guard, mut token) = HUB.command_buffers.write(&mut token); let cmb = &mut cmb_guard[command_buffer_id]; - let buffer_guard = HUB.buffers.read(); + let (buffer_guard, _) = HUB.buffers.read(&mut token); // we can't hold both src_pending and dst_pending in scope because they // borrow the buffer tracker mutably... let mut barriers = Vec::new(); @@ -125,10 +126,11 @@ pub extern "C" fn wgpu_command_buffer_copy_buffer_to_texture( destination: &TextureCopyView, copy_size: Extent3d, ) { - let mut cmb_guard = HUB.command_buffers.write(); + let mut token = Token::root(); + let (mut cmb_guard, mut token) = HUB.command_buffers.write(&mut token); let cmb = &mut cmb_guard[command_buffer_id]; - let buffer_guard = HUB.buffers.read(); - let texture_guard = HUB.textures.read(); + let (buffer_guard, mut token) = HUB.buffers.read(&mut token); + let (texture_guard, _) = HUB.textures.read(&mut token); let aspects = texture_guard[destination.texture].full_range.aspects; let (src_buffer, src_pending) = cmb @@ -202,10 +204,11 @@ pub extern "C" fn wgpu_command_buffer_copy_texture_to_buffer( destination: &BufferCopyView, copy_size: Extent3d, ) { - let mut cmb_guard = HUB.command_buffers.write(); + let mut token = Token::root(); + let (mut cmb_guard, mut token) = HUB.command_buffers.write(&mut token); let cmb = &mut cmb_guard[command_buffer_id]; - let buffer_guard = HUB.buffers.read(); - let texture_guard = HUB.textures.read(); + let (buffer_guard, mut token) = HUB.buffers.read(&mut token); + let (texture_guard, _) = HUB.textures.read(&mut token); let aspects = texture_guard[source.texture].full_range.aspects; let (src_texture, src_pending) = cmb.trackers.textures.use_replace( @@ -278,9 +281,11 @@ pub extern "C" fn wgpu_command_buffer_copy_texture_to_texture( destination: &TextureCopyView, copy_size: Extent3d, ) { - let mut cmb_guard = HUB.command_buffers.write(); + let mut token = Token::root(); + let (mut cmb_guard, mut token) = HUB.command_buffers.write(&mut token); let cmb = &mut cmb_guard[command_buffer_id]; - let texture_guard = HUB.textures.read(); + let (_, mut token) = HUB.buffers.read(&mut token); // skip token + let (texture_guard, _) = HUB.textures.read(&mut token); // we can't hold both src_pending and dst_pending in scope because they // borrow the buffer tracker mutably... let mut barriers = Vec::new(); diff --git a/wgpu-native/src/device.rs b/wgpu-native/src/device.rs index e4fc8c4de..bc824f3cf 100644 --- a/wgpu-native/src/device.rs +++ b/wgpu-native/src/device.rs @@ -2,7 +2,7 @@ use crate::{ binding_model, command, conv, - hub::HUB, + hub::{HUB, Root, Token}, pipeline, resource, swap_chain, @@ -253,14 +253,27 @@ impl PendingResources { } } +type DeviceToken<'a> = Token<'a, Device>; + impl PendingResources { - fn triage_referenced(&mut self, trackers: &mut TrackerSet) { + fn triage_referenced(&mut self, trackers: &mut TrackerSet, mut token: &mut DeviceToken) { + // Before destruction, a resource is expected to have the following strong refs: + // 1. in resource itself + // 2. in the device tracker + // 3. in this list + const MIN_REFS: usize = 4; + + if self.referenced.iter().all(|r| r.1.load() >= MIN_REFS) { + return; + } + //TODO: lock less, if possible + let (mut bind_group_guard, mut token) = HUB.bind_groups.write(&mut token); + let (mut buffer_guard, mut token) = HUB.buffers.write(&mut token); + let (mut texture_guard, mut token) = HUB.textures.write(&mut token); + let (mut teview_view_guard, _) = HUB.texture_views.write(&mut token); + for i in (0 .. self.referenced.len()).rev() { let num_refs = self.referenced[i].1.load(); - // Before destruction, a resource is expected to have the following strong refs: - // 1. in resource itself - // 2. in the device tracker - // 3. in this list if num_refs <= 3 { let resource_id = self.referenced.swap_remove(i).0; assert_eq!( @@ -270,16 +283,20 @@ impl PendingResources { ); let (life_guard, resource) = match resource_id { ResourceId::Buffer(id) => { - if HUB.buffers.read()[id].pending_map_operation.is_some() { + if buffer_guard[id].pending_map_operation.is_some() { continue } trackers.buffers.remove(id); - let buf = HUB.buffers.unregister(id); + let buf = buffer_guard.remove(id); + #[cfg(feature = "local")] + HUB.buffers.identity.lock().free(id); (buf.life_guard, NativeResource::Buffer(buf.raw, buf.memory)) } ResourceId::Texture(id) => { trackers.textures.remove(id); - let tex = HUB.textures.unregister(id); + let tex = texture_guard.remove(id); + #[cfg(feature = "local")] + HUB.textures.identity.lock().free(id); let memory = match tex.placement { // swapchain-owned images don't need explicit destruction resource::TexturePlacement::SwapChain(_) => continue, @@ -290,12 +307,16 @@ impl PendingResources { } ResourceId::TextureView(id) => { trackers.views.remove(id); - let view = HUB.texture_views.unregister(id); + let view = teview_view_guard.remove(id); + #[cfg(feature = "local")] + HUB.texture_views.identity.lock().free(id); (view.life_guard, NativeResource::ImageView(view.raw)) } ResourceId::BindGroup(id) => { trackers.bind_groups.remove(id); - let bind_group = HUB.bind_groups.unregister(id); + let bind_group = bind_group_guard.remove(id); + #[cfg(feature = "local")] + HUB.bind_groups.identity.lock().free(id); (bind_group.life_guard, NativeResource::DescriptorSet(bind_group.raw)) } }; @@ -309,8 +330,13 @@ impl PendingResources { } } } + } - let buffer_guard = HUB.buffers.read(); + fn triage_mapped(&mut self, mut token: &mut DeviceToken) { + if self.mapped.is_empty() { + return; + } + let (buffer_guard, _) = HUB.buffers.read(&mut token); for stored in self.mapped.drain(..) { let resource_id = stored.value; @@ -338,8 +364,9 @@ impl PendingResources { FramebufferKey, ::Framebuffer, >, + mut token: &mut DeviceToken, ) { - let texture_view_guard = HUB.texture_views.read(); + let (texture_view_guard, _) = HUB.texture_views.read(&mut token); let remove_list = framebuffers .keys() .filter_map(|key| { @@ -375,11 +402,15 @@ impl PendingResources { fn handle_mapping( &mut self, raw: &::Device, + mut token: &mut DeviceToken, ) -> Vec { + if self.ready_to_map.is_empty() { + return Vec::new() + } + let (mut buffer_guard, _) = HUB.buffers.write(&mut token); self.ready_to_map .drain(..) .map(|buffer_id| { - let mut buffer_guard = HUB.buffers.write(); let buffer = &mut buffer_guard[buffer_id]; let operation = buffer.pending_map_operation.take().unwrap(); let result = match operation { @@ -505,14 +536,17 @@ impl Device { } impl Device { - fn maintain(&self, force_wait: bool) -> Vec { + fn maintain( + &self, force_wait: bool, token: &mut DeviceToken, + ) -> Vec { let mut pending = self.pending.lock(); let mut trackers = self.trackers.lock(); - pending.triage_referenced(&mut *trackers); - pending.triage_framebuffers(&mut *self.framebuffers.lock()); + pending.triage_referenced(&mut *trackers, token); + pending.triage_mapped(token); + pending.triage_framebuffers(&mut *self.framebuffers.lock(), token); let last_done = pending.cleanup(&self.raw, &self.mem_allocator, &self.desc_allocator, force_wait); - let callbacks = pending.handle_mapping(&self.raw); + let callbacks = pending.handle_mapping(&self.raw, token); unsafe { self.desc_allocator @@ -555,7 +589,7 @@ pub fn device_create_buffer( device_id: DeviceId, desc: &resource::BufferDescriptor, ) -> resource::Buffer { - let device_guard = HUB.devices.read(); + let (device_guard, _) = HUB.devices.read(&mut Token::root()); let device = &device_guard[device_id]; let (usage, _memory_properties) = conv::map_buffer_usage(desc.usage); @@ -612,8 +646,10 @@ pub fn device_track_buffer( buffer_id: BufferId, ref_count: RefCount, flags: resource::BufferUsage, + token: &mut Token, ) { - let ok = HUB.devices.read()[device_id] + let (device_guard, _) = HUB.devices.read(token); + let ok = device_guard[device_id] .trackers .lock() .buffers @@ -629,8 +665,9 @@ pub extern "C" fn wgpu_device_create_buffer( ) -> BufferId { let buffer = device_create_buffer(device_id, desc); let ref_count = buffer.life_guard.ref_count.clone(); - let id = HUB.buffers.register_local(buffer); - device_track_buffer(device_id, id, ref_count, resource::BufferUsage::empty()); + let mut token = Token::root(); + let id = HUB.buffers.register_local(buffer, &mut token); + device_track_buffer(device_id, id, ref_count, resource::BufferUsage::empty(), &mut token); id } @@ -644,9 +681,10 @@ pub extern "C" fn wgpu_device_create_buffer_mapped( let mut desc = desc.clone(); desc.usage |= resource::BufferUsage::MAP_WRITE; let mut buffer = device_create_buffer(device_id, &desc); + let mut token = Token::root(); { - let device_guard = HUB.devices.read(); + let (device_guard, _) = HUB.devices.read(&mut token); let device = &device_guard[device_id]; match map_buffer( @@ -668,15 +706,16 @@ pub extern "C" fn wgpu_device_create_buffer_mapped( } let ref_count = buffer.life_guard.ref_count.clone(); - let id = HUB.buffers.register_local(buffer); - device_track_buffer(device_id, id, ref_count, resource::BufferUsage::MAP_WRITE); + let id = HUB.buffers.register_local(buffer, &mut token); + device_track_buffer(device_id, id, ref_count, resource::BufferUsage::MAP_WRITE, &mut token); id } #[no_mangle] pub extern "C" fn wgpu_buffer_destroy(buffer_id: BufferId) { - let device_guard = HUB.devices.read(); - let buffer_guard = HUB.buffers.read(); + let mut token = Token::root(); + let (device_guard, mut token) = HUB.devices.read(&mut token); + let (buffer_guard, _) = HUB.buffers.read(&mut token); let buffer = &buffer_guard[buffer_id]; device_guard[buffer.device_id.value].pending.lock().destroy( ResourceId::Buffer(buffer_id), @@ -692,7 +731,8 @@ pub fn device_create_texture( let format = conv::map_texture_format(desc.format); let aspects = format.surface_desc().aspects; let usage = conv::map_texture_usage(desc.usage, aspects); - let device_guard = HUB.devices.read(); + + let (device_guard, _) = HUB.devices.read(&mut Token::root()); let device = &device_guard[device_id]; assert!((desc.mip_level_count as usize) < MAX_MIP_LEVELS); @@ -750,8 +790,10 @@ pub fn device_track_texture( texture_id: TextureId, ref_count: RefCount, full_range: hal::image::SubresourceRange, + token: &mut Token, ) { - let ok = HUB.devices.read()[device_id] + let (device_guard, _) = HUB.devices.read(token); + let ok = device_guard[device_id] .trackers .lock() .textures @@ -773,8 +815,9 @@ pub extern "C" fn wgpu_device_create_texture( let texture = device_create_texture(device_id, desc); let ref_count = texture.life_guard.ref_count.clone(); let range = texture.full_range.clone(); - let id = HUB.textures.register_local(texture); - device_track_texture(device_id, id, ref_count, range); + let mut token = Token::root(); + let id = HUB.textures.register_local(texture, &mut token); + device_track_texture(device_id, id, ref_count, range, &mut token); id } @@ -783,9 +826,10 @@ pub fn texture_create_view( format: resource::TextureFormat, view_kind: hal::image::ViewKind, range: hal::image::SubresourceRange, + token: &mut Token, ) -> resource::TextureView { - let device_guard = HUB.devices.read(); - let texture_guard = HUB.textures.read(); + let (device_guard, mut token) = HUB.devices.read(token); + let (texture_guard, _) = HUB.textures.read(&mut token); let texture = &texture_guard[texture_id]; let raw = unsafe { @@ -816,9 +860,17 @@ pub fn texture_create_view( } } -pub fn device_track_view(texture_id: TextureId, view_id: TextureViewId, ref_count: RefCount) { - let device_id = HUB.textures.read()[texture_id].device_id.value; - let ok = HUB.devices.read()[device_id] +pub fn device_track_view( + texture_id: TextureId, + view_id: TextureViewId, + ref_count: RefCount, + token: &mut Token, +) { + let (device_guard, mut token) = HUB.devices.read(token); + let (texture_guard, _) = HUB.textures.read(&mut token); + + let device_id = texture_guard[texture_id].device_id.value; + let ok = device_guard[device_id] .trackers .lock() .views @@ -832,6 +884,7 @@ pub extern "C" fn wgpu_texture_create_view( texture_id: TextureId, desc: &resource::TextureViewDescriptor, ) -> TextureViewId { + let mut token = Token::root(); let view = texture_create_view( texture_id, desc.format, @@ -842,18 +895,20 @@ pub extern "C" fn wgpu_texture_create_view( layers: desc.base_array_layer as u16 .. (desc.base_array_layer + desc.array_count) as u16, }, + &mut token, ); let ref_count = view.life_guard.ref_count.clone(); - let id = HUB.texture_views.register_local(view); - device_track_view(texture_id, id, ref_count); + let id = HUB.texture_views.register_local(view, &mut token); + device_track_view(texture_id, id, ref_count, &mut token); id } #[cfg(feature = "local")] #[no_mangle] pub extern "C" fn wgpu_texture_create_default_view(texture_id: TextureId) -> TextureViewId { + let mut token = Token::root(); let (format, view_kind, range) = { - let texture_guard = HUB.textures.read(); + let (texture_guard, _) = HUB.textures.read(&mut token); let texture = &texture_guard[texture_id]; let view_kind = match texture.kind { hal::image::Kind::D1(_, 1) => hal::image::ViewKind::D1, @@ -864,17 +919,18 @@ pub extern "C" fn wgpu_texture_create_default_view(texture_id: TextureId) -> Tex }; (texture.format, view_kind, texture.full_range.clone()) }; - let view = texture_create_view(texture_id, format, view_kind, range); + let view = texture_create_view(texture_id, format, view_kind, range, &mut token); let ref_count = view.life_guard.ref_count.clone(); - let id = HUB.texture_views.register_local(view); - device_track_view(texture_id, id, ref_count); + let id = HUB.texture_views.register_local(view, &mut token); + device_track_view(texture_id, id, ref_count, &mut token); id } #[no_mangle] pub extern "C" fn wgpu_texture_destroy(texture_id: TextureId) { - let device_guard = HUB.devices.read(); - let texture_guard = HUB.textures.read(); + let mut token = Token::root(); + let (device_guard, mut token) = HUB.devices.read(&mut token); + let (texture_guard, _) = HUB.textures.read(&mut token); let texture = &texture_guard[texture_id]; device_guard[texture.device_id.value] .pending @@ -887,9 +943,10 @@ pub extern "C" fn wgpu_texture_destroy(texture_id: TextureId) { #[no_mangle] pub extern "C" fn wgpu_texture_view_destroy(texture_view_id: TextureViewId) { - let device_guard = HUB.devices.read(); - let texture_guard = HUB.textures.read(); - let texture_view_guard = HUB.texture_views.read(); + let mut token = Token::root(); + let (device_guard, mut token) = HUB.devices.read(&mut token); + let (texture_guard, mut token) = HUB.textures.read(&mut token); + let (texture_view_guard, _) = HUB.texture_views.read(&mut token); let view = &texture_view_guard[texture_view_id]; let device_id = texture_guard[view.texture_id.value].device_id.value; device_guard[device_id].pending.lock().destroy( @@ -901,8 +958,9 @@ pub extern "C" fn wgpu_texture_view_destroy(texture_view_id: TextureViewId) { pub fn device_create_sampler( device_id: DeviceId, desc: &resource::SamplerDescriptor, + token: &mut Token, ) -> resource::Sampler { - let device_guard = HUB.devices.read(); + let (device_guard, _) = HUB.devices.read(token); let device = &device_guard[device_id]; let info = hal::image::SamplerInfo { @@ -935,13 +993,15 @@ pub extern "C" fn wgpu_device_create_sampler( device_id: DeviceId, desc: &resource::SamplerDescriptor, ) -> SamplerId { - let sampler = device_create_sampler(device_id, desc); - HUB.samplers.register_local(sampler) + let mut token = Token::root(); + let sampler = device_create_sampler(device_id, desc, &mut token); + HUB.samplers.register_local(sampler, &mut token) } pub fn device_create_bind_group_layout( device_id: DeviceId, desc: &binding_model::BindGroupLayoutDescriptor, + token: &mut Token, ) -> binding_model::BindGroupLayout { let bindings = unsafe { slice::from_raw_parts(desc.bindings, desc.bindings_length) }; @@ -957,7 +1017,8 @@ pub fn device_create_bind_group_layout( .collect::>(); //TODO: avoid heap allocation let raw = unsafe { - HUB.devices.read()[device_id] + let (device_guard, _) = HUB.devices.read(token); + device_guard[device_id] .raw .create_descriptor_set_layout(&raw_bindings, &[]) .unwrap() @@ -984,18 +1045,20 @@ pub extern "C" fn wgpu_device_create_bind_group_layout( device_id: DeviceId, desc: &binding_model::BindGroupLayoutDescriptor, ) -> BindGroupLayoutId { - let layout = device_create_bind_group_layout(device_id, desc); - HUB.bind_group_layouts.register_local(layout) + let mut token = Token::root(); + let layout = device_create_bind_group_layout(device_id, desc, &mut token); + HUB.bind_group_layouts.register_local(layout, &mut token) } pub fn device_create_pipeline_layout( device_id: DeviceId, desc: &binding_model::PipelineLayoutDescriptor, + token: &mut Token, ) -> binding_model::PipelineLayout { - let device_guard = HUB.devices.read(); + let (device_guard, mut token) = HUB.devices.read(token); let bind_group_layout_ids = unsafe { slice::from_raw_parts(desc.bind_group_layouts, desc.bind_group_layouts_length) }; - let bind_group_layout_guard = HUB.bind_group_layouts.read(); + let (bind_group_layout_guard, _) = HUB.bind_group_layouts.read(&mut token); let descriptor_set_layouts = bind_group_layout_ids .iter() .map(|&id| &bind_group_layout_guard[id].raw); @@ -1020,17 +1083,19 @@ pub extern "C" fn wgpu_device_create_pipeline_layout( device_id: DeviceId, desc: &binding_model::PipelineLayoutDescriptor, ) -> PipelineLayoutId { - let layout = device_create_pipeline_layout(device_id, desc); - HUB.pipeline_layouts.register_local(layout) + let mut token = Token::root(); + let layout = device_create_pipeline_layout(device_id, desc, &mut token); + HUB.pipeline_layouts.register_local(layout, &mut token) } pub fn device_create_bind_group( device_id: DeviceId, desc: &binding_model::BindGroupDescriptor, + token: &mut Token, ) -> binding_model::BindGroup { - let device_guard = HUB.devices.read(); + let (device_guard, mut token) = HUB.devices.read(token); let device = &device_guard[device_id]; - let bind_group_layout_guard = HUB.bind_group_layouts.read(); + let (bind_group_layout_guard, _) = HUB.bind_group_layouts.read(&mut token); let bind_group_layout = &bind_group_layout_guard[desc.layout]; let bindings = unsafe { slice::from_raw_parts(desc.bindings, desc.bindings_length as usize) }; assert_eq!(bindings.len(), bind_group_layout.bindings.len()); @@ -1050,9 +1115,10 @@ pub fn device_create_bind_group( desc_sets.pop().unwrap() }; - let buffer_guard = HUB.buffers.read(); - let sampler_guard = HUB.samplers.read(); - let texture_view_guard = HUB.texture_views.read(); + let (buffer_guard, mut token) = HUB.buffers.read(&mut token); + let (_, mut token) = HUB.textures.read(&mut token); //skip token + let (texture_view_guard, mut token) = HUB.texture_views.read(&mut token); + let (sampler_guard, _) = HUB.samplers.read(&mut token); //TODO: group writes into contiguous sections let mut writes = Vec::new(); @@ -1148,8 +1214,10 @@ pub fn device_track_bind_group( device_id: DeviceId, bind_group_id: BindGroupId, ref_count: RefCount, + token: &mut Token, ) { - let ok = HUB.devices.read()[device_id] + let (device_guard, _) = HUB.devices.read(token); + let ok = device_guard[device_id] .trackers .lock() .bind_groups @@ -1163,18 +1231,19 @@ pub extern "C" fn wgpu_device_create_bind_group( device_id: DeviceId, desc: &binding_model::BindGroupDescriptor, ) -> BindGroupId { - let bind_group = device_create_bind_group(device_id, desc); + let mut token = Token::root(); + let bind_group = device_create_bind_group(device_id, desc, &mut token); let ref_count = bind_group.life_guard.ref_count.clone(); - let id = HUB.bind_groups.register_local(bind_group); - device_track_bind_group(device_id, id, ref_count); - + let id = HUB.bind_groups.register_local(bind_group, &mut token); + device_track_bind_group(device_id, id, ref_count, &mut token); id } #[no_mangle] pub extern "C" fn wgpu_bind_group_destroy(bind_group_id: BindGroupId) { - let device_guard = HUB.devices.read(); - let bind_group_guard = HUB.bind_groups.read(); + let mut token = Token::root(); + let (device_guard, mut token) = HUB.devices.read(&mut token); + let (bind_group_guard, _) = HUB.bind_groups.read(&mut token); let bind_group = &bind_group_guard[bind_group_id]; device_guard[bind_group.device_id.value].pending.lock().destroy( ResourceId::BindGroup(bind_group_id), @@ -1185,10 +1254,12 @@ pub extern "C" fn wgpu_bind_group_destroy(bind_group_id: BindGroupId) { pub fn device_create_shader_module( device_id: DeviceId, desc: &pipeline::ShaderModuleDescriptor, + token: &mut Token, ) -> ShaderModule { let spv = unsafe { slice::from_raw_parts(desc.code.bytes, desc.code.length) }; + let (device_guard, _) = HUB.devices.read(token); let shader = unsafe { - HUB.devices.read()[device_id] + device_guard[device_id] .raw .create_shader_module(spv) .unwrap() @@ -1203,15 +1274,17 @@ pub extern "C" fn wgpu_device_create_shader_module( device_id: DeviceId, desc: &pipeline::ShaderModuleDescriptor, ) -> ShaderModuleId { - let module = device_create_shader_module(device_id, desc); - HUB.shader_modules.register_local(module) + let mut token = Token::root(); + let module = device_create_shader_module(device_id, desc, &mut token); + HUB.shader_modules.register_local(module, &mut token) } pub fn device_create_command_encoder( device_id: DeviceId, _desc: &command::CommandEncoderDescriptor, + token: &mut Token, ) -> command::CommandBuffer { - let device_guard = HUB.devices.read(); + let (device_guard, _) = HUB.devices.read(token); let device = &device_guard[device_id]; let dev_stored = Stored { @@ -1234,8 +1307,9 @@ pub extern "C" fn wgpu_device_create_command_encoder( device_id: DeviceId, desc: &command::CommandEncoderDescriptor, ) -> CommandEncoderId { - let cmb = device_create_command_encoder(device_id, desc); - HUB.command_buffers.register_local(cmb) + let mut token = Token::root(); + let cmb = device_create_command_encoder(device_id, desc, &mut token); + HUB.command_buffers.register_local(cmb, &mut token) } #[no_mangle] @@ -1253,8 +1327,9 @@ pub extern "C" fn wgpu_queue_submit( unsafe { slice::from_raw_parts(command_buffers, command_buffers_length) }; let (submit_index, fence) = { - let surface_guard = HUB.surfaces.read(); - let mut device_guard = HUB.devices.write(); + let mut token = Token::root(); + let (surface_guard, mut token) = HUB.surfaces.read(&mut token); + let (mut device_guard, mut token) = HUB.devices.write(&mut token); let device = &mut device_guard[queue_id]; let mut trackers = device.trackers.lock(); let mut wait_semaphores = Vec::new(); @@ -1268,11 +1343,11 @@ pub extern "C" fn wgpu_queue_submit( // native command buffer of the previous chain instead of always creating // a temporary one, since the chains are not finished. { - let mut command_buffer_guard = HUB.command_buffers.write(); - let buffer_guard = HUB.buffers.read(); - let texture_guard = HUB.textures.read(); - let texture_view_guard = HUB.texture_views.read(); - let bind_group_guard = HUB.bind_groups.read(); + let (mut command_buffer_guard, mut token) = HUB.command_buffers.write(&mut token); + let (bind_group_guard, mut token) = HUB.bind_groups.read(&mut token); + let (buffer_guard, mut token) = HUB.buffers.read(&mut token); + let (texture_guard, mut token) = HUB.textures.read(&mut token); + let (texture_view_guard, _) = HUB.texture_views.read(&mut token); // finish all the command buffers first for &cmb_id in command_buffer_ids { @@ -1352,7 +1427,7 @@ pub extern "C" fn wgpu_queue_submit( // now prepare the GPU submission let fence = device.raw.create_fence(false).unwrap(); { - let command_buffer_guard = HUB.command_buffers.read(); + let (command_buffer_guard, _) = HUB.command_buffers.read(&mut token); let submission = hal::queue::Submission::<_, _, &[::Semaphore]> { //TODO: may `OneShot` be enough? @@ -1375,10 +1450,11 @@ pub extern "C" fn wgpu_queue_submit( // No need for write access to the device from here on out let callbacks = { - let device_guard = HUB.devices.read(); + let mut token = Token::root(); + let (device_guard, mut token) = HUB.devices.read(&mut token); let device = &device_guard[queue_id]; - let callbacks = device.maintain(false); + let callbacks = device.maintain(false, &mut token); device.pending.lock().active.alloc().init(ActiveSubmission { index: submit_index, fence, @@ -1388,7 +1464,7 @@ pub extern "C" fn wgpu_queue_submit( // finally, return the command buffers to the allocator for &cmb_id in command_buffer_ids { - let cmd_buf = HUB.command_buffers.unregister(cmb_id); + let (cmd_buf, _) = HUB.command_buffers.unregister(cmb_id, &mut token); device.com_allocator.after_submit(cmd_buf, submit_index); } @@ -1401,17 +1477,18 @@ pub extern "C" fn wgpu_queue_submit( pub fn device_create_render_pipeline( device_id: DeviceId, desc: &pipeline::RenderPipelineDescriptor, + token: &mut Token, ) -> pipeline::RenderPipeline { let sc = desc.sample_count; assert!(sc == 1 || sc == 2 || sc == 4 || sc == 8 || sc == 16 || sc == 32, "Invalid sample_count of {}", sc); let sc = sc as u8; - let device_guard = HUB.devices.read(); + let (device_guard, mut token) = HUB.devices.read(token); let device = &device_guard[device_id]; - let pipeline_layout_guard = HUB.pipeline_layouts.read(); + let (pipeline_layout_guard, mut token) = HUB.pipeline_layouts.read(&mut token); let layout = &pipeline_layout_guard[desc.layout].raw; - let shader_module_guard = HUB.shader_modules.read(); + let (shader_module_guard, _) = HUB.shader_modules.read(&mut token); let color_states = unsafe { slice::from_raw_parts(desc.color_states, desc.color_states_length) }; @@ -1646,20 +1723,22 @@ pub extern "C" fn wgpu_device_create_render_pipeline( device_id: DeviceId, desc: &pipeline::RenderPipelineDescriptor, ) -> RenderPipelineId { - let pipeline = device_create_render_pipeline(device_id, desc); - HUB.render_pipelines.register_local(pipeline) + let mut token = Token::root(); + let pipeline = device_create_render_pipeline(device_id, desc, &mut token); + HUB.render_pipelines.register_local(pipeline, &mut token) } pub fn device_create_compute_pipeline( device_id: DeviceId, desc: &pipeline::ComputePipelineDescriptor, + token: &mut Token, ) -> pipeline::ComputePipeline { - let device_guard = HUB.devices.read(); + let (device_guard, mut token) = HUB.devices.read(token); let device = &device_guard[device_id].raw; - let pipeline_layout_guard = HUB.pipeline_layouts.read(); + let (pipeline_layout_guard, mut token) = HUB.pipeline_layouts.read(&mut token); let layout = &pipeline_layout_guard[desc.layout].raw; let pipeline_stage = &desc.compute_stage; - let shader_module_guard = HUB.shader_modules.read(); + let (shader_module_guard, _) = HUB.shader_modules.read(&mut token); let shader = hal::pso::EntryPoint:: { entry: unsafe { ffi::CStr::from_ptr(pipeline_stage.entry_point) } @@ -1700,24 +1779,26 @@ pub extern "C" fn wgpu_device_create_compute_pipeline( device_id: DeviceId, desc: &pipeline::ComputePipelineDescriptor, ) -> ComputePipelineId { - let pipeline = device_create_compute_pipeline(device_id, desc); - HUB.compute_pipelines.register_local(pipeline) + let mut token = Token::root(); + let pipeline = device_create_compute_pipeline(device_id, desc, &mut token); + HUB.compute_pipelines.register_local(pipeline, &mut token) } pub fn device_create_swap_chain( device_id: DeviceId, surface_id: SurfaceId, desc: &swap_chain::SwapChainDescriptor, + token: &mut Token, ) -> Vec> { info!("creating swap chain {:?}", desc); - let mut surface_guard = HUB.surfaces.write(); - let device_guard = HUB.devices.read(); + let (mut surface_guard, mut token) = HUB.surfaces.write(token); + let (adapter_guard, mut token) = HUB.adapters.read(&mut token); + let (device_guard, _) = HUB.devices.read(&mut token); let device = &device_guard[device_id]; let surface = &mut surface_guard[surface_id]; let (caps, formats, _present_modes) = { - let adapter_guard = HUB.adapters.read(); let adapter = &adapter_guard[device.adapter_id]; assert!(surface .raw @@ -1827,10 +1908,11 @@ pub fn device_create_swap_chain( pub fn swap_chain_populate_textures( swap_chain_id: SwapChainId, textures: Vec>, + token: &mut Token, ) { - let mut surface_guard = HUB.surfaces.write(); + let (mut surface_guard, mut token) = HUB.surfaces.write(token); let swap_chain = surface_guard[swap_chain_id].swap_chain.as_mut().unwrap(); - let device_guard = HUB.devices.read(); + let (device_guard, mut token) = HUB.devices.read(&mut token); let device = &device_guard[swap_chain.device_id.value]; let mut trackers = device.trackers.lock(); @@ -1858,7 +1940,7 @@ pub fn swap_chain_populate_textures( }); let texture_id = Stored { ref_count: texture.life_guard.ref_count.clone(), - value: HUB.textures.register_local(texture), + value: HUB.textures.register_local(texture, &mut token), }; trackers.textures.init( texture_id.value, @@ -1879,7 +1961,7 @@ pub fn swap_chain_populate_textures( }; let view_id = Stored { ref_count: view.life_guard.ref_count.clone(), - value: HUB.texture_views.register_local(view), + value: HUB.texture_views.register_local(view, &mut token), }; trackers .views @@ -1905,21 +1987,23 @@ pub extern "C" fn wgpu_device_create_swap_chain( surface_id: SurfaceId, desc: &swap_chain::SwapChainDescriptor, ) -> SwapChainId { - let textures = device_create_swap_chain(device_id, surface_id, desc); - swap_chain_populate_textures(surface_id, textures); + let mut token = Token::root(); + let textures = device_create_swap_chain(device_id, surface_id, desc, &mut token); + swap_chain_populate_textures(surface_id, textures, &mut token); surface_id } #[no_mangle] pub extern "C" fn wgpu_device_poll(device_id: DeviceId, force_wait: bool) { - let callbacks = HUB.devices.read()[device_id].maintain(force_wait); + let (device_guard, mut token) = HUB.devices.read(&mut Token::root()); + let callbacks = device_guard[device_id].maintain(force_wait, &mut token); Device::fire_map_callbacks(callbacks); } #[no_mangle] pub extern "C" fn wgpu_device_destroy(device_id: DeviceId) { - let device = HUB.devices.unregister(device_id); - device.maintain(true); + let (device, mut token) = HUB.devices.unregister(device_id, &mut Token::root()); + device.maintain(true, &mut token); device.com_allocator.destroy(&device.raw); } @@ -1936,8 +2020,11 @@ pub extern "C" fn wgpu_buffer_map_read_async( callback: BufferMapReadCallback, userdata: *mut u8, ) { + let mut token = Token::root(); + let (device_guard, mut token) = HUB.devices.read(&mut token); + let (device_id, ref_count) = { - let mut buffer_guard = HUB.buffers.write(); + let (mut buffer_guard, _) = HUB.buffers.write(&mut token); let buffer = &mut buffer_guard[buffer_id]; if buffer.pending_map_operation.is_some() { @@ -1951,7 +2038,6 @@ pub extern "C" fn wgpu_buffer_map_read_async( (buffer.device_id.value, buffer.life_guard.ref_count.clone()) }; - let device_guard = HUB.devices.read(); let device = &device_guard[device_id]; device @@ -1971,8 +2057,11 @@ pub extern "C" fn wgpu_buffer_map_write_async( callback: BufferMapWriteCallback, userdata: *mut u8, ) { + let mut token = Token::root(); + let (device_guard, mut token) = HUB.devices.read(&mut token); + let (device_id, ref_count) = { - let mut buffer_guard = HUB.buffers.write(); + let (mut buffer_guard, _) = HUB.buffers.write(&mut token); let buffer = &mut buffer_guard[buffer_id]; if buffer.pending_map_operation.is_some() { @@ -1986,8 +2075,7 @@ pub extern "C" fn wgpu_buffer_map_write_async( (buffer.device_id.value, buffer.life_guard.ref_count.clone()) }; - let device_guard = HUB.devices.read(); - let device = &device_guard[device_id]; + let device = &device_guard[device_id]; device .trackers @@ -2000,12 +2088,12 @@ pub extern "C" fn wgpu_buffer_map_write_async( #[no_mangle] pub extern "C" fn wgpu_buffer_unmap(buffer_id: BufferId) { - // get the device ID first in order to have a clean locking order - let device_id = HUB.buffers.read()[buffer_id].device_id.value; - let device_guard = HUB.devices.read(); - let device_raw = &device_guard[device_id].raw; - let mut buffer_guard = HUB.buffers.write(); + let mut token = Token::root(); + let (device_guard, mut token) = HUB.devices.read(&mut token); + let (mut buffer_guard, _) = HUB.buffers.write(&mut token); + let buffer = &mut buffer_guard[buffer_id]; + let device_raw = &device_guard[buffer.device_id.value].raw; if !buffer.mapped_write_ranges.is_empty() { unsafe { diff --git a/wgpu-native/src/hub.rs b/wgpu-native/src/hub.rs index c8290845a..e59f2bbaa 100644 --- a/wgpu-native/src/hub.rs +++ b/wgpu-native/src/hub.rs @@ -37,20 +37,27 @@ use crate::{ }; #[cfg(not(feature = "gfx-backend-gl"))] use crate::{InstanceHandle, InstanceId}; + use lazy_static::lazy_static; #[cfg(feature = "local")] use parking_lot::Mutex; -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use vec_map::VecMap; -use std::{ops, sync::Arc}; +use std::{ + cell::Cell, + marker::PhantomData, + ops, + sync::Arc, +}; + /// A simple structure to manage identities of objects. #[derive(Debug)] pub struct IdentityManager { free: Vec, epochs: Vec, - phantom: std::marker::PhantomData, + phantom: PhantomData, } impl Default for IdentityManager { @@ -58,7 +65,7 @@ impl Default for IdentityManager { IdentityManager { free: Default::default(), epochs: Default::default(), - phantom: std::marker::PhantomData, + phantom: PhantomData, } } } @@ -92,7 +99,7 @@ impl IdentityManager { pub struct Storage { //TODO: consider concurrent hashmap? map: VecMap<(T, Epoch)>, - _phantom: std::marker::PhantomData, + _phantom: PhantomData, } impl ops::Index for Storage { @@ -119,12 +126,117 @@ impl Storage { _ => false, } } + + pub fn remove(&mut self, id: I) -> T { + let (value, epoch) = self.map.remove(id.index() as usize).unwrap(); + assert_eq!(epoch, id.epoch()); + value + } } + +/// Type system for enforcing the lock order on shared HUB structures. +/// The main property we are trying to establish is that each type +/// of resources have only a single path from Root{} to reach by, +/// when locking multiple resources types. +pub trait Access {} + +pub enum Root {} +//TODO: establish an order instead of declaring all the pairs. +impl Access for Root {} +impl Access for Root {} +impl Access for InstanceHandle {} +impl Access for Root {} +impl Access for SurfaceHandle {} +impl Access for Root {} +impl Access for SurfaceHandle {} +impl Access for AdapterHandle {} +impl Access for Root {} +impl Access for DeviceHandle {} +impl Access for Root {} +impl Access for DeviceHandle {} +impl Access for Root {} +impl Access for DeviceHandle {} +impl Access for PipelineLayoutHandle {} +impl Access for CommandBufferHandle {} +impl Access for Root {} +impl Access for DeviceHandle {} +impl Access for Root {} +impl Access for BindGroupHandle {} +impl Access for CommandBufferHandle {} +impl Access for Root {} +impl Access for BindGroupHandle {} +impl Access for CommandBufferHandle {} +impl Access for Root {} +impl Access for ComputePassHandle {} +impl Access for Root {} +impl Access for RenderPassHandle {} +impl Access for Root {} +impl Access for PipelineLayoutHandle {} +impl Access for Root {} +impl Access for DeviceHandle {} +impl Access for BindGroupLayoutHandle {} +impl Access for BindGroupHandle {} +impl Access for CommandBufferHandle {} +impl Access for ComputePassHandle {} +impl Access for ComputePipelineHandle {} +impl Access for RenderPassHandle {} +impl Access for RenderPipelineHandle {} +impl Access for Root {} +impl Access for DeviceHandle {} +impl Access for BufferHandle {} +impl Access for Root {} +impl Access for DeviceHandle {} +impl Access for TextureHandle {} +impl Access for Root {} +impl Access for TextureViewHandle {} + +thread_local! { + static ACTIVE_TOKEN: Cell = Cell::new(false); +} + +pub struct Token<'a, T: 'a> { + level: PhantomData<&'a T>, + is_root: bool, +} + +impl<'a, T> Token<'a, T> { + fn new() -> Self { + Token { + level: PhantomData, + is_root: false, + } + } +} + +impl Token<'static, Root> { + pub fn root() -> Self { + ACTIVE_TOKEN.with(|active| { + assert!(!active.replace(true)); + }); + + Token { + level: PhantomData, + is_root: true, + } + } +} + +impl<'a, T> Drop for Token<'a, T> { + fn drop(&mut self) { + if self.is_root { + ACTIVE_TOKEN.with(|active| { + assert!(active.replace(false)); + }); + } + } +} + + #[derive(Debug)] pub struct Registry { #[cfg(feature = "local")] - identity: Mutex>, + pub identity: Mutex>, data: RwLock>, } @@ -135,27 +247,16 @@ impl Default for Registry { identity: Mutex::new(IdentityManager::default()), data: RwLock::new(Storage { map: VecMap::new(), - _phantom: std::marker::PhantomData, + _phantom: PhantomData, }), } } } -impl ops::Deref for Registry { - type Target = RwLock>; - fn deref(&self) -> &Self::Target { - &self.data - } -} - -impl ops::DerefMut for Registry { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.data - } -} - impl Registry { - pub fn register(&self, id: I, value: T) { + pub fn register>( + &self, id: I, value: T, _token: &mut Token + ) { let old = self .data .write() @@ -165,19 +266,34 @@ impl Registry { } #[cfg(feature = "local")] - pub fn register_local(&self, value: T) -> I { + pub fn register_local>( + &self, value: T, token: &mut Token + ) -> I { let id = self.identity.lock().alloc(); - self.register(id, value); + self.register(id, value, token); id } - pub fn unregister(&self, id: I) -> T { - let (value, epoch) = self.data.write().map.remove(id.index() as usize).unwrap(); - assert_eq!(epoch, id.epoch()); + pub fn unregister>( + &self, id: I, _token: &mut Token + ) -> (T, Token) { + let value = self.data.write().remove(id); //Note: careful about the order here! #[cfg(feature = "local")] self.identity.lock().free(id); - value + (value, Token::new()) + } + + pub fn read>( + &self, _token: &mut Token + ) -> (RwLockReadGuard>, Token) { + (self.data.read(), Token::new()) + } + + pub fn write>( + &self, _token: &mut Token + ) -> (RwLockWriteGuard>, Token) { + (self.data.write(), Token::new()) } } @@ -185,19 +301,18 @@ impl Registry { pub struct Hub { #[cfg(not(feature = "gfx-backend-gl"))] pub instances: Arc>, - pub surfaces: Arc>, pub adapters: Arc>, pub devices: Arc>, pub pipeline_layouts: Arc>, + pub shader_modules: Arc>, pub bind_group_layouts: Arc>, pub bind_groups: Arc>, - pub shader_modules: Arc>, pub command_buffers: Arc>, - pub render_pipelines: Arc>, - pub compute_pipelines: Arc>, pub render_passes: Arc>, + pub render_pipelines: Arc>, pub compute_passes: Arc>, + pub compute_pipelines: Arc>, pub buffers: Arc>, pub textures: Arc>, pub texture_views: Arc>, diff --git a/wgpu-native/src/instance.rs b/wgpu-native/src/instance.rs index 76a44d17a..6c3a219de 100644 --- a/wgpu-native/src/instance.rs +++ b/wgpu-native/src/instance.rs @@ -1,6 +1,6 @@ use crate::{ binding_model::MAX_BIND_GROUPS, - hub::HUB, + hub::{HUB, Root, Token}, AdapterHandle, AdapterId, DeviceHandle, @@ -72,14 +72,14 @@ pub fn create_instance() -> ::back::Instance { #[no_mangle] pub extern "C" fn wgpu_create_instance() -> InstanceId { let inst = create_instance(); - HUB.instances.register_local(inst) + HUB.instances.register_local(inst, &mut Token::root()) } #[cfg(all(feature = "local", feature = "gfx-backend-gl"))] pub fn wgpu_create_gl_instance(windowed_context: back::glutin::WindowedContext) -> InstanceId { let raw = back::Surface::from_window(windowed_context); let surface = SurfaceHandle::new(raw); - HUB.surfaces.register_local(surface) + HUB.surfaces.register_local(surface, &mut Token::root()) } #[cfg(all(feature = "window-winit", not(feature = "gfx-backend-gl")))] @@ -88,9 +88,11 @@ pub extern "C" fn wgpu_instance_create_surface_from_winit( instance_id: InstanceId, window: &winit::Window, ) -> SurfaceId { - let raw = HUB.instances.read()[instance_id].create_surface(window); + let mut token = Token::root(); + let (instance_guard, mut token) = HUB.instances.read(&mut token); + let raw = instance_guard[instance_id].create_surface(window); let surface = SurfaceHandle::new(raw); - HUB.surfaces.register_local(surface) + HUB.surfaces.register_local(surface, &mut token) } #[cfg(not(feature = "gfx-backend-gl"))] @@ -99,12 +101,16 @@ pub fn instance_create_surface_from_xlib( instance_id: InstanceId, display: *mut *const std::ffi::c_void, window: u64, + token: &mut Token, ) -> SurfaceHandle { #[cfg(not(all(unix, feature = "gfx-backend-vulkan")))] unimplemented!(); #[cfg(all(unix, feature = "gfx-backend-vulkan"))] - SurfaceHandle::new(HUB.instances.read()[instance_id].create_surface_from_xlib(display, window)) + { + let (instance_guard, _) = HUB.instances.read(token); + SurfaceHandle::new(instance_guard[instance_id].create_surface_from_xlib(display, window)) + } } #[cfg(all(feature = "local", not(feature = "gfx-backend-gl")))] @@ -114,8 +120,9 @@ pub extern "C" fn wgpu_instance_create_surface_from_xlib( display: *mut *const std::ffi::c_void, window: u64, ) -> SurfaceId { - let surface = instance_create_surface_from_xlib(instance_id, display, window); - HUB.surfaces.register_local(surface) + let mut token = Token::root(); + let surface = instance_create_surface_from_xlib(instance_id, display, window, &mut token); + HUB.surfaces.register_local(surface, &mut token) } #[cfg(not(feature = "gfx-backend-gl"))] @@ -123,15 +130,19 @@ pub extern "C" fn wgpu_instance_create_surface_from_xlib( pub fn instance_create_surface_from_macos_layer( instance_id: InstanceId, layer: *mut std::ffi::c_void, + token: &mut Token, ) -> SurfaceHandle { #[cfg(not(feature = "gfx-backend-metal"))] unimplemented!(); #[cfg(feature = "gfx-backend-metal")] - SurfaceHandle::new( - HUB.instances.read()[instance_id] - .create_surface_from_layer(layer as *mut _, cfg!(debug_assertions)), - ) + { + let (instance_guard, _) = HUB.instances.read(token); + SurfaceHandle::new( + instance_guard[instance_id] + .create_surface_from_layer(layer as *mut _, cfg!(debug_assertions)), + ) + } } #[cfg(not(feature = "gfx-backend-gl"))] @@ -141,8 +152,9 @@ pub extern "C" fn wgpu_instance_create_surface_from_macos_layer( instance_id: InstanceId, layer: *mut std::ffi::c_void, ) -> SurfaceId { - let surface = instance_create_surface_from_macos_layer(instance_id, layer); - HUB.surfaces.register_local(surface) + let mut token = Token::root(); + let surface = instance_create_surface_from_macos_layer(instance_id, layer, &mut token); + HUB.surfaces.register_local(surface, &mut token) } #[cfg(not(feature = "gfx-backend-gl"))] @@ -151,7 +163,10 @@ pub fn instance_create_surface_from_windows_hwnd( instance_id: InstanceId, hinstance: *mut std::ffi::c_void, hwnd: *mut std::ffi::c_void, + token: &mut Token, ) -> SurfaceHandle { + let (instance_guard, _) = HUB.instances.read(token); + #[cfg(not(any( feature = "gfx-backend-dx11", feature = "gfx-backend-dx12", @@ -160,10 +175,10 @@ pub fn instance_create_surface_from_windows_hwnd( let raw = unimplemented!(); #[cfg(any(feature = "gfx-backend-dx11", feature = "gfx-backend-dx12"))] - let raw = HUB.instances.read()[instance_id].create_surface_from_hwnd(hwnd); + let raw = instance_guard[instance_id].create_surface_from_hwnd(hwnd); #[cfg(all(target_os = "windows", feature = "gfx-backend-vulkan"))] - let raw = HUB.instances.read()[instance_id].create_surface_from_hwnd(hinstance, hwnd); + let raw = instance_guard[instance_id].create_surface_from_hwnd(hinstance, hwnd); #[allow(unreachable_code)] SurfaceHandle::new(raw) @@ -177,8 +192,9 @@ pub extern "C" fn wgpu_instance_create_surface_from_windows_hwnd( hinstance: *mut std::ffi::c_void, hwnd: *mut std::ffi::c_void, ) -> SurfaceId { - let surface = instance_create_surface_from_windows_hwnd(instance_id, hinstance, hwnd); - HUB.surfaces.register_local(surface) + let mut token = Token::root(); + let surface = instance_create_surface_from_windows_hwnd(instance_id, hinstance, hwnd, &mut token); + HUB.surfaces.register_local(surface, &mut token) } #[cfg(all(feature = "local", feature = "gfx-backend-gl"))] @@ -186,18 +202,21 @@ pub fn wgpu_instance_get_gl_surface(instance_id: InstanceId) -> SurfaceId { instance_id } -pub fn instance_get_adapter(instance_id: InstanceId, desc: &AdapterDescriptor) -> AdapterHandle { +pub fn instance_get_adapter( + instance_id: InstanceId, + desc: &AdapterDescriptor, + token: &mut Token, +) -> AdapterHandle { + let (instance_guard, mut token) = HUB.instances.read(token); #[cfg(not(feature = "gfx-backend-gl"))] let adapters = { - let instance_guard = HUB.instances.read(); - let instance = &instance_guard[instance_id]; - instance.enumerate_adapters() + let _ = &mut token; + instance_guard[instance_id].enumerate_adapters() }; #[cfg(feature = "gfx-backend-gl")] let adapters = { - let surface_guard = HUB.surfaces.read(); - let surface = &surface_guard[instance_id]; - surface.raw.enumerate_adapters() + let (surface_guard, _) = HUB.surfaces.read(&mut token); + surface_guard[instance_id].raw.enumerate_adapters() }; let (mut integrated_first, mut discrete_first, mut discrete_last, mut alternative) = @@ -242,7 +261,8 @@ pub extern "C" fn wgpu_instance_get_adapter( instance_id: InstanceId, desc: &AdapterDescriptor, ) -> AdapterId { - let adapter = instance_get_adapter(instance_id, desc); + let mut token = Token::root(); + let adapter = instance_get_adapter(instance_id, desc, &mut token); let limits = adapter.physical_device.limits(); info!("Adapter {:?}", adapter.info); @@ -256,11 +276,15 @@ pub extern "C" fn wgpu_instance_get_adapter( "Adapter uniform buffer offset alignment not compatible with WGPU" ); - HUB.adapters.register_local(adapter) + HUB.adapters.register_local(adapter, &mut token) } -pub fn adapter_create_device(adapter_id: AdapterId, _desc: &DeviceDescriptor) -> DeviceHandle { - let adapter_guard = HUB.adapters.read(); +pub fn adapter_create_device( + adapter_id: AdapterId, + _desc: &DeviceDescriptor, + token: &mut Token, +) -> DeviceHandle { + let (adapter_guard, _) = HUB.adapters.read(token); let adapter = &adapter_guard[adapter_id]; let (raw, queue_group) = adapter.open_with::<_, hal::General>(1, |_qf| true).unwrap(); let mem_props = adapter.physical_device.memory_properties(); @@ -274,6 +298,7 @@ pub extern "C" fn wgpu_adapter_request_device( adapter_id: AdapterId, desc: &DeviceDescriptor, ) -> DeviceId { - let device = adapter_create_device(adapter_id, desc); - HUB.devices.register_local(device) + let mut token = Token::root(); + let device = adapter_create_device(adapter_id, desc, &mut token); + HUB.devices.register_local(device, &mut token) } diff --git a/wgpu-native/src/lib.rs b/wgpu-native/src/lib.rs index bcc231dbf..0c4f965f2 100644 --- a/wgpu-native/src/lib.rs +++ b/wgpu-native/src/lib.rs @@ -35,7 +35,7 @@ pub use self::binding_model::*; pub use self::command::*; pub use self::device::*; #[cfg(not(feature = "local"))] -pub use self::hub::{IdentityManager, Registry, HUB}; +pub use self::hub::{Access, IdentityManager, Registry, Token, HUB}; pub use self::instance::*; pub use self::pipeline::*; pub use self::resource::*; diff --git a/wgpu-native/src/swap_chain.rs b/wgpu-native/src/swap_chain.rs index 931eee4cd..115fe9218 100644 --- a/wgpu-native/src/swap_chain.rs +++ b/wgpu-native/src/swap_chain.rs @@ -1,7 +1,7 @@ use crate::{ conv, device::all_image_stages, - hub::HUB, + hub::{HUB, Token}, resource, DeviceId, Extent3d, @@ -113,8 +113,9 @@ pub struct SwapChainOutput { #[no_mangle] pub extern "C" fn wgpu_swap_chain_get_next_texture(swap_chain_id: SwapChainId) -> SwapChainOutput { + let mut token = Token::root(); let (image_index, device_id, descriptor) = { - let mut surface_guard = HUB.surfaces.write(); + let (mut surface_guard, _) = HUB.surfaces.write(&mut token); let swap_chain = surface_guard[swap_chain_id].swap_chain.as_mut().unwrap(); let result = unsafe { swap_chain @@ -135,12 +136,12 @@ pub extern "C" fn wgpu_swap_chain_get_next_texture(swap_chain_id: SwapChainId) - use crate::device::{device_create_swap_chain, swap_chain_populate_textures}; if image_index.is_none() { warn!("acquire_image failed, re-creating"); - let textures = device_create_swap_chain(device_id, swap_chain_id, &descriptor); - swap_chain_populate_textures(swap_chain_id, textures); + let textures = device_create_swap_chain(device_id, swap_chain_id, &descriptor, &mut token); + swap_chain_populate_textures(swap_chain_id, textures, &mut token); } } - let mut surface_guard = HUB.surfaces.write(); + let (mut surface_guard, mut token) = HUB.surfaces.write(&mut token); let swap_chain = surface_guard[swap_chain_id].swap_chain.as_mut().unwrap(); let image_index = match image_index { @@ -159,7 +160,7 @@ pub extern "C" fn wgpu_swap_chain_get_next_texture(swap_chain_id: SwapChainId) - }, }; - let device_guard = HUB.devices.read(); + let (device_guard, mut token) = HUB.devices.read(&mut token); let device = &device_guard[device_id]; assert_ne!( @@ -184,7 +185,8 @@ pub extern "C" fn wgpu_swap_chain_get_next_texture(swap_chain_id: SwapChainId) - mem::swap(&mut frame.sem_available, &mut swap_chain.sem_available); frame.need_waiting.store(true, Ordering::Release); - let frame_epoch = HUB.textures.read()[frame.texture_id.value] + let (texture_guard, _) = HUB.textures.read(&mut token); + let frame_epoch = texture_guard[frame.texture_id.value] .placement .as_swap_chain() .bump_epoch(); @@ -203,7 +205,8 @@ pub extern "C" fn wgpu_swap_chain_get_next_texture(swap_chain_id: SwapChainId) - #[no_mangle] pub extern "C" fn wgpu_swap_chain_present(swap_chain_id: SwapChainId) { - let mut surface_guard = HUB.surfaces.write(); + let mut token = Token::root(); + let (mut surface_guard, mut token) = HUB.surfaces.write(&mut token); let swap_chain = surface_guard[swap_chain_id].swap_chain.as_mut().unwrap(); let image_index = swap_chain.acquired.remove(0); @@ -220,10 +223,10 @@ pub extern "C" fn wgpu_swap_chain_present(swap_chain_id: SwapChainId) { image_index ); - let mut device_guard = HUB.devices.write(); + let (mut device_guard, mut token) = HUB.devices.write(&mut token); let device = &mut device_guard[swap_chain.device_id.value]; - let texture_guard = HUB.textures.read(); + let (texture_guard, _) = HUB.textures.read(&mut token); let texture = &texture_guard[frame.texture_id.value]; texture.placement.as_swap_chain().bump_epoch(); diff --git a/wgpu-remote/src/server.rs b/wgpu-remote/src/server.rs index 08c58f4b8..ca92f70d6 100644 --- a/wgpu-remote/src/server.rs +++ b/wgpu-remote/src/server.rs @@ -20,22 +20,23 @@ enum ControlFlow { } fn process(message: GlobalMessage) -> ControlFlow { + let mut token = &mut wgn::Token::root(); match message { GlobalMessage::Instance(msg) => match msg { InstanceMessage::Create(instance_id) => { let instance = wgn::create_instance(); - wgn::HUB.instances.register(instance_id, instance); + wgn::HUB.instances.register(instance_id, instance, &mut token); } InstanceMessage::InstanceGetAdapter(instance_id, ref desc, id) => { let adapter = wgn::instance_get_adapter(instance_id, desc); - wgn::HUB.adapters.register(id, adapter); + wgn::HUB.adapters.register(id, adapter, &mut token); } InstanceMessage::AdapterCreateDevice(adapter_id, ref desc, id) => { let device = wgn::adapter_create_device(adapter_id, desc); - wgn::HUB.devices.register(id, device); + wgn::HUB.devices.register(id, device, &mut token); } InstanceMessage::Destroy(instance_id) => { - wgn::HUB.instances.unregister(instance_id); + wgn::HUB.instances.unregister(instance_id, &mut token); } }, GlobalMessage::Terminate => return ControlFlow::Terminate,