mirror of
https://github.com/storytold/wgpu.git
synced 2026-11-21 06:50:59 +01:00
Fix crash in lock validation due to TLS destructor call order (#10030)
This commit is contained in:
1 parent
4bd4405d02
commit
efcf445784
3 files changed
+158
-108
No files matched your search
@@ -253,60 +253,64 @@ impl Drop for LockStateGuard {
|
||||
/// Return the `Option<HeldLock>` state that must be restored when this lock is
|
||||
/// released.
|
||||
fn acquire(new_rank: LockRank, location: &'static Location<'static>) -> Option<HeldLock> {
|
||||
LOCK_STATE.with_borrow_mut(|state| match *state {
|
||||
ThreadState::Disabled => None,
|
||||
ThreadState::Initial => {
|
||||
let Ok(dir) = std::env::var("WGPU_CORE_LOCK_OBSERVE_DIR") else {
|
||||
*state = ThreadState::Disabled;
|
||||
return None;
|
||||
};
|
||||
let result = LOCK_STATE.try_with(|refcell| {
|
||||
let state = &mut *refcell.borrow_mut();
|
||||
match *state {
|
||||
ThreadState::Disabled => None,
|
||||
ThreadState::Initial => {
|
||||
let Ok(dir) = std::env::var("WGPU_CORE_LOCK_OBSERVE_DIR") else {
|
||||
*state = ThreadState::Disabled;
|
||||
return None;
|
||||
};
|
||||
|
||||
// Create the observation log file.
|
||||
let mut log = ObservationLog::create(dir)
|
||||
.expect("Failed to open lock observation file (does the dir exist?)");
|
||||
// Create the observation log file.
|
||||
let mut log = ObservationLog::create(dir)
|
||||
.expect("Failed to open lock observation file (does the dir exist?)");
|
||||
|
||||
// Log the full set of lock ranks, so that the analysis can even see
|
||||
// locks that are only acquired in isolation.
|
||||
for rank in LockRankSet::all().iter() {
|
||||
log.write_rank(rank);
|
||||
// Log the full set of lock ranks, so that the analysis can even see
|
||||
// locks that are only acquired in isolation.
|
||||
for rank in LockRankSet::all().iter() {
|
||||
log.write_rank(rank);
|
||||
}
|
||||
|
||||
// Update our state to reflect that we are logging acquisitions, and
|
||||
// that we have acquired this lock.
|
||||
*state = ThreadState::Enabled {
|
||||
held_lock: Some(HeldLock {
|
||||
rank: new_rank,
|
||||
location,
|
||||
}),
|
||||
log,
|
||||
};
|
||||
|
||||
// Since this is the first acquisition on this thread, we know that
|
||||
// there is no prior lock held, and thus nothing to log yet.
|
||||
None
|
||||
}
|
||||
ThreadState::Enabled {
|
||||
ref mut held_lock,
|
||||
ref mut log,
|
||||
} => {
|
||||
if let Some(ref held_lock) = held_lock {
|
||||
log.write_acquisition(held_lock, new_rank, location);
|
||||
}
|
||||
|
||||
// Update our state to reflect that we are logging acquisitions, and
|
||||
// that we have acquired this lock.
|
||||
*state = ThreadState::Enabled {
|
||||
held_lock: Some(HeldLock {
|
||||
held_lock.replace(HeldLock {
|
||||
rank: new_rank,
|
||||
location,
|
||||
}),
|
||||
log,
|
||||
};
|
||||
|
||||
// Since this is the first acquisition on this thread, we know that
|
||||
// there is no prior lock held, and thus nothing to log yet.
|
||||
None
|
||||
}
|
||||
ThreadState::Enabled {
|
||||
ref mut held_lock,
|
||||
ref mut log,
|
||||
} => {
|
||||
if let Some(ref held_lock) = held_lock {
|
||||
log.write_acquisition(held_lock, new_rank, location);
|
||||
})
|
||||
}
|
||||
|
||||
held_lock.replace(HeldLock {
|
||||
rank: new_rank,
|
||||
location,
|
||||
})
|
||||
}
|
||||
})
|
||||
});
|
||||
result.unwrap_or(None)
|
||||
}
|
||||
|
||||
/// Record the release of a lock whose saved state was `saved`.
|
||||
fn release(saved: Option<HeldLock>) {
|
||||
LOCK_STATE.with_borrow_mut(|state| {
|
||||
let _ = LOCK_STATE.try_with(|refcell| {
|
||||
if let ThreadState::Enabled {
|
||||
ref mut held_lock, ..
|
||||
} = *state
|
||||
} = *refcell.borrow_mut()
|
||||
{
|
||||
*held_lock = saved;
|
||||
}
|
||||
|
||||
@@ -134,29 +134,32 @@ impl Drop for LockStateGuard {
|
||||
///
|
||||
/// Return the `LockState` that must be restored when this thread is released.
|
||||
fn acquire(new_rank: LockRank, location: &'static Location<'static>) -> LockState {
|
||||
let state = LOCK_STATE.get();
|
||||
// Initially, it's fine to acquire any lock. So we only
|
||||
// need to check when `last_acquired` is `Some`.
|
||||
if let Some((ref last_rank, ref last_location)) = state.last_acquired {
|
||||
assert!(
|
||||
last_rank.followers.contains(new_rank.bit),
|
||||
"Attempt to acquire nested mutexes in wrong order:\n\
|
||||
last locked {:<35} at {}\n\
|
||||
now locking {:<35} at {}\n\
|
||||
Locking {} after locking {} is not permitted.",
|
||||
last_rank.bit.member_name(),
|
||||
last_location,
|
||||
new_rank.bit.member_name(),
|
||||
location,
|
||||
new_rank.bit.member_name(),
|
||||
last_rank.bit.member_name(),
|
||||
);
|
||||
}
|
||||
LOCK_STATE.set(LockState {
|
||||
last_acquired: Some((new_rank, location)),
|
||||
depth: state.depth + 1,
|
||||
let result = LOCK_STATE.try_with(|cell| {
|
||||
let state = cell.get();
|
||||
// Initially, it's fine to acquire any lock. So we only
|
||||
// need to check when `last_acquired` is `Some`.
|
||||
if let Some((ref last_rank, ref last_location)) = state.last_acquired {
|
||||
assert!(
|
||||
last_rank.followers.contains(new_rank.bit),
|
||||
"Attempt to acquire nested mutexes in wrong order:\n\
|
||||
last locked {:<35} at {}\n\
|
||||
now locking {:<35} at {}\n\
|
||||
Locking {} after locking {} is not permitted.",
|
||||
last_rank.bit.member_name(),
|
||||
last_location,
|
||||
new_rank.bit.member_name(),
|
||||
location,
|
||||
new_rank.bit.member_name(),
|
||||
last_rank.bit.member_name(),
|
||||
);
|
||||
}
|
||||
cell.set(LockState {
|
||||
last_acquired: Some((new_rank, location)),
|
||||
depth: state.depth + 1,
|
||||
});
|
||||
state
|
||||
});
|
||||
state
|
||||
result.unwrap_or(LockState::INITIAL)
|
||||
}
|
||||
|
||||
/// Record the release of a lock whose saved state was `saved`.
|
||||
@@ -164,53 +167,55 @@ fn acquire(new_rank: LockRank, location: &'static Location<'static>) -> LockStat
|
||||
/// Check that locks are being acquired in stacking order, and update the
|
||||
/// per-thread state accordingly.
|
||||
fn release(saved: LockState) {
|
||||
let saved_info = saved.last_acquired;
|
||||
let _ = LOCK_STATE.try_with(|cell| {
|
||||
let saved_info = saved.last_acquired;
|
||||
|
||||
let prior = LOCK_STATE.replace(saved);
|
||||
let prior = cell.replace(saved);
|
||||
|
||||
let (prior_rank, prior_location) = prior
|
||||
.last_acquired
|
||||
.expect("Releasing a lock, but no acquisition recorded");
|
||||
let (prior_rank, prior_location) = prior
|
||||
.last_acquired
|
||||
.expect("Releasing a lock, but no acquisition recorded");
|
||||
|
||||
// Although Rust allows mutex guards to be dropped in any
|
||||
// order, this analysis requires that locks be acquired and
|
||||
// released in stack order: the next lock to be released must be
|
||||
// the most recently acquired lock still held.
|
||||
// Although Rust allows mutex guards to be dropped in any
|
||||
// order, this analysis requires that locks be acquired and
|
||||
// released in stack order: the next lock to be released must be
|
||||
// the most recently acquired lock still held.
|
||||
|
||||
match (saved.depth, saved_info) {
|
||||
(saved_depth @ 0, None) => {
|
||||
assert_eq!(
|
||||
prior.depth,
|
||||
saved_depth + 1,
|
||||
"Lock not released in stacking order\n\
|
||||
match (saved.depth, saved_info) {
|
||||
(saved_depth @ 0, None) => {
|
||||
assert_eq!(
|
||||
prior.depth,
|
||||
saved_depth + 1,
|
||||
"Lock not released in stacking order\n\
|
||||
released {:<35} locked at {:?}\n\
|
||||
when not expecting any locks to be held\n",
|
||||
prior_rank.bit.member_name(),
|
||||
prior_location,
|
||||
);
|
||||
}
|
||||
(0, Some(_)) => {
|
||||
panic!("Found previous lock acquisition information, but saved.depth = 0");
|
||||
}
|
||||
(saved_depth, Some((saved_rank, saved_location))) => {
|
||||
assert_eq!(
|
||||
prior.depth,
|
||||
saved_depth + 1,
|
||||
"Lock not released in stacking order\n\
|
||||
prior_rank.bit.member_name(),
|
||||
prior_location,
|
||||
);
|
||||
}
|
||||
(0, Some(_)) => {
|
||||
panic!("Found previous lock acquisition information, but saved.depth = 0");
|
||||
}
|
||||
(saved_depth, Some((saved_rank, saved_location))) => {
|
||||
assert_eq!(
|
||||
prior.depth,
|
||||
saved_depth + 1,
|
||||
"Lock not released in stacking order\n\
|
||||
expecting release of {:<35} locked at {:?}\n\
|
||||
but instead released {:<35} locked at {:?}\n",
|
||||
saved_rank.bit.member_name(),
|
||||
saved_location,
|
||||
prior_rank.bit.member_name(),
|
||||
prior_location,
|
||||
);
|
||||
}
|
||||
(saved_depth, None) => {
|
||||
panic!(
|
||||
saved_rank.bit.member_name(),
|
||||
saved_location,
|
||||
prior_rank.bit.member_name(),
|
||||
prior_location,
|
||||
);
|
||||
}
|
||||
(saved_depth, None) => {
|
||||
panic!(
|
||||
"Found saved.depth = {saved_depth}, but no previous lock acquisition information"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
impl<T> Mutex<T> {
|
||||
|
||||
+51
-10
@@ -93,21 +93,25 @@ mod trace {
|
||||
backtrace: Backtrace::capture(),
|
||||
};
|
||||
|
||||
if let Some(prev) = SNATCH_LOCK_TRACE.take() {
|
||||
let current = thread::current();
|
||||
let name = current.name().unwrap_or("<unnamed>");
|
||||
panic!(
|
||||
"thread '{name}' attempted to acquire a snatch lock recursively.\n\
|
||||
// try_with() because lock may be unavailable due to TLS destruction
|
||||
// call order
|
||||
let _ = SNATCH_LOCK_TRACE.try_with(|trace| {
|
||||
if let Some(prev) = trace.take() {
|
||||
let current = thread::current();
|
||||
let name = current.name().unwrap_or("<unnamed>");
|
||||
panic!(
|
||||
"thread '{name}' attempted to acquire a snatch lock recursively.\n\
|
||||
- Currently trying to acquire {new}\n\
|
||||
- Previously acquired {prev}",
|
||||
);
|
||||
} else {
|
||||
SNATCH_LOCK_TRACE.set(Some(new));
|
||||
}
|
||||
);
|
||||
} else {
|
||||
trace.set(Some(new));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn exit() {
|
||||
SNATCH_LOCK_TRACE.take();
|
||||
let _ = SNATCH_LOCK_TRACE.try_with(|trace| trace.take());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,3 +204,40 @@ impl Drop for ExclusiveSnatchGuard<'_> {
|
||||
LockTrace::exit();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "std"))]
|
||||
mod tests {
|
||||
use super::SnatchLock;
|
||||
use crate::lock::rank;
|
||||
use core::cell::Cell;
|
||||
|
||||
/// A thing that reads a snatchlock during drop, to test destruction call order problems
|
||||
struct LockOnDrop(SnatchLock);
|
||||
impl Drop for LockOnDrop {
|
||||
fn drop(&mut self) {
|
||||
drop(self.0.read());
|
||||
}
|
||||
}
|
||||
|
||||
std::thread_local! {
|
||||
static HOLDER: Cell<Option<LockOnDrop>> = const { Cell::new(None) };
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_during_tls_destruction() {
|
||||
std::thread::spawn(|| {
|
||||
// our thread local is touched first, so its destructor will be called last
|
||||
HOLDER.set(Some(LockOnDrop(unsafe {
|
||||
SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK)
|
||||
})));
|
||||
|
||||
// take the lock which causes SNATCH_LOCK_TRACE to be created. it
|
||||
// will be destroyed first
|
||||
let lock = unsafe { SnatchLock::new(rank::DEVICE_SNATCHABLE_LOCK) };
|
||||
drop(lock.read());
|
||||
})
|
||||
.join()
|
||||
.unwrap();
|
||||
// just testing that the thread exited successfully
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user