Stop deadlocks as they form and watch every lock for one that never lets go (#5)

* Stop deadlocks as they form and watch every lock for one that never lets go

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Give the stuck-lock banner's test a program of its own

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon ThomasandClaude Opus 5.5 authored and GitHub committed 2026-10-31 20:28:16 -04:00
1 parent 6a120011a1
commit 292d5af06e
47 files changed
+1097 -35

No files matched your search

+3
View File
@@ -68,6 +68,9 @@ jobs:
- name: Dependency layering
run: cargo run --locked -q -p xtask -- layers
- name: Session locks taken twice
run: cargo run --locked -q -p xtask -- locks
- name: Tool list
run: cargo run --locked -q -p xtask -- tools --check
+7
View File
@@ -70,6 +70,12 @@ Layering is enforced by `cargo xtask layers`. Register new crates in `xtask/src/
- a missing program, non-zero exit, bad output or timeout is a `ToolError` with a plain-English remediation,
never a panic or a hang;
- nothing slow ever runs on the UI thread: the UI sends work to background threads and polls results.
11. **Never lock up on a lock.** Use `craft_anything_platform::sync::Mutex`, never `std::sync::Mutex` (clippy
refuses it). It stops a real deadlock as soon as it forms, the waiter's work unwinding to its `catch_unwind` as
an error; the watchdog reports anything held too long. Long jobs are fine: label the guard (`guard.label(id)`,
as `Engine::call` does for tools) and they're waited for, not treated as stuck. A guard that isn't bound to a
name lives until the statement ends, so read the session once: `cargo xtask locks` refuses a second
`.session()` in reach of the first.
## Working rules (mandatory)
@@ -147,6 +153,7 @@ cargo fmt --all
cargo test -p <crates you touched>
cargo clippy -p <crates> --all-targets -- -D warnings
cargo xtask layers
cargo xtask locks
cargo xtask secrets # no keys, tokens or local paths in what you commit
cargo xtask wasm # if you touched L0–L2
cargo xtask tools # if you added tools (commit docs/tools.md)
Generated
+4
View File
@@ -1126,6 +1126,7 @@ version = "0.1.0"
dependencies = [
"craft-anything-ai",
"craft-anything-engine",
"craft-anything-platform",
"craft-anything-setup",
"craft-anything-ui-egui",
"eframe",
@@ -1313,6 +1314,7 @@ version = "0.1.0"
dependencies = [
"craft-anything-engine",
"craft-anything-evidence",
"craft-anything-platform",
"log",
"serde",
"serde_json",
@@ -7959,6 +7961,8 @@ name = "xtask"
version = "0.1.0"
dependencies = [
"craft-anything-engine",
"proc-macro2",
"syn 2.0.119",
"toml 0.9.12+spec-1.1.0",
]
+1
View File
@@ -193,6 +193,7 @@ fn usage(msg: &str) -> u8 {
/// The engine with settings loaded, the run's mode, and the project (if any) opened.
pub fn make_engine(project: Option<&Path>, deep_for_run: Option<bool>) -> Result<Arc<Engine>, ToolError> {
craft_anything_platform::sync::start_watchdog(craft_anything_engine::settings::config_dir().join("logs"));
let mut session = Session::new(Settings::load());
session.deep_for_run = deep_for_run;
if let Some(dir) = project {
+2
View File
@@ -3,6 +3,7 @@
use crate::triage::{Look, MAX_BYTES, look};
use craft_anything_evidence::Confidence;
use egui::{Color32, RichText};
#[allow(clippy::disallowed_types)] // The browser has one thread: nothing can wait here.
use std::sync::{Arc, Mutex, PoisonError};
// A few tokens copied from the desktop app's Studio theme (crates/ui-egui/src/theme.rs).
@@ -21,6 +22,7 @@ const BEST_GUESS: Color32 = Color32::from_rgb(0xe0, 0xa2, 0x3a);
/// A file handed over from outside a frame (the browser's file picker).
/// A file (name and contents), or a plain message saying why it couldn't be read.
#[allow(clippy::disallowed_types)]
pub type Inbox = Arc<Mutex<Option<Result<(String, Vec<u8>), String>>>>;
pub struct Viewer {
+1
View File
@@ -12,6 +12,7 @@ workspace = true
[dependencies]
craft-anything-engine = { workspace = true }
craft-anything-platform = { workspace = true }
craft-anything-ai = { workspace = true }
craft-anything-setup = { workspace = true }
craft-anything-ui-egui = { workspace = true }
+3 -2
View File
@@ -71,7 +71,7 @@ const LOG_MAX: u64 = 512 * 1024;
/// panics, which would otherwise vanish with the window.
fn log_to_file() {
use std::io::Write as _;
struct FileLog(std::sync::Mutex<Option<std::fs::File>>);
struct FileLog(craft_anything_platform::sync::Mutex<Option<std::fs::File>>);
impl log::Log for FileLog {
fn enabled(&self, m: &log::Metadata<'_>) -> bool {
m.level() <= log::Level::Warn || (m.level() <= log::Level::Info && m.target().starts_with("craft_anything"))
@@ -94,7 +94,7 @@ fn log_to_file() {
let _ = std::fs::create_dir_all(&dir);
let fresh = std::fs::metadata(&path).is_ok_and(|m| m.len() > LOG_MAX);
let file = std::fs::OpenOptions::new().create(true).append(!fresh).write(true).truncate(fresh).open(&path).ok();
if log::set_logger(Box::leak(Box::new(FileLog(std::sync::Mutex::new(file))))).is_ok() {
if log::set_logger(Box::leak(Box::new(FileLog(craft_anything_platform::sync::Mutex::new(file))))).is_ok() {
log::set_max_level(log::LevelFilter::Info);
}
let default_hook = std::panic::take_hook();
@@ -102,4 +102,5 @@ fn log_to_file() {
log::error!("panic: {info}");
default_hook(info);
}));
craft_anything_platform::sync::start_watchdog(craft_anything_engine::settings::config_dir().join("logs"));
}
+6
View File
@@ -2,3 +2,9 @@ allow-unwrap-in-tests = true
allow-expect-in-tests = true
allow-panic-in-tests = true
allow-indexing-slicing-in-tests = true
# Locks must be able to notice a deadlock and get out of it; see crates/platform/src/sync.rs.
disallowed-types = [
{ path = "std::sync::Mutex", reason = "use craft_anything_platform::sync::Mutex, which can't hang" },
{ path = "std::sync::MutexGuard", reason = "use craft_anything_platform::sync::MutexGuard" },
]
+2 -1
View File
@@ -11,11 +11,12 @@ use crate::prompt;
use crate::{AgentEvent, Conversation};
use craft_anything_engine::{CallOptions, Caller, CancelToken, Consent, Engine, Registry, ToolOutput, Toolset};
use craft_anything_evidence::{ErrorCode, ToolError};
use craft_anything_platform::sync::Mutex;
use serde_json::{Value, json};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::Duration;
/// Byte budget for one tool result sent to the model.
+3 -2
View File
@@ -9,6 +9,7 @@
use super::{McpCommand, display_for};
use crate::{AgentEvent, Conversation};
use craft_anything_platform::sync::Mutex;
use craft_anything_platform::{Process, RunningProcess, StdinMode};
use serde_json::{Value, json};
use std::collections::{HashMap, VecDeque};
@@ -17,7 +18,7 @@ use std::path::PathBuf;
use std::process::ChildStdin;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, Sender};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::Duration;
/// Agents known to speak ACP, and the arguments that start them in that mode.
@@ -458,7 +459,7 @@ impl AcpConversation {
}
}
fn lock(&self) -> std::sync::MutexGuard<'_, Option<Shared>> {
fn lock(&self) -> craft_anything_platform::sync::MutexGuard<'_, Option<Shared>> {
self.shared.lock().unwrap_or_else(PoisonError::into_inner)
}
+2 -1
View File
@@ -18,13 +18,14 @@ use crate::model::Stop;
use crate::{AgentEvent, BackendConfig, Conversation};
use craft_anything_evidence::ToolError;
use craft_anything_platform::process::Process;
use craft_anything_platform::sync::Mutex;
use serde_json::{Value, json};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, Sender};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::Duration;
/// Name our MCP server is registered under.
+2 -1
View File
@@ -27,12 +27,13 @@ use craft_anything_engines::game;
use craft_anything_evidence::{ErrorCode, ToolError};
pub use craft_anything_platform::serve::StaticServer;
use craft_anything_platform::serve::serve_dir;
use craft_anything_platform::sync::Mutex;
use serde_json::{Value, json};
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc::{Receiver, Sender};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::{Duration, Instant};
mod brief;
+1 -1
View File
@@ -2,10 +2,10 @@
//! engine runs the tools. No real network.
use super::*;
use craft_anything_platform::sync::Mutex;
use serde_json::{Value, json};
use std::io::{BufRead, BufReader, Read, Write};
use std::net::TcpListener;
use std::sync::Mutex;
#[derive(Debug, Clone)]
struct Seen {
+17 -2
View File
@@ -37,8 +37,9 @@ pub use target::{Target, TargetKind};
pub use tool::{Caller, CancelToken, Ctx, Effects, Progress, Requires, ToolOutput, ToolSpec, Toolset};
use craft_anything_evidence::{ErrorCode, ToolError};
use craft_anything_platform::sync::{Mutex, MutexGuard};
use serde_json::Value;
use std::sync::{Mutex, MutexGuard, PoisonError};
use std::sync::PoisonError;
/// Whether the caller has the user's consent for a tool that runs code or changes a device.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -140,11 +141,13 @@ impl Engine {
}
/// Lock the session (recovering from a poisoned lock rather than cascading a panic).
#[track_caller]
pub fn session(&self) -> MutexGuard<'_, Session> {
self.session.lock().unwrap_or_else(PoisonError::into_inner)
}
/// Try to lock without blocking (the UI uses this so a long tool never freezes a frame).
#[track_caller]
pub fn try_session(&self) -> Option<MutexGuard<'_, Session>> {
match self.session.try_lock() {
Ok(g) => Some(g),
@@ -231,7 +234,19 @@ impl Engine {
let spec = *self.registry.get(id).ok_or_else(|| {
ToolError::invalid(format!("No tool named {id}")).with_remediation("List tools with `craft-anything-cli commands` or the command palette.")
})?;
let mut session = self.session();
// Stop works while waiting for the session, and a holder that looks stuck is an error here,
// not a wait forever. Tools are long jobs: the label says so (they have their own deadlines).
let cancel = opts.cancel.clone();
let mut session = match self.session.lock_unless(&|| cancel.is_cancelled()) {
Ok(g) => g.unwrap_or_else(PoisonError::into_inner),
Err(craft_anything_platform::sync::GaveUp::Stopped) => return Err(ToolError::cancelled()),
Err(craft_anything_platform::sync::GaveUp::Stuck(why)) => {
return Err(ToolError::new(ErrorCode::TimedOut, format!("{} couldn't start because another part of the app is stuck.", spec.title))
.with_remediation("Restart the app; your work is kept.")
.with_details(serde_json::json!({"lock": why})));
}
};
session.label(spec.id);
if session.clean_room() && !clean_room::allows(spec.id) {
return Err(clean_room::refusal(spec.id));
}
+2 -1
View File
@@ -20,13 +20,14 @@ use crate::inbox::now_ms;
use crate::versioned;
use crate::workspace::{self, Workspace};
use craft_anything_evidence::{ErrorCode, ToolError};
use craft_anything_platform::sync::Mutex;
use craft_anything_platform::{Ended, Launch, Launched};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Mutex, PoisonError};
use std::sync::PoisonError;
use std::time::Duration;
/// The folder in a project that holds its pins.
+2 -1
View File
@@ -26,11 +26,12 @@ use crate::session::Session;
use crate::tool::{Ctx, Effects, Requires, ToolOutput};
use craft_anything_engines::mcp_client::{McpClient, RemoteTool, Server, ServerCommand, ServerUrl, content_text, url_allowed};
use craft_anything_evidence::{Authority, Confidence, Environment, ErrorCode, EvidenceBuilder, Subject, ToolError};
use craft_anything_platform::sync::Mutex;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::collections::{BTreeMap, HashMap};
use std::path::PathBuf;
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::Duration;
/// How long a server may take to start and answer the handshake or its tool list.
+2 -1
View File
@@ -5,8 +5,9 @@
use crate::inbox::now_ms;
use crate::workspace::Workspace;
use craft_anything_evidence::ToolError;
use craft_anything_platform::sync::Mutex;
use serde::{Deserialize, Serialize};
use std::sync::{Mutex, PoisonError};
use std::sync::PoisonError;
/// Where they're kept in the project folder.
pub const QUESTIONS_FILE: &str = "questions.json";
+2 -1
View File
@@ -36,7 +36,8 @@ impl RuntimeTool {
/// A run-time group's id kept for the process, once per distinct id.
pub fn custom_toolset(id: &str) -> Option<Toolset> {
use std::sync::{Mutex, OnceLock, PoisonError};
use craft_anything_platform::sync::Mutex;
use std::sync::{OnceLock, PoisonError};
static SEEN: OnceLock<Mutex<Vec<&'static str>>> = OnceLock::new();
if let Some(t) = Toolset::from_id(id) {
return Some(t);
+2 -1
View File
@@ -673,9 +673,10 @@ fn android_jeb_decompile(ctx: &mut Ctx<'_>, params: Value) -> Result<ToolOutput,
mod tests {
use super::*;
use crate::{CallOptions, Caller, Engine};
use craft_anything_platform::sync::Mutex;
use std::io::BufRead;
use std::net::TcpListener;
use std::sync::{Arc, Mutex};
use std::sync::Arc;
fn engine(url: Option<String>) -> Engine {
let mut s = crate::testing::session();
+1 -1
View File
@@ -374,7 +374,7 @@ mod tests {
/// Stands in for fal.ai: records each request's endpoint and answers with a few bytes.
#[derive(Default)]
struct StandIn(std::sync::Mutex<Vec<String>>);
struct StandIn(craft_anything_platform::sync::Mutex<Vec<String>>);
impl craft_anything_engines::assets::AssetProvider for StandIn {
fn id(&self) -> &'static str {
+2 -1
View File
@@ -18,12 +18,13 @@ use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_p
use craft_anything_engines::frida::{Attach, Frida, Remote, Selector};
use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, Evidence, EvidenceBuilder, Limitation, Subject, ToolError, sha256_hex};
use craft_anything_platform::process::{Cancel, Output};
use craft_anything_platform::sync::Mutex;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::Duration;
pub fn specs() -> Vec<ToolSpec> {
+2 -1
View File
@@ -8,13 +8,14 @@ use super::native_common::spec;
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
use craft_anything_engines::gdb::GdbSession;
use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, EvidenceBuilder, Limitation, Subject, ToolError, sha256_hex};
use craft_anything_platform::sync::Mutex;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::Duration;
pub fn specs() -> Vec<ToolSpec> {
@@ -11,11 +11,12 @@ use crate::project::write_atomic;
use crate::session::Session;
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, empty_schema, parse_params, schema_of};
use craft_anything_evidence::{Confidence, ErrorCode, ToolError};
use craft_anything_platform::sync::Mutex;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::path::PathBuf;
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
pub fn specs() -> Vec<ToolSpec> {
let w = Effects::MUTATES_SESSION.union(Effects::WRITES_FILESYSTEM);
+2 -1
View File
@@ -7,12 +7,13 @@ use crate::inbox::{Answer, Inbox};
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
use crate::workspace::Workspace;
use craft_anything_evidence::{ErrorCode, ToolError};
use craft_anything_platform::sync::Mutex;
use schemars::JsonSchema;
use serde::Deserialize;
use serde_json::{Value, json};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
/// Most feed items one call returns.
const MAX_ITEMS: usize = 200;
@@ -7,13 +7,14 @@ use super::native_common::spec;
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
use craft_anything_engines::rizin::{RizinSession, Target};
use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, EvidenceBuilder, Limitation, Subject, ToolError, sha256_hex};
use craft_anything_platform::sync::Mutex;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::Duration;
pub fn specs() -> Vec<ToolSpec> {
+2 -1
View File
@@ -13,6 +13,7 @@ use crate::versioned::{Step, upgrade};
use craft_anything_analysis::standin::{self, Answer, BodyLeftOut, BuildOptions, NearbyRoute, NoMatchReason, STANDIN_VERSION, StandIn, Turns};
use craft_anything_evidence::{Authority, Confidence, Coverage, ErrorCode, EvidenceBuilder, Limitation, Location, Subject, ToolError, sha256_hex};
use craft_anything_platform::standin_server::{self, Answerer, Limits, LoopbackServer, Request, Response};
use craft_anything_platform::sync::Mutex;
use schemars::JsonSchema;
use serde::Deserialize;
use serde_json::{Value, json};
@@ -20,7 +21,7 @@ use std::collections::BTreeMap;
#[cfg(test)]
use std::path::Path;
use std::path::PathBuf;
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
/// What `web_standin_build` writes in the folder it's given.
pub const STANDIN_FILE: &str = "standin.json";
+1 -1
View File
@@ -1230,7 +1230,7 @@ mod tests {
/// The last lines a launched browser wrote to stderr, for a failure's message.
#[derive(Clone, Default)]
struct StderrTail(std::sync::Arc<std::sync::Mutex<std::collections::VecDeque<String>>>);
struct StderrTail(std::sync::Arc<craft_anything_platform::sync::Mutex<std::collections::VecDeque<String>>>);
impl StderrTail {
fn push(&self, line: &str) {
+2 -1
View File
@@ -14,11 +14,12 @@ use super::web_common::{self, Browser, LaunchOptions};
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
use craft_anything_engines::cdp::{self, CdpClient, CdpEvent, Deadline, Endpoint, PAGE_MAX_MESSAGE};
use craft_anything_evidence::{Authority, Confidence, Coverage, ErrorCode, EvidenceBuilder, Limitation, Location, Subject, ToolError, sha256_hex};
use craft_anything_platform::sync::Mutex;
use schemars::JsonSchema;
use serde::Deserialize;
use serde_json::{Value, json};
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::sync::Arc;
use std::time::Duration;
pub fn specs() -> Vec<ToolSpec> {
+2 -1
View File
@@ -179,9 +179,10 @@ fn base64_decode(s: &str) -> Result<Vec<u8>, ToolError> {
mod tests {
use super::*;
use crate::assets::Catalogue;
use craft_anything_platform::sync::Mutex;
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
use std::sync::{Arc, Mutex};
use std::sync::Arc;
/// A tiny local server that answers each request in turn and records what it was sent.
fn serve(answers: Vec<(u16, String)>) -> (String, Arc<Mutex<Vec<String>>>) {
+2 -1
View File
@@ -4,6 +4,7 @@
//! structured records. The engine's GDB tools keep sessions by id and turn results into evidence.
use craft_anything_evidence::{ErrorCode, ToolError};
use craft_anything_platform::sync::Mutex;
use craft_anything_platform::{Process, RunningProcess, StdinMode};
use serde::Serialize;
use serde_json::{Value, json};
@@ -11,7 +12,7 @@ use std::collections::VecDeque;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::ChildStdin;
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::{Duration, Instant};
/// Unread MI lines kept per session; older ones are dropped.
+3 -2
View File
@@ -24,14 +24,15 @@ pub use client::Client;
pub use protocol::{GhidraError, Request, Response, ct_eq};
use craft_anything_platform::process::{Process, RunningProcess, StdinMode};
use craft_anything_platform::sync::Mutex;
use craft_anything_platform::write_atomic_private;
use serde::Serialize;
use serde_json::Value;
use std::ffi::OsString;
use std::net::{Ipv4Addr, SocketAddr};
use std::path::{Path, PathBuf};
use std::sync::PoisonError;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Mutex, PoisonError};
use std::time::{Duration, Instant, SystemTime};
/// The Java post-script we ship and run inside Ghidra (our own code).
@@ -571,7 +572,7 @@ impl Session {
})
}
fn lock(&self) -> std::sync::MutexGuard<'_, Client> {
fn lock(&self) -> craft_anything_platform::sync::MutexGuard<'_, Client> {
self.client.lock().unwrap_or_else(PoisonError::into_inner)
}
+2 -1
View File
@@ -5,6 +5,7 @@
//! by id and turn results into evidence.
use craft_anything_evidence::{ErrorCode, ToolError};
use craft_anything_platform::sync::Mutex;
use craft_anything_platform::{Process, RunningProcess, StdinMode};
use serde::Serialize;
use serde_json::{Value, json};
@@ -12,7 +13,7 @@ use std::collections::VecDeque;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::ChildStdin;
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::{Duration, Instant};
/// Unread lines kept per session; older ones are dropped.
+1
View File
@@ -12,6 +12,7 @@ workspace = true
[dependencies]
craft-anything-evidence = { workspace = true }
craft-anything-engine = { workspace = true }
craft-anything-platform = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
log = { workspace = true }
+2 -1
View File
@@ -20,11 +20,12 @@ pub mod prompts;
use craft_anything_engine::tools::investigation_tools;
use craft_anything_engine::{CallOptions, Caller, CancelToken, Consent, Engine, Progress, ToolOutput, Toolset};
use craft_anything_evidence::{ErrorCode, ToolError, is_evidence_id};
use craft_anything_platform::sync::Mutex;
use serde_json::{Value, json};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::io::{BufRead, Write};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
+2 -1
View File
@@ -30,10 +30,11 @@ use std::ffi::OsString;
use std::path::PathBuf;
use std::time::Duration;
// Only the Unix capture (and its tests) use these.
use crate::sync::Mutex;
#[cfg(unix)]
use std::sync::atomic::{AtomicU64, Ordering};
#[cfg(unix)]
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
#[cfg(unix)]
use std::time::{Instant, SystemTime};
+1
View File
@@ -11,6 +11,7 @@ pub mod fs;
pub mod process;
pub mod serve;
pub mod standin_server;
pub mod sync;
pub mod which;
pub mod window;
+8 -1
View File
@@ -20,6 +20,7 @@
//! Never call [`Process::run`] on a UI thread: it blocks until the child finishes or a deadline hits.
//! Use [`Process::spawn`] and poll [`RunningProcess::try_result`] instead.
use crate::sync::Mutex;
use craft_anything_evidence::{ErrorCode, ToolError};
use process_wrap::std::{ChildWrapper, CommandWrap};
use std::ffi::{OsStr, OsString};
@@ -28,7 +29,7 @@ use std::path::PathBuf;
use std::process::{ChildStdin, Stdio};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::mpsc::{Receiver, RecvTimeoutError, TryRecvError};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
@@ -995,6 +996,12 @@ pub fn system_opener() -> &'static str {
}
}
/// Start this program again, for a restart: the caller then closes itself.
pub fn start_another_copy() -> Result<u32, ToolError> {
let exe = std::env::current_exe().map_err(|e| ToolError::new(ErrorCode::FileError, format!("Couldn't find this program to restart it: {e}")))?;
Launch::new(exe).start()
}
/// Show a file or folder with the system's default handler (a folder opens in the file manager).
/// Fire-and-forget, like [`Launch`].
pub fn open_path(path: impl AsRef<std::path::Path>) -> Result<(), ToolError> {
+633
View File
@@ -0,0 +1,633 @@
//! Locks that can't hang the app. Use [`Mutex`] here instead of `std::sync::Mutex` (clippy refuses
//! the standard one): it has the same API, and it knows who holds what.
//!
//! - **A real deadlock is broken at once.** A thread that waits on a lock it holds itself, or on a
//! cycle of threads waiting on each other, can never get it. The waiter that finds this stops
//! waiting and unwinds (a caught, reported error, never a crash): its locks are released, so
//! everyone else carries on. Every thread's work runs inside a `catch_unwind` that turns this
//! into an error the person sees.
//! - **Long work is left alone.** A holder labelled as a job ([`MutexGuard::label`], such as a tool,
//! which has its own deadlines and Stop) may take hours. Waiting for it is reported, never cut off.
//! - **A holder that isn't a job and never lets go** is reported after [`Limits::stuck_after`], and
//! anyone still waiting gives up after [`Limits::give_up_after`] rather than wait forever.
//! - **[`start_watchdog`] looks every few seconds** and writes what it finds to a log.
//! [`problems`] is what the app shows.
#![allow(clippy::disallowed_types)]
use std::fmt;
use std::ops::{Deref, DerefMut};
use std::panic::Location;
use std::sync::{LockResult, PoisonError, TryLockError, TryLockResult};
use std::time::Duration;
/// When waiting on a lock stops being normal.
#[derive(Debug, Clone, Copy)]
pub struct Limits {
/// How long a holder that isn't a labelled job may keep others waiting before it's reported.
pub stuck_after: Duration,
/// How long anyone waits on such a holder before giving up.
pub give_up_after: Duration,
/// How long waiting on a labelled job goes on before it's worth saying (it isn't cut off).
pub slow_job_after: Duration,
}
impl Default for Limits {
fn default() -> Self {
Limits { stuck_after: Duration::from_secs(120), give_up_after: Duration::from_secs(900), slow_job_after: Duration::from_secs(1800) }
}
}
/// How serious a [`Problem`] is.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ProblemKind {
/// A deadlock that was broken: the thread that found it stopped, and the rest carried on.
Broken,
/// A deadlock that couldn't be broken from here: restarting the app clears it.
Deadlock,
/// Something holds a lock far longer than it should, and others are waiting.
Stuck,
/// Work waits on a long job. Normal for big jobs; it's said so nobody wonders.
SlowJob,
}
/// Something the watchdog or a waiting thread found.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Problem {
pub kind: ProblemKind,
pub summary: String,
pub log: Option<std::path::PathBuf>,
}
/// Why [`Mutex::lock_unless`] came back without the lock.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GaveUp {
Stopped,
/// The holder looks stuck; says who holds it.
Stuck(String),
}
/// A mutual-exclusion lock whose waits are watched (see the module docs).
pub struct Mutex<T: ?Sized> {
inner: std::sync::Mutex<T>,
}
/// Holds a [`Mutex`] until dropped.
pub struct MutexGuard<'a, T: ?Sized + 'a> {
inner: std::sync::MutexGuard<'a, T>,
id: usize,
}
impl<T> Mutex<T> {
pub const fn new(value: T) -> Self {
Mutex { inner: std::sync::Mutex::new(value) }
}
pub fn into_inner(self) -> LockResult<T> {
self.inner.into_inner()
}
}
impl<T: ?Sized> Mutex<T> {
fn id(&self) -> usize {
(&raw const self.inner).cast::<()>() as usize
}
fn wrap<'a>(id: usize, g: std::sync::MutexGuard<'a, T>, at: &'static Location<'static>) -> MutexGuard<'a, T> {
tracker::held(id, at);
MutexGuard { inner: g, id }
}
/// Take the lock, waiting while it's busy. Comes back with the lock unless waiting could never
/// end: then this thread's work unwinds, releasing what it holds, and its `catch_unwind`
/// reports the error.
#[track_caller]
pub fn lock(&self) -> LockResult<MutexGuard<'_, T>> {
let at = Location::caller();
match self.wait(at, &|| false, Limits::default(), false) {
Ok(r) => r,
Err(GaveUp::Stuck(why)) => tracker::unwind(why),
// `stopped` above never says stop.
Err(GaveUp::Stopped) => tracker::unwind(format!("A wait at {at} stopped for no reason.")),
}
}
/// [`Mutex::lock`] for work that can report an error: gives up when `stop` says so, or as soon
/// as the holder looks stuck, instead of waiting on. A deadlock still unwinds.
#[track_caller]
pub fn lock_unless(&self, stop: &dyn Fn() -> bool) -> Result<LockResult<MutexGuard<'_, T>>, GaveUp> {
self.wait(Location::caller(), stop, Limits::default(), true)
}
fn wait(&self, at: &'static Location<'static>, stop: &dyn Fn() -> bool, limits: Limits, early: bool) -> Result<LockResult<MutexGuard<'_, T>>, GaveUp> {
let id = self.id();
let mut pause = Duration::from_millis(1);
let mut waiting = false;
loop {
match self.inner.try_lock() {
Ok(g) => {
if waiting {
tracker::end_wait();
}
return Ok(Ok(Self::wrap(id, g, at)));
}
Err(TryLockError::Poisoned(p)) => {
if waiting {
tracker::end_wait();
}
return Ok(Err(PoisonError::new(Self::wrap(id, p.into_inner(), at))));
}
Err(TryLockError::WouldBlock) => {}
}
if !waiting {
tracker::begin_wait(id, at);
waiting = true;
}
// Under one lock, so two threads in a cycle can't both decide to break it.
if let Some(why) = tracker::must_give_up(id, limits, early) {
match why {
tracker::Verdict::Deadlock(text) => tracker::unwind(text),
tracker::Verdict::Stuck(text) => {
tracker::end_wait();
return Err(GaveUp::Stuck(text));
}
}
}
if stop() {
tracker::end_wait();
return Err(GaveUp::Stopped);
}
tracker::pause(pause);
pause = (pause * 2).min(Duration::from_millis(20));
}
}
#[track_caller]
pub fn try_lock(&self) -> TryLockResult<MutexGuard<'_, T>> {
let at = Location::caller();
let id = self.id();
match self.inner.try_lock() {
Ok(g) => Ok(Self::wrap(id, g, at)),
Err(TryLockError::Poisoned(p)) => Err(TryLockError::Poisoned(PoisonError::new(Self::wrap(id, p.into_inner(), at)))),
Err(TryLockError::WouldBlock) => Err(TryLockError::WouldBlock),
}
}
pub fn get_mut(&mut self) -> LockResult<&mut T> {
self.inner.get_mut()
}
}
impl<T: Default> Default for Mutex<T> {
fn default() -> Self {
Mutex::new(T::default())
}
}
impl<T> From<T> for Mutex<T> {
fn from(value: T) -> Self {
Mutex::new(value)
}
}
impl<T: ?Sized> fmt::Debug for Mutex<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("Mutex")
}
}
impl<T: ?Sized> MutexGuard<'_, T> {
/// Say this hold is a job that may take a long time, by name. It's then never treated as stuck.
pub fn label(&self, job: &'static str) {
tracker::label(self.id, job);
}
}
impl<T: ?Sized> Deref for MutexGuard<'_, T> {
type Target = T;
fn deref(&self) -> &T {
&self.inner
}
}
impl<T: ?Sized> DerefMut for MutexGuard<'_, T> {
fn deref_mut(&mut self) -> &mut T {
&mut self.inner
}
}
impl<T: ?Sized + fmt::Debug> fmt::Debug for MutexGuard<'_, T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
(**self).fmt(f)
}
}
impl<T: ?Sized + fmt::Display> fmt::Display for MutexGuard<'_, T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
(**self).fmt(f)
}
}
impl<T: ?Sized> Drop for MutexGuard<'_, T> {
fn drop(&mut self) {
tracker::released(self.id);
}
}
/// What's wrong right now and what went wrong recently, for the app to show.
pub fn problems() -> Vec<Problem> {
tracker::problems()
}
/// Add a problem found some other way, so it's logged and shown like the rest.
pub fn report(kind: ProblemKind, summary: impl Into<String>) {
tracker::report(kind, summary.into());
}
/// Forget a problem the person has dismissed.
pub fn dismiss(summary: &str) {
tracker::dismiss(summary);
}
/// Check every few seconds for locks in trouble, writing each new problem to a log in `log_dir`.
/// Starts once per process; apps call it, tests don't.
pub fn start_watchdog(log_dir: std::path::PathBuf) {
tracker::start_watchdog(log_dir);
}
#[cfg(target_arch = "wasm32")]
mod tracker {
//! One thread and no clock: nothing to watch, and nothing that could wait.
use super::*;
pub enum Verdict {
Deadlock(String),
Stuck(String),
}
pub fn held(_: usize, _: &'static Location<'static>) {}
pub fn released(_: usize) {}
pub fn label(_: usize, _: &'static str) {}
pub fn begin_wait(_: usize, _: &'static Location<'static>) {}
pub fn end_wait() {}
pub fn pause(_: Duration) {}
pub fn must_give_up(id: usize, _: Limits, _: bool) -> Option<Verdict> {
Some(Verdict::Deadlock(format!("Lock {id} is busy on the only thread.")))
}
pub fn unwind(text: String) -> ! {
std::panic::resume_unwind(Box::new(text))
}
pub fn problems() -> Vec<Problem> {
Vec::new()
}
pub fn dismiss(_: &str) {}
pub fn report(_: ProblemKind, _: String) {}
pub fn start_watchdog(_: std::path::PathBuf) {}
}
#[cfg(not(target_arch = "wasm32"))]
mod tracker {
use super::*;
use std::collections::HashMap;
use std::sync::OnceLock;
use std::thread::ThreadId;
use std::time::Instant;
/// How often the watchdog looks.
const WATCH_EVERY: Duration = Duration::from_secs(5);
/// Problems remembered for the app.
const KEEP_PROBLEMS: usize = 8;
pub enum Verdict {
Deadlock(String),
Stuck(String),
}
struct Hold {
thread: ThreadId,
thread_name: String,
at: &'static Location<'static>,
since: Instant,
job: Option<&'static str>,
}
struct Wait {
lock: usize,
thread_name: String,
at: &'static Location<'static>,
since: Instant,
}
#[derive(Default)]
struct Registry {
holds: HashMap<usize, Hold>,
waits: HashMap<ThreadId, Wait>,
problems: Vec<Problem>,
dismissed: Vec<String>,
}
/// Held only for short bookkeeping, never while taking another lock, so it can't deadlock.
fn registry() -> std::sync::MutexGuard<'static, Registry> {
static REGISTRY: OnceLock<std::sync::Mutex<Registry>> = OnceLock::new();
REGISTRY.get_or_init(Default::default).lock().unwrap_or_else(PoisonError::into_inner)
}
fn thread_name() -> String {
std::thread::current().name().unwrap_or("unnamed").to_string()
}
pub fn held(id: usize, at: &'static Location<'static>) {
let hold = Hold { thread: std::thread::current().id(), thread_name: thread_name(), at, since: Instant::now(), job: None };
registry().holds.insert(id, hold);
}
pub fn released(id: usize) {
registry().holds.remove(&id);
}
pub fn label(id: usize, job: &'static str) {
if let Some(h) = registry().holds.get_mut(&id) {
h.job = Some(job);
}
}
pub fn begin_wait(lock: usize, at: &'static Location<'static>) {
let me = std::thread::current().id();
registry().waits.insert(me, Wait { lock, thread_name: thread_name(), at, since: Instant::now() });
}
pub fn end_wait() {
let me = std::thread::current().id();
registry().waits.remove(&me);
}
pub fn pause(d: Duration) {
std::thread::sleep(d);
}
/// The chain of threads waiting on each other that leads from `start` back to it, if there is one.
fn cycle(r: &Registry, start: ThreadId) -> Option<String> {
let mut parts = Vec::new();
let mut thread = start;
// A chain can't be longer than the number of waiting threads.
for _ in 0..=r.waits.len() {
let wait = r.waits.get(&thread)?;
let hold = r.holds.get(&wait.lock)?;
parts.push(format!("thread '{}' at {} waits on a lock taken at {} by thread '{}'", wait.thread_name, wait.at, hold.at, hold.thread_name));
if hold.thread == start {
return Some(parts.join("; "));
}
thread = hold.thread;
}
None
}
fn remember(r: &mut Registry, kind: ProblemKind, summary: String) {
if r.dismissed.contains(&summary) || r.problems.iter().any(|p| p.summary == summary) {
return;
}
log::error!("{summary}");
r.problems.push(Problem { kind, summary, log: None });
if r.problems.len() > KEEP_PROBLEMS {
r.problems.remove(0);
}
}
/// Should the waiting thread stop? Removes its wait when it should, in the same step as the
/// decision, so the next thread in a cycle no longer sees one and carries on.
pub fn must_give_up(lock: usize, limits: Limits, early: bool) -> Option<Verdict> {
let me = std::thread::current().id();
let mut r = registry();
if let Some(chain) = cycle(&r, me) {
// Unwinding during an unwind would abort the process; left waiting, it's reported
// by the watchdog as a deadlock to restart from.
if std::thread::panicking() {
return None;
}
r.waits.remove(&me);
let text = format!("A deadlock was found and broken: {chain}.");
remember(&mut r, ProblemKind::Broken, text.clone());
return Some(Verdict::Deadlock(text));
}
let waited = r.waits.get(&me)?.since.elapsed();
let hold = r.holds.get(&lock)?;
if hold.job.is_some() {
return None;
}
let limit = if early { limits.stuck_after } else { limits.give_up_after };
if waited < limit || hold.since.elapsed() < limit {
return None;
}
let text = format!(
"A lock taken at {} by thread '{}' hasn't been released for {} s, so a wait was given up.",
hold.at,
hold.thread_name,
hold.since.elapsed().as_secs()
);
if std::thread::panicking() {
return None;
}
r.waits.remove(&me);
remember(&mut r, ProblemKind::Stuck, text.clone());
Some(if early { Verdict::Stuck(text) } else { Verdict::Deadlock(text) })
}
pub fn unwind(text: String) -> ! {
end_wait();
// Not `panic!`: this is a caught, expected way out, and the panic hook isn't what reports it.
std::panic::resume_unwind(Box::new(text))
}
pub fn problems() -> Vec<Problem> {
registry().problems.clone()
}
pub fn report(kind: ProblemKind, summary: String) {
remember(&mut registry(), kind, summary);
}
pub fn dismiss(summary: &str) {
let mut r = registry();
r.problems.retain(|p| p.summary != summary);
r.dismissed.push(summary.to_string());
}
/// What's wrong right now, whole-app: the watchdog's look at everything.
pub(super) fn scan(limits: Limits) {
let mut r = registry();
let mut found = Vec::new();
for (thread, wait) in &r.waits {
if let Some(chain) = cycle(&r, *thread) {
found.push((ProblemKind::Deadlock, format!("A deadlock can't be broken from here, so restart the app: {chain}.")));
continue;
}
let Some(hold) = r.holds.get(&wait.lock) else { continue };
let waited = wait.since.elapsed();
let (kind, limit, what) = match hold.job {
Some(job) => (ProblemKind::SlowJob, limits.slow_job_after, format!("'{job}' is still running")),
None => (ProblemKind::Stuck, limits.stuck_after, format!("a lock taken at {} by thread '{}' hasn't been released", hold.at, hold.thread_name)),
};
if waited >= limit && hold.since.elapsed() >= limit {
found.push((kind, format!("Thread '{}' at {} is waiting because {what}.", wait.thread_name, wait.at)));
}
}
for (kind, text) in found {
remember(&mut r, kind, text);
}
}
pub fn start_watchdog(log_dir: std::path::PathBuf) {
static STARTED: OnceLock<()> = OnceLock::new();
if STARTED.set(()).is_err() {
return;
}
let spawned = std::thread::Builder::new().name("lock-watch".into()).spawn(move || {
loop {
std::thread::sleep(WATCH_EVERY);
scan(Limits::default());
write_logs(&log_dir);
}
});
if let Err(e) = spawned {
log::warn!("couldn't start the lock watchdog: {e}");
}
}
/// Give each new problem its log file. Files are written outside the registry's lock.
pub(super) fn write_logs(dir: &std::path::Path) {
let new: Vec<(usize, String)> = registry().problems.iter().enumerate().filter(|(_, p)| p.log.is_none()).map(|(i, p)| (i, p.summary.clone())).collect();
for (n, summary) in new {
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);
let path = dir.join(format!("lock-{stamp}-{n}.log"));
let text = format!("Craft Anything lock report\n\n{summary}\n\nEverything holding or waiting on a lock now:\n{}\n", snapshot());
let saved = std::fs::create_dir_all(dir)
.map_err(|e| e.to_string())
.and_then(|()| crate::write_atomic_private(&path, text.as_bytes()).map_err(|e| e.message));
match saved {
Ok(()) => {
if let Some(p) = registry().problems.iter_mut().find(|p| p.summary == summary) {
p.log = Some(path);
}
}
Err(e) => log::warn!("couldn't save a lock report: {e}"),
}
}
}
fn snapshot() -> String {
let r = registry();
let holds = r.holds.values().map(|h| {
let job = h.job.map(|j| format!(" for '{j}'")).unwrap_or_default();
format!(" held by thread '{}' at {} for {} s{job}", h.thread_name, h.at, h.since.elapsed().as_secs())
});
let waits = r.waits.values().map(|w| format!(" waited on by thread '{}' at {} for {} s", w.thread_name, w.at, w.since.elapsed().as_secs()));
holds.chain(waits).collect::<Vec<_>>().join("\n")
}
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use super::*;
use std::sync::Arc;
use std::sync::mpsc::channel;
const LIMIT: Duration = Duration::from_secs(20);
fn text(p: Box<dyn std::any::Any + Send>) -> String {
p.downcast_ref::<String>().cloned().unwrap_or_default()
}
#[test]
fn a_thread_that_locks_twice_stops_instead_of_hanging() {
let m = Arc::new(Mutex::new(1));
let m2 = m.clone();
let (done, finished) = channel();
std::thread::spawn(move || {
// Exactly the shape of the bug: both guards live until the statement ends.
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| *m2.lock().unwrap() + *m2.lock().unwrap()));
let _ = done.send(r.map_err(text));
});
let why = finished.recv_timeout(LIMIT).expect("it hung").unwrap_err();
assert!(why.contains("deadlock") && why.contains("sync.rs"), "says where: {why}");
assert!(problems().iter().any(|p| p.kind == ProblemKind::Broken && p.summary == why));
// The lock is free again, poisoned by the unwind and still usable.
assert_eq!(*m.lock().unwrap_or_else(PoisonError::into_inner), 1);
}
#[test]
fn two_threads_in_a_cycle_are_untangled_and_one_finishes() {
let (a, b) = (Arc::new(Mutex::new(0)), Arc::new(Mutex::new(0)));
let (done, finished) = channel();
for (first, second) in [(a.clone(), b.clone()), (b, a)] {
let done = done.clone();
std::thread::spawn(move || {
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _one = first.lock().unwrap_or_else(PoisonError::into_inner);
std::thread::sleep(Duration::from_millis(100));
let _two = second.lock().unwrap_or_else(PoisonError::into_inner);
}));
let _ = done.send(r.is_ok());
});
}
let results = [finished.recv_timeout(LIMIT).expect("it hung"), finished.recv_timeout(LIMIT).expect("it hung")];
assert_eq!(results.iter().filter(|ok| **ok).count(), 1, "one stops, so the other finishes: {results:?}");
}
#[test]
fn a_long_job_is_waited_for_however_short_the_limits() {
let m = Arc::new(Mutex::new(0));
let g = m.lock().unwrap();
g.label("big-job");
let (m2, (done, finished)) = (m.clone(), channel());
std::thread::spawn(move || {
let limits = Limits { stuck_after: Duration::ZERO, give_up_after: Duration::ZERO, slow_job_after: Duration::ZERO };
let got = m2.wait(Location::caller(), &|| false, limits, true).map(|_| ());
let _ = done.send(got);
});
assert!(finished.recv_timeout(Duration::from_millis(300)).is_err(), "still waiting on the job");
drop(g);
assert_eq!(finished.recv_timeout(LIMIT).expect("it hung"), Ok(()));
}
#[test]
fn a_holder_that_never_lets_go_is_given_up_on_and_stop_works_while_waiting() {
let m = Arc::new(Mutex::new(0));
let g = m.lock().unwrap();
let (m2, (done, finished)) = (m.clone(), channel());
std::thread::spawn(move || {
let limits = Limits { stuck_after: Duration::from_millis(50), ..Limits::default() };
let _ = done.send(m2.wait(Location::caller(), &|| false, limits, true).map(|_| ()));
});
match finished.recv_timeout(LIMIT).expect("it hung") {
Err(GaveUp::Stuck(why)) => assert!(why.contains("hasn't been released"), "{why}"),
other => panic!("{other:?}"),
}
let stopped = m.clone();
let (done, finished) = channel();
std::thread::spawn(move || {
let _ = done.send(stopped.lock_unless(&|| true).map(|_| ()));
});
assert_eq!(finished.recv_timeout(LIMIT).expect("it hung"), Err(GaveUp::Stopped));
drop(g);
assert!(m.try_lock().is_ok());
}
#[test]
fn the_watchdog_reports_a_stuck_holder_in_a_log_and_can_forget_it() {
let dir = tempfile::tempdir().unwrap();
let m = Arc::new(Mutex::new(0));
let g = m.lock().unwrap();
let (m2, (done, finished)) = (m.clone(), channel());
std::thread::spawn(move || {
let _ = done.send(m2.lock_unless(&|| false).map(|_| ()));
});
std::thread::sleep(Duration::from_millis(50));
tracker::scan(Limits { stuck_after: Duration::ZERO, ..Limits::default() });
tracker::write_logs(dir.path());
let p = problems().into_iter().find(|p| p.kind == ProblemKind::Stuck && p.summary.contains("sync.rs")).expect("reported");
let log = std::fs::read_to_string(p.log.clone().expect("has a log")).unwrap();
assert!(log.contains(&p.summary) && log.contains("held by thread"), "{log}");
dismiss(&p.summary);
assert!(!problems().iter().any(|q| q.summary == p.summary));
drop(g);
assert_eq!(finished.recv_timeout(LIMIT).expect("it hung"), Ok(()));
}
}
+2 -1
View File
@@ -2,8 +2,9 @@
//! a worker thread; the UI polls finished results once per frame and never waits.
use craft_anything_engine::{CancelToken, Progress};
use craft_anything_platform::sync::Mutex;
use std::sync::mpsc::{Receiver, Sender, channel};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, PoisonError};
use std::time::Instant;
/// What a worker hands back. `T` is the app's result type.
+32
View File
@@ -836,6 +836,37 @@ impl CraftApp {
}
}
/// Says when a lock got stuck (crates/platform/src/sync.rs), with a way out.
fn stuck_banner(&mut self, ui: &mut egui::Ui, ctx: &egui::Context) {
use craft_anything_platform::sync::{self, ProblemKind};
let Some(problem) = sync::problems().into_iter().find(|p| p.kind != ProblemKind::SlowJob) else { return };
let t = theme::Tokens::get(ctx);
let text = match problem.kind {
ProblemKind::Broken => "Part of the app got stuck, so it was stopped and the rest carried on. Your work is kept.",
_ => "Part of the app is stuck. Restarting clears it, and your work is kept.",
};
egui::Panel::top("stuck-banner").frame(egui::Frame::new().fill(t.card).inner_margin(egui::Margin::symmetric(14, 8))).show(ui, |ui| {
ui.horizontal_wrapped(|ui| {
icons::show(ui, "triangle-alert", 16.0, t.danger);
ui.label(egui::RichText::new(i18n::tr(text)).color(t.text)).on_hover_text(problem.summary.clone());
if let Some(path) = &problem.log
&& widgets::secondary(ui, None, "Open log").on_hover_text(path.display().to_string()).clicked()
{
self.open_log(path);
}
if widgets::secondary(ui, None, "Restart").clicked() {
match craft_anything_platform::process::start_another_copy() {
Ok(_) => ctx.send_viewport_cmd(egui::ViewportCommand::Close),
Err(e) => self.toast(e.message, true),
}
}
if widgets::secondary(ui, None, "Dismiss").clicked() {
sync::dismiss(&problem.summary);
}
});
});
}
/// Show a failure log in the system's viewer for text files.
pub(crate) fn open_log(&mut self, path: &std::path::Path) {
if let Err(e) = craft_anything_platform::open_path(path) {
@@ -1428,6 +1459,7 @@ impl CraftApp {
egui::Panel::top("titlebar").exact_size(titlebar::HEIGHT).frame(egui::Frame::new().fill(t.chrome)).show(ui, |ui| {
guard_ui(ui, "title bar", |ui| titlebar::show(self, ui));
});
self.stuck_banner(ui, &ctx);
if !self.settings.first_run_done {
egui::CentralPanel::default().frame(egui::Frame::new().fill(t.canvas)).show(ui, |ui| guard_ui(ui, "first run", |ui| first_run::show(self, ui)));
titlebar::window_edge(&ctx);
+1
View File
@@ -129,6 +129,7 @@ const ACTIONS: &[Action] = &[
/// `CraftApp` methods that aren't person-facing actions: callbacks, polling, drawing, helpers.
#[rustfmt::skip]
const PLUMBING: &[(&str, &str)] = &[
("stuck_banner", "draws what the lock watchdog found; `sync::problems` and its log are the headless side"),
("act", "runs a button's tool on a worker (play, preview, open folder)"),
("waiting_question", "finds the question an answer is for"),
("acted", "shows a button tool's result (toast, open the preview)"),
+29
View File
@@ -0,0 +1,29 @@
//! The stuck-lock banner. In a test program of its own: what it shows is the whole process's, and
//! a banner appearing moves everything under it, which makes clicks in tests running beside it miss.
use craft_anything_engine::Mode;
use craft_anything_platform::sync;
use craft_anything_ui_egui::demo;
use egui_kittest::Harness;
use egui_kittest::kittest::Queryable;
#[test]
fn a_stuck_lock_is_shown_with_a_way_out() {
let mut h = Harness::builder().with_size(egui::vec2(1280.0, 820.0)).with_max_steps(32).build_eframe(move |cc| {
craft_anything_ui_egui::CraftApp::setup_context(&cc.egui_ctx);
demo::app(demo::engine(Mode::Beginner, true))
});
let summary = "a lock taken at x.rs:1 is stuck";
h.run_steps(2);
assert!(h.query_by_label("Restart").is_none(), "nothing wrong, nothing shown");
sync::report(sync::ProblemKind::SlowJob, "a long job is still running");
h.run_steps(2);
assert!(h.query_by_label("Restart").is_none(), "a long job is no alarm");
sync::report(sync::ProblemKind::Stuck, summary);
h.run_steps(4);
h.get_by_label("Restart");
h.get_by_label("Dismiss").click();
h.run_steps(4);
assert!(!sync::problems().iter().any(|p| p.summary == summary));
assert!(h.query_by_label("Restart").is_none());
}
+5
View File
@@ -43,6 +43,11 @@ against a stand-in server, not yet a real Cutter). Every tool is listed in `docs
## Known limits
- Locks: every mutex but the browser viewer's goes through `platform::sync`. A deadlock on one thread or across
threads is broken when it forms, with an error and a report in `logs/`; a watchdog looks every 5 s; a holder
that isn't a labelled job is reported after 2 minutes and waited on for 15 at most. Tools are labelled, so they
may run as long as they need. Checked by unit tests of each case, not by a live deadlock in the desktop app.
Waits on channels and `join` aren't watched; each child program has its own deadlines.
- A failed agent CLI run says what ran and what it printed, and writes a private log to `logs/` in the
config folder (the newest 50 are kept; the person's message is left out; **Open log** in the app, `log:`
in the CLI). Checked live against a GitHub ssh alias from the CLI; the button by a UI test, not by
+11
View File
@@ -68,6 +68,10 @@ what happens without a program builds its session in the mode the tool runs in.
`a`, so `b`'s tests never ran; the rerun was filtered to the failing test, and a broken engine id
reached `main`. Run with `--no-fail-fast`, and after a fix rerun the whole crates, not a filter.
**A test that changes the whole process.** A UI test raised the process-wide stuck-lock banner; in
CI the banner moved the layout under another test running beside it, and its click missed. It passed
25 runs on a Mac. Anything a test sets for the whole process goes in a test program of its own.
**Fixtures that leak into each other.** A "nothing here" path sat beside a fake game folder in the
same temp directory, and game detection walked up and found the game. Give each case its own
`tempfile::tempdir()`.
@@ -296,6 +300,13 @@ and run `cargo xtask secrets` before every commit (the `.githooks/pre-commit` ho
Update STATUS, the roadmap and generated docs (`cargo xtask tools`, `cargo xtask schemas`) in the
same PR as the change.
**A second lock in one statement.** `CliSpec { deep: !engine.session().clean_room(), logs: engine.session().config_dir(), … }`
looked like two quick reads, but an unbound guard lives to the end of the statement, so the second
waited on the first forever. Every build froze after studying its target while the card calmly
said "This is a big one", and no test ran that line. Read the session once; `cargo xtask locks`
and `platform::sync` now catch this. A wait message must not reassure about what it can't see:
say how long it's been since anything happened. Find a hang with `sample <pid>`: it names the lock.
**An error that points at nothing.** "Check the output below; run Claude Code by hand" came with no
output below (the event carried only message and remediation) and no command, and the program
that failed was really ssh, talking to a git alias from `~/.ssh/config` that had been offered as a
+2
View File
@@ -8,3 +8,5 @@ publish = false
[dependencies]
craft-anything-engine = { workspace = true }
toml = { workspace = true }
syn = { version = "2", features = ["full", "visit"] }
proc-macro2 = { version = "1", features = ["span-locations"] }
+274
View File
@@ -0,0 +1,274 @@
//! `cargo xtask locks`: finds code that would wait on the engine's session lock while already
//! holding it. `.session()` hands back a guard, and a guard that isn't bound to a name still lives
//! until the end of the whole statement, so a second `.session()` in that statement waits forever.
//! At run time such a wait is caught and stopped (crates/platform/src/sync.rs); this finds it first.
//!
//! Flags, in non-test code:
//! - two `.session()` calls whose guards are alive together in one statement;
//! - `match`, `if let` or `while let` on a `.session()`, with another in its body;
//! - `let g = ….session();` followed in that block by `.session()` or `engine.call(…)`, before `drop(g)`.
use std::path::{Path, PathBuf};
use syn::visit::{self, Visit};
fn is_session(m: &syn::ExprMethodCall) -> bool {
m.args.is_empty() && (m.method == "session" || m.method == "try_session")
}
fn is_engine_call(m: &syn::ExprMethodCall) -> bool {
if m.method != "call" {
return false;
}
match &*m.receiver {
syn::Expr::Field(f) => matches!(&f.member, syn::Member::Named(n) if n == "engine"),
syn::Expr::Path(p) => p.path.is_ident("engine"),
_ => false,
}
}
/// `.session()` calls anywhere under a node, closures aside (they run later, if at all).
#[derive(Default)]
struct Everywhere {
lines: Vec<usize>,
with_calls: bool,
}
impl Visit<'_> for Everywhere {
fn visit_expr_closure(&mut self, _: &syn::ExprClosure) {}
fn visit_item(&mut self, _: &syn::Item) {}
fn visit_expr_method_call(&mut self, m: &syn::ExprMethodCall) {
if is_session(m) || (self.with_calls && is_engine_call(m)) {
self.lines.push(m.method.span().start().line);
}
visit::visit_expr_method_call(self, m);
}
}
/// `.session()` calls whose guards live to the end of this statement: nested blocks have their own
/// statements, and a plain `if`'s condition ends before its body.
#[derive(Default)]
struct SameStatement {
lines: Vec<usize>,
/// (first, second) lines of scrutinee problems found on the way.
held_into_body: Vec<(usize, usize)>,
}
impl SameStatement {
fn scrutinee(&mut self, scrutinee: &syn::Expr, body: &dyn Fn(&mut Everywhere)) {
let mut head = Everywhere::default();
head.visit_expr(scrutinee);
if let Some(first) = head.lines.first() {
let mut rest = Everywhere::default();
body(&mut rest);
if let Some(second) = rest.lines.first() {
self.held_into_body.push((*first, *second));
}
}
self.visit_expr(scrutinee);
}
}
impl Visit<'_> for SameStatement {
fn visit_expr_closure(&mut self, _: &syn::ExprClosure) {}
fn visit_item(&mut self, _: &syn::Item) {}
fn visit_block(&mut self, _: &syn::Block) {}
fn visit_expr_method_call(&mut self, m: &syn::ExprMethodCall) {
if is_session(m) {
self.lines.push(m.method.span().start().line);
}
visit::visit_expr_method_call(self, m);
}
fn visit_expr_if(&mut self, e: &syn::ExprIf) {
if let syn::Expr::Let(l) = &*e.cond {
self.scrutinee(&l.expr, &|rest| rest.visit_block(&e.then_branch));
}
// A plain condition is its own scope, checked as if it were a statement of its own.
}
fn visit_expr_while(&mut self, e: &syn::ExprWhile) {
if let syn::Expr::Let(l) = &*e.cond {
self.scrutinee(&l.expr, &|rest| rest.visit_block(&e.body));
}
}
fn visit_expr_match(&mut self, e: &syn::ExprMatch) {
self.scrutinee(&e.expr, &|rest| {
for arm in &e.arms {
rest.visit_arm(arm);
}
});
}
}
/// The name a statement binds a session guard to, when it does.
fn bound_guard(stmt: &syn::Stmt) -> Option<(String, usize)> {
let syn::Stmt::Local(local) = stmt else { return None };
let syn::Pat::Ident(name) = &local.pat else { return None };
let syn::Expr::MethodCall(m) = &*local.init.as_ref()?.expr else { return None };
is_session(m).then(|| (name.ident.to_string(), m.method.span().start().line))
}
fn drops(stmt: &syn::Stmt, name: &str) -> bool {
let syn::Stmt::Expr(syn::Expr::Call(c), _) = stmt else { return false };
let syn::Expr::Path(f) = &*c.func else { return false };
f.path.is_ident("drop") && matches!(c.args.first(), Some(syn::Expr::Path(a)) if a.path.is_ident(name))
}
#[derive(Default)]
struct Finder {
found: Vec<String>,
}
impl Finder {
fn plain_conditions(&mut self, e: &syn::Expr) {
let mut one = SameStatement::default();
one.visit_expr(e);
self.report(one);
}
fn report(&mut self, s: SameStatement) {
if let [first, second, ..] = s.lines[..] {
self.found.push(format!("{first}: this statement takes the session lock again on line {second} while still holding it"));
}
for (first, second) in s.held_into_body {
self.found.push(format!("{first}: the session lock taken here is held through its body, which takes it again on line {second}"));
}
}
}
impl Visit<'_> for Finder {
fn visit_item_mod(&mut self, m: &syn::ItemMod) {
let test = m.attrs.iter().any(|a| a.path().is_ident("cfg") && a.meta.to_token_stream_string().contains("test"));
if !test {
visit::visit_item_mod(self, m);
}
}
fn visit_item_fn(&mut self, f: &syn::ItemFn) {
if !f.attrs.iter().any(|a| a.path().is_ident("test")) {
visit::visit_item_fn(self, f);
}
}
fn visit_expr_if(&mut self, e: &syn::ExprIf) {
if !matches!(&*e.cond, syn::Expr::Let(_)) {
self.plain_conditions(&e.cond);
}
visit::visit_expr_if(self, e);
}
fn visit_block(&mut self, block: &syn::Block) {
for (i, stmt) in block.stmts.iter().enumerate() {
let mut one = SameStatement::default();
one.visit_stmt(stmt);
self.report(one);
if let Some((name, line)) = bound_guard(stmt) {
for later in block.stmts.iter().skip(i + 1) {
if drops(later, &name) {
break;
}
let mut rest = Everywhere { with_calls: true, ..Default::default() };
rest.visit_stmt(later);
if let Some(second) = rest.lines.first() {
self.found.push(format!("{line}: `{name}` holds the session lock, and line {second} needs it again; drop it first"));
break;
}
}
}
}
visit::visit_block(self, block);
}
}
trait MetaText {
fn to_token_stream_string(&self) -> String;
}
impl MetaText for syn::Meta {
fn to_token_stream_string(&self) -> String {
match self {
syn::Meta::List(l) => l.tokens.to_string(),
_ => String::new(),
}
}
}
fn problems_in(source: &str) -> Result<Vec<String>, String> {
let file = syn::parse_file(source).map_err(|e| e.to_string())?;
let mut f = Finder::default();
f.visit_file(&file);
Ok(f.found)
}
fn rust_files(dir: &Path, out: &mut Vec<PathBuf>) {
let Ok(entries) = std::fs::read_dir(dir) else { return };
for e in entries.flatten() {
let p = e.path();
let name = e.file_name().to_string_lossy().into_owned();
if p.is_dir() {
if !matches!(name.as_str(), "tests" | "examples" | "target") {
rust_files(&p, out);
}
} else if name.ends_with(".rs") && name != "tests.rs" {
out.push(p);
}
}
}
pub fn check(root: &Path) -> bool {
let mut files = Vec::new();
rust_files(&root.join("crates"), &mut files);
rust_files(&root.join("apps"), &mut files);
let mut bad = 0;
for file in &files {
let Ok(source) = std::fs::read_to_string(file) else { continue };
if !source.contains("session()") {
continue;
}
let shown = file.strip_prefix(root).unwrap_or(file).display().to_string();
match problems_in(&source) {
Ok(found) => {
for f in found {
eprintln!("{shown}:{f}");
bad += 1;
}
}
Err(e) => {
eprintln!("{shown}: couldn't read it: {e}");
bad += 1;
}
}
}
if bad == 0 {
println!("locks: ok ({} files)", files.len());
} else {
eprintln!("locks: {bad} place(s) would wait on a lock they already hold");
}
bad == 0
}
#[cfg(test)]
mod tests {
use super::problems_in;
fn flagged(body: &str) -> bool {
!problems_in(&format!("fn f() {{ {body} }}")).unwrap().is_empty()
}
#[test]
fn finds_a_lock_held_into_a_second() {
// The bug that started this: both guards live until the struct expression ends.
assert!(flagged("let s = Spec { a: !e.session().x(), b: e.session().y() };"));
assert!(flagged("let n = e.session().a() + e.session().b();"));
assert!(flagged("match e.session().x() { A => e.session().y(), B => 0 };"));
assert!(flagged("if let Some(x) = e.session().x() { e.session().y(); }"));
assert!(flagged("let g = e.session(); e.session().y();"));
assert!(flagged("let g = self.engine.session(); self.engine.call(\"t\", p, o);"));
assert!(flagged("if e.session().a() && e.session().b() { }"));
}
#[test]
fn leaves_correct_code_alone() {
assert!(!flagged("let a = e.session().x(); let b = e.session().y();"));
assert!(!flagged("if e.session().a() { e.session().b(); }"), "a plain condition's guard ends before the body");
assert!(!flagged("let g = e.session(); g.a(); drop(g); e.session().y();"));
assert!(!flagged("let (a, b) = { let s = e.session(); (s.a(), s.b()) }; e.session().y();"));
assert!(!flagged("let g = e.session(); let f = move || other.session();"), "a closure runs later");
assert!(!flagged("match x { A => e.session().y(), B => e.session().z() };"), "only one arm runs");
}
}
+5 -2
View File
@@ -1,10 +1,11 @@
//! `cargo xtask <layers | tools [--check] | wasm | licenses | schemas [--check] | pending [--count] | secrets [--history] | ci | version [set X.Y.Z] | ico <out.ico> <png>...>`
//! `cargo xtask <layers | locks | tools [--check] | wasm | licenses | schemas [--check] | pending [--count] | secrets [--history] | ci | version [set X.Y.Z] | ico <out.ico> <png>...>`
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::{Command, ExitCode};
mod ico;
mod locks;
mod secrets;
/// Workspace crates and their layer. A crate may depend only on strictly lower layers
@@ -35,6 +36,7 @@ fn main() -> ExitCode {
let root = workspace_root();
let ok = match args.first().map(String::as_str) {
Some("layers") => layers(&root),
Some("locks") => locks::check(&root),
Some("tools") => tools(&root, args.iter().any(|a| a == "--check")),
Some("schemas") => schemas(&root, args.iter().any(|a| a == "--check")),
Some("wasm") => wasm(),
@@ -48,6 +50,7 @@ fn main() -> ExitCode {
}
Some("ci") => {
layers(&root)
&& locks::check(&root)
&& secrets::secrets(&root, false)
&& run("cargo", &["fmt", "--all", "--check"])
&& run("cargo", &["clippy", "--workspace", "--all-targets", "--", "-D", "warnings"])
@@ -60,7 +63,7 @@ fn main() -> ExitCode {
}
_ => {
eprintln!(
"usage: cargo xtask <layers | tools [--check] | wasm | licenses | schemas [--check] | pending [--count] | secrets [--history] | ci | version [set X.Y.Z] | ico <out.ico> <png>...>"
"usage: cargo xtask <layers | locks | tools [--check] | wasm | licenses | schemas [--check] | pending [--count] | secrets [--history] | ci | version [set X.Y.Z] | ico <out.ico> <png>...>"
);
false
}