mirror of
https://github.com/storytold/craft-anything.git
synced 2026-11-19 05:50:53 +01:00
Stop deadlocks as they form and watch every lock for one that never lets go (#5)
* Stop deadlocks as they form and watch every lock for one that never lets go Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Give the stuck-lock banner's test a program of its own Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6a120011a1
commit
292d5af06e
47 files changed
+1097
-35
No files matched your search
@@ -68,6 +68,9 @@ jobs:
|
||||
- name: Dependency layering
|
||||
run: cargo run --locked -q -p xtask -- layers
|
||||
|
||||
- name: Session locks taken twice
|
||||
run: cargo run --locked -q -p xtask -- locks
|
||||
|
||||
- name: Tool list
|
||||
run: cargo run --locked -q -p xtask -- tools --check
|
||||
|
||||
|
||||
@@ -70,6 +70,12 @@ Layering is enforced by `cargo xtask layers`. Register new crates in `xtask/src/
|
||||
- a missing program, non-zero exit, bad output or timeout is a `ToolError` with a plain-English remediation,
|
||||
never a panic or a hang;
|
||||
- nothing slow ever runs on the UI thread: the UI sends work to background threads and polls results.
|
||||
11. **Never lock up on a lock.** Use `craft_anything_platform::sync::Mutex`, never `std::sync::Mutex` (clippy
|
||||
refuses it). It stops a real deadlock as soon as it forms, the waiter's work unwinding to its `catch_unwind` as
|
||||
an error; the watchdog reports anything held too long. Long jobs are fine: label the guard (`guard.label(id)`,
|
||||
as `Engine::call` does for tools) and they're waited for, not treated as stuck. A guard that isn't bound to a
|
||||
name lives until the statement ends, so read the session once: `cargo xtask locks` refuses a second
|
||||
`.session()` in reach of the first.
|
||||
|
||||
## Working rules (mandatory)
|
||||
|
||||
@@ -147,6 +153,7 @@ cargo fmt --all
|
||||
cargo test -p <crates you touched>
|
||||
cargo clippy -p <crates> --all-targets -- -D warnings
|
||||
cargo xtask layers
|
||||
cargo xtask locks
|
||||
cargo xtask secrets # no keys, tokens or local paths in what you commit
|
||||
cargo xtask wasm # if you touched L0–L2
|
||||
cargo xtask tools # if you added tools (commit docs/tools.md)
|
||||
|
||||
Generated
+4
@@ -1126,6 +1126,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-anything-ai",
|
||||
"craft-anything-engine",
|
||||
"craft-anything-platform",
|
||||
"craft-anything-setup",
|
||||
"craft-anything-ui-egui",
|
||||
"eframe",
|
||||
@@ -1313,6 +1314,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-anything-engine",
|
||||
"craft-anything-evidence",
|
||||
"craft-anything-platform",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -7959,6 +7961,8 @@ name = "xtask"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-anything-engine",
|
||||
"proc-macro2",
|
||||
"syn 2.0.119",
|
||||
"toml 0.9.12+spec-1.1.0",
|
||||
]
|
||||
|
||||
|
||||
@@ -193,6 +193,7 @@ fn usage(msg: &str) -> u8 {
|
||||
|
||||
/// The engine with settings loaded, the run's mode, and the project (if any) opened.
|
||||
pub fn make_engine(project: Option<&Path>, deep_for_run: Option<bool>) -> Result<Arc<Engine>, ToolError> {
|
||||
craft_anything_platform::sync::start_watchdog(craft_anything_engine::settings::config_dir().join("logs"));
|
||||
let mut session = Session::new(Settings::load());
|
||||
session.deep_for_run = deep_for_run;
|
||||
if let Some(dir) = project {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use crate::triage::{Look, MAX_BYTES, look};
|
||||
use craft_anything_evidence::Confidence;
|
||||
use egui::{Color32, RichText};
|
||||
#[allow(clippy::disallowed_types)] // The browser has one thread: nothing can wait here.
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
// A few tokens copied from the desktop app's Studio theme (crates/ui-egui/src/theme.rs).
|
||||
@@ -21,6 +22,7 @@ const BEST_GUESS: Color32 = Color32::from_rgb(0xe0, 0xa2, 0x3a);
|
||||
|
||||
/// A file handed over from outside a frame (the browser's file picker).
|
||||
/// A file (name and contents), or a plain message saying why it couldn't be read.
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub type Inbox = Arc<Mutex<Option<Result<(String, Vec<u8>), String>>>>;
|
||||
|
||||
pub struct Viewer {
|
||||
|
||||
@@ -12,6 +12,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
craft-anything-engine = { workspace = true }
|
||||
craft-anything-platform = { workspace = true }
|
||||
craft-anything-ai = { workspace = true }
|
||||
craft-anything-setup = { workspace = true }
|
||||
craft-anything-ui-egui = { workspace = true }
|
||||
|
||||
@@ -71,7 +71,7 @@ const LOG_MAX: u64 = 512 * 1024;
|
||||
/// panics, which would otherwise vanish with the window.
|
||||
fn log_to_file() {
|
||||
use std::io::Write as _;
|
||||
struct FileLog(std::sync::Mutex<Option<std::fs::File>>);
|
||||
struct FileLog(craft_anything_platform::sync::Mutex<Option<std::fs::File>>);
|
||||
impl log::Log for FileLog {
|
||||
fn enabled(&self, m: &log::Metadata<'_>) -> bool {
|
||||
m.level() <= log::Level::Warn || (m.level() <= log::Level::Info && m.target().starts_with("craft_anything"))
|
||||
@@ -94,7 +94,7 @@ fn log_to_file() {
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let fresh = std::fs::metadata(&path).is_ok_and(|m| m.len() > LOG_MAX);
|
||||
let file = std::fs::OpenOptions::new().create(true).append(!fresh).write(true).truncate(fresh).open(&path).ok();
|
||||
if log::set_logger(Box::leak(Box::new(FileLog(std::sync::Mutex::new(file))))).is_ok() {
|
||||
if log::set_logger(Box::leak(Box::new(FileLog(craft_anything_platform::sync::Mutex::new(file))))).is_ok() {
|
||||
log::set_max_level(log::LevelFilter::Info);
|
||||
}
|
||||
let default_hook = std::panic::take_hook();
|
||||
@@ -102,4 +102,5 @@ fn log_to_file() {
|
||||
log::error!("panic: {info}");
|
||||
default_hook(info);
|
||||
}));
|
||||
craft_anything_platform::sync::start_watchdog(craft_anything_engine::settings::config_dir().join("logs"));
|
||||
}
|
||||
@@ -2,3 +2,9 @@ allow-unwrap-in-tests = true
|
||||
allow-expect-in-tests = true
|
||||
allow-panic-in-tests = true
|
||||
allow-indexing-slicing-in-tests = true
|
||||
|
||||
# Locks must be able to notice a deadlock and get out of it; see crates/platform/src/sync.rs.
|
||||
disallowed-types = [
|
||||
{ path = "std::sync::Mutex", reason = "use craft_anything_platform::sync::Mutex, which can't hang" },
|
||||
{ path = "std::sync::MutexGuard", reason = "use craft_anything_platform::sync::MutexGuard" },
|
||||
]
|
||||
@@ -11,11 +11,12 @@ use crate::prompt;
|
||||
use crate::{AgentEvent, Conversation};
|
||||
use craft_anything_engine::{CallOptions, Caller, CancelToken, Consent, Engine, Registry, ToolOutput, Toolset};
|
||||
use craft_anything_evidence::{ErrorCode, ToolError};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Byte budget for one tool result sent to the model.
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
use super::{McpCommand, display_for};
|
||||
use crate::{AgentEvent, Conversation};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use craft_anything_platform::{Process, RunningProcess, StdinMode};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
@@ -17,7 +18,7 @@ use std::path::PathBuf;
|
||||
use std::process::ChildStdin;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Receiver, Sender};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Agents known to speak ACP, and the arguments that start them in that mode.
|
||||
@@ -458,7 +459,7 @@ impl AcpConversation {
|
||||
}
|
||||
}
|
||||
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, Option<Shared>> {
|
||||
fn lock(&self) -> craft_anything_platform::sync::MutexGuard<'_, Option<Shared>> {
|
||||
self.shared.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
|
||||
@@ -18,13 +18,14 @@ use crate::model::Stop;
|
||||
use crate::{AgentEvent, BackendConfig, Conversation};
|
||||
use craft_anything_evidence::ToolError;
|
||||
use craft_anything_platform::process::Process;
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
use std::ffi::OsStr;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::mpsc::{Receiver, Sender};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Name our MCP server is registered under.
|
||||
|
||||
@@ -27,12 +27,13 @@ use craft_anything_engines::game;
|
||||
use craft_anything_evidence::{ErrorCode, ToolError};
|
||||
pub use craft_anything_platform::serve::StaticServer;
|
||||
use craft_anything_platform::serve::serve_dir;
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::mpsc::{Receiver, Sender};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
mod brief;
|
||||
|
||||
@@ -2,10 +2,10 @@
|
||||
//! engine runs the tools. No real network.
|
||||
|
||||
use super::*;
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use serde_json::{Value, json};
|
||||
use std::io::{BufRead, BufReader, Read, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::Mutex;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct Seen {
|
||||
|
||||
@@ -37,8 +37,9 @@ pub use target::{Target, TargetKind};
|
||||
pub use tool::{Caller, CancelToken, Ctx, Effects, Progress, Requires, ToolOutput, ToolSpec, Toolset};
|
||||
|
||||
use craft_anything_evidence::{ErrorCode, ToolError};
|
||||
use craft_anything_platform::sync::{Mutex, MutexGuard};
|
||||
use serde_json::Value;
|
||||
use std::sync::{Mutex, MutexGuard, PoisonError};
|
||||
use std::sync::PoisonError;
|
||||
|
||||
/// Whether the caller has the user's consent for a tool that runs code or changes a device.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
@@ -140,11 +141,13 @@ impl Engine {
|
||||
}
|
||||
|
||||
/// Lock the session (recovering from a poisoned lock rather than cascading a panic).
|
||||
#[track_caller]
|
||||
pub fn session(&self) -> MutexGuard<'_, Session> {
|
||||
self.session.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// Try to lock without blocking (the UI uses this so a long tool never freezes a frame).
|
||||
#[track_caller]
|
||||
pub fn try_session(&self) -> Option<MutexGuard<'_, Session>> {
|
||||
match self.session.try_lock() {
|
||||
Ok(g) => Some(g),
|
||||
@@ -231,7 +234,19 @@ impl Engine {
|
||||
let spec = *self.registry.get(id).ok_or_else(|| {
|
||||
ToolError::invalid(format!("No tool named {id}")).with_remediation("List tools with `craft-anything-cli commands` or the command palette.")
|
||||
})?;
|
||||
let mut session = self.session();
|
||||
// Stop works while waiting for the session, and a holder that looks stuck is an error here,
|
||||
// not a wait forever. Tools are long jobs: the label says so (they have their own deadlines).
|
||||
let cancel = opts.cancel.clone();
|
||||
let mut session = match self.session.lock_unless(&|| cancel.is_cancelled()) {
|
||||
Ok(g) => g.unwrap_or_else(PoisonError::into_inner),
|
||||
Err(craft_anything_platform::sync::GaveUp::Stopped) => return Err(ToolError::cancelled()),
|
||||
Err(craft_anything_platform::sync::GaveUp::Stuck(why)) => {
|
||||
return Err(ToolError::new(ErrorCode::TimedOut, format!("{} couldn't start because another part of the app is stuck.", spec.title))
|
||||
.with_remediation("Restart the app; your work is kept.")
|
||||
.with_details(serde_json::json!({"lock": why})));
|
||||
}
|
||||
};
|
||||
session.label(spec.id);
|
||||
if session.clean_room() && !clean_room::allows(spec.id) {
|
||||
return Err(clean_room::refusal(spec.id));
|
||||
}
|
||||
|
||||
@@ -20,13 +20,14 @@ use crate::inbox::now_ms;
|
||||
use crate::versioned;
|
||||
use crate::workspace::{self, Workspace};
|
||||
use craft_anything_evidence::{ErrorCode, ToolError};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use craft_anything_platform::{Ended, Launch, Launched};
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Mutex, PoisonError};
|
||||
use std::sync::PoisonError;
|
||||
use std::time::Duration;
|
||||
|
||||
/// The folder in a project that holds its pins.
|
||||
|
||||
@@ -26,11 +26,12 @@ use crate::session::Session;
|
||||
use crate::tool::{Ctx, Effects, Requires, ToolOutput};
|
||||
use craft_anything_engines::mcp_client::{McpClient, RemoteTool, Server, ServerCommand, ServerUrl, content_text, url_allowed};
|
||||
use craft_anything_evidence::{Authority, Confidence, Environment, ErrorCode, EvidenceBuilder, Subject, ToolError};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
/// How long a server may take to start and answer the handshake or its tool list.
|
||||
|
||||
@@ -5,8 +5,9 @@
|
||||
use crate::inbox::now_ms;
|
||||
use crate::workspace::Workspace;
|
||||
use craft_anything_evidence::ToolError;
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::{Mutex, PoisonError};
|
||||
use std::sync::PoisonError;
|
||||
|
||||
/// Where they're kept in the project folder.
|
||||
pub const QUESTIONS_FILE: &str = "questions.json";
|
||||
|
||||
@@ -36,7 +36,8 @@ impl RuntimeTool {
|
||||
|
||||
/// A run-time group's id kept for the process, once per distinct id.
|
||||
pub fn custom_toolset(id: &str) -> Option<Toolset> {
|
||||
use std::sync::{Mutex, OnceLock, PoisonError};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use std::sync::{OnceLock, PoisonError};
|
||||
static SEEN: OnceLock<Mutex<Vec<&'static str>>> = OnceLock::new();
|
||||
if let Some(t) = Toolset::from_id(id) {
|
||||
return Some(t);
|
||||
|
||||
@@ -673,9 +673,10 @@ fn android_jeb_decompile(ctx: &mut Ctx<'_>, params: Value) -> Result<ToolOutput,
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{CallOptions, Caller, Engine};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use std::io::BufRead;
|
||||
use std::net::TcpListener;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn engine(url: Option<String>) -> Engine {
|
||||
let mut s = crate::testing::session();
|
||||
|
||||
@@ -374,7 +374,7 @@ mod tests {
|
||||
|
||||
/// Stands in for fal.ai: records each request's endpoint and answers with a few bytes.
|
||||
#[derive(Default)]
|
||||
struct StandIn(std::sync::Mutex<Vec<String>>);
|
||||
struct StandIn(craft_anything_platform::sync::Mutex<Vec<String>>);
|
||||
|
||||
impl craft_anything_engines::assets::AssetProvider for StandIn {
|
||||
fn id(&self) -> &'static str {
|
||||
|
||||
@@ -18,12 +18,13 @@ use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_p
|
||||
use craft_anything_engines::frida::{Attach, Frida, Remote, Selector};
|
||||
use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, Evidence, EvidenceBuilder, Limitation, Subject, ToolError, sha256_hex};
|
||||
use craft_anything_platform::process::{Cancel, Output};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
pub fn specs() -> Vec<ToolSpec> {
|
||||
|
||||
@@ -8,13 +8,14 @@ use super::native_common::spec;
|
||||
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
|
||||
use craft_anything_engines::gdb::GdbSession;
|
||||
use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, EvidenceBuilder, Limitation, Subject, ToolError, sha256_hex};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
pub fn specs() -> Vec<ToolSpec> {
|
||||
|
||||
@@ -11,11 +11,12 @@ use crate::project::write_atomic;
|
||||
use crate::session::Session;
|
||||
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, empty_schema, parse_params, schema_of};
|
||||
use craft_anything_evidence::{Confidence, ErrorCode, ToolError};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
|
||||
pub fn specs() -> Vec<ToolSpec> {
|
||||
let w = Effects::MUTATES_SESSION.union(Effects::WRITES_FILESYSTEM);
|
||||
|
||||
@@ -7,12 +7,13 @@ use crate::inbox::{Answer, Inbox};
|
||||
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
|
||||
use crate::workspace::Workspace;
|
||||
use craft_anything_evidence::{ErrorCode, ToolError};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
|
||||
/// Most feed items one call returns.
|
||||
const MAX_ITEMS: usize = 200;
|
||||
|
||||
@@ -7,13 +7,14 @@ use super::native_common::spec;
|
||||
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
|
||||
use craft_anything_engines::rizin::{RizinSession, Target};
|
||||
use craft_anything_evidence::{Authority, Confidence, Coverage, Environment, ErrorCode, EvidenceBuilder, Limitation, Subject, ToolError, sha256_hex};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::Duration;
|
||||
|
||||
pub fn specs() -> Vec<ToolSpec> {
|
||||
|
||||
@@ -13,6 +13,7 @@ use crate::versioned::{Step, upgrade};
|
||||
use craft_anything_analysis::standin::{self, Answer, BodyLeftOut, BuildOptions, NearbyRoute, NoMatchReason, STANDIN_VERSION, StandIn, Turns};
|
||||
use craft_anything_evidence::{Authority, Confidence, Coverage, ErrorCode, EvidenceBuilder, Limitation, Location, Subject, ToolError, sha256_hex};
|
||||
use craft_anything_platform::standin_server::{self, Answerer, Limits, LoopbackServer, Request, Response};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
@@ -20,7 +21,7 @@ use std::collections::BTreeMap;
|
||||
#[cfg(test)]
|
||||
use std::path::Path;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
|
||||
/// What `web_standin_build` writes in the folder it's given.
|
||||
pub const STANDIN_FILE: &str = "standin.json";
|
||||
|
||||
@@ -1230,7 +1230,7 @@ mod tests {
|
||||
|
||||
/// The last lines a launched browser wrote to stderr, for a failure's message.
|
||||
#[derive(Clone, Default)]
|
||||
struct StderrTail(std::sync::Arc<std::sync::Mutex<std::collections::VecDeque<String>>>);
|
||||
struct StderrTail(std::sync::Arc<craft_anything_platform::sync::Mutex<std::collections::VecDeque<String>>>);
|
||||
|
||||
impl StderrTail {
|
||||
fn push(&self, line: &str) {
|
||||
|
||||
@@ -14,11 +14,12 @@ use super::web_common::{self, Browser, LaunchOptions};
|
||||
use crate::tool::{Ctx, Effects, Requires, ToolOutput, ToolSpec, Toolset, parse_params, schema_of};
|
||||
use craft_anything_engines::cdp::{self, CdpClient, CdpEvent, Deadline, Endpoint, PAGE_MAX_MESSAGE};
|
||||
use craft_anything_evidence::{Authority, Confidence, Coverage, ErrorCode, EvidenceBuilder, Limitation, Location, Subject, ToolError, sha256_hex};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
pub fn specs() -> Vec<ToolSpec> {
|
||||
|
||||
@@ -179,9 +179,10 @@ fn base64_decode(s: &str) -> Result<Vec<u8>, ToolError> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::assets::Catalogue;
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::TcpListener;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// A tiny local server that answers each request in turn and records what it was sent.
|
||||
fn serve(answers: Vec<(u16, String)>) -> (String, Arc<Mutex<Vec<String>>>) {
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
//! structured records. The engine's GDB tools keep sessions by id and turn results into evidence.
|
||||
|
||||
use craft_anything_evidence::{ErrorCode, ToolError};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use craft_anything_platform::{Process, RunningProcess, StdinMode};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
@@ -11,7 +12,7 @@ use std::collections::VecDeque;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ChildStdin;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Unread MI lines kept per session; older ones are dropped.
|
||||
|
||||
@@ -24,14 +24,15 @@ pub use client::Client;
|
||||
pub use protocol::{GhidraError, Request, Response, ct_eq};
|
||||
|
||||
use craft_anything_platform::process::{Process, RunningProcess, StdinMode};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use craft_anything_platform::write_atomic_private;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use std::ffi::OsString;
|
||||
use std::net::{Ipv4Addr, SocketAddr};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::PoisonError;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Mutex, PoisonError};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
/// The Java post-script we ship and run inside Ghidra (our own code).
|
||||
@@ -571,7 +572,7 @@ impl Session {
|
||||
})
|
||||
}
|
||||
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, Client> {
|
||||
fn lock(&self) -> craft_anything_platform::sync::MutexGuard<'_, Client> {
|
||||
self.client.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
//! by id and turn results into evidence.
|
||||
|
||||
use craft_anything_evidence::{ErrorCode, ToolError};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use craft_anything_platform::{Process, RunningProcess, StdinMode};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Value, json};
|
||||
@@ -12,7 +13,7 @@ use std::collections::VecDeque;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::ChildStdin;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Unread lines kept per session; older ones are dropped.
|
||||
|
||||
@@ -12,6 +12,7 @@ workspace = true
|
||||
[dependencies]
|
||||
craft-anything-evidence = { workspace = true }
|
||||
craft-anything-engine = { workspace = true }
|
||||
craft-anything-platform = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
log = { workspace = true }
|
||||
@@ -20,11 +20,12 @@ pub mod prompts;
|
||||
use craft_anything_engine::tools::investigation_tools;
|
||||
use craft_anything_engine::{CallOptions, Caller, CancelToken, Consent, Engine, Progress, ToolOutput, Toolset};
|
||||
use craft_anything_evidence::{ErrorCode, ToolError, is_evidence_id};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::io::{BufRead, Write};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
|
||||
@@ -30,10 +30,11 @@ use std::ffi::OsString;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
// Only the Unix capture (and its tests) use these.
|
||||
use crate::sync::Mutex;
|
||||
#[cfg(unix)]
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
#[cfg(unix)]
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
#[cfg(unix)]
|
||||
use std::time::{Instant, SystemTime};
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ pub mod fs;
|
||||
pub mod process;
|
||||
pub mod serve;
|
||||
pub mod standin_server;
|
||||
pub mod sync;
|
||||
pub mod which;
|
||||
pub mod window;
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
//! Never call [`Process::run`] on a UI thread: it blocks until the child finishes or a deadline hits.
|
||||
//! Use [`Process::spawn`] and poll [`RunningProcess::try_result`] instead.
|
||||
|
||||
use crate::sync::Mutex;
|
||||
use craft_anything_evidence::{ErrorCode, ToolError};
|
||||
use process_wrap::std::{ChildWrapper, CommandWrap};
|
||||
use std::ffi::{OsStr, OsString};
|
||||
@@ -28,7 +29,7 @@ use std::path::PathBuf;
|
||||
use std::process::{ChildStdin, Stdio};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::mpsc::{Receiver, RecvTimeoutError, TryRecvError};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
@@ -995,6 +996,12 @@ pub fn system_opener() -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// Start this program again, for a restart: the caller then closes itself.
|
||||
pub fn start_another_copy() -> Result<u32, ToolError> {
|
||||
let exe = std::env::current_exe().map_err(|e| ToolError::new(ErrorCode::FileError, format!("Couldn't find this program to restart it: {e}")))?;
|
||||
Launch::new(exe).start()
|
||||
}
|
||||
|
||||
/// Show a file or folder with the system's default handler (a folder opens in the file manager).
|
||||
/// Fire-and-forget, like [`Launch`].
|
||||
pub fn open_path(path: impl AsRef<std::path::Path>) -> Result<(), ToolError> {
|
||||
|
||||
@@ -0,0 +1,633 @@
|
||||
//! Locks that can't hang the app. Use [`Mutex`] here instead of `std::sync::Mutex` (clippy refuses
|
||||
//! the standard one): it has the same API, and it knows who holds what.
|
||||
//!
|
||||
//! - **A real deadlock is broken at once.** A thread that waits on a lock it holds itself, or on a
|
||||
//! cycle of threads waiting on each other, can never get it. The waiter that finds this stops
|
||||
//! waiting and unwinds (a caught, reported error, never a crash): its locks are released, so
|
||||
//! everyone else carries on. Every thread's work runs inside a `catch_unwind` that turns this
|
||||
//! into an error the person sees.
|
||||
//! - **Long work is left alone.** A holder labelled as a job ([`MutexGuard::label`], such as a tool,
|
||||
//! which has its own deadlines and Stop) may take hours. Waiting for it is reported, never cut off.
|
||||
//! - **A holder that isn't a job and never lets go** is reported after [`Limits::stuck_after`], and
|
||||
//! anyone still waiting gives up after [`Limits::give_up_after`] rather than wait forever.
|
||||
//! - **[`start_watchdog`] looks every few seconds** and writes what it finds to a log.
|
||||
//! [`problems`] is what the app shows.
|
||||
|
||||
#![allow(clippy::disallowed_types)]
|
||||
|
||||
use std::fmt;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::panic::Location;
|
||||
use std::sync::{LockResult, PoisonError, TryLockError, TryLockResult};
|
||||
use std::time::Duration;
|
||||
|
||||
/// When waiting on a lock stops being normal.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct Limits {
|
||||
/// How long a holder that isn't a labelled job may keep others waiting before it's reported.
|
||||
pub stuck_after: Duration,
|
||||
/// How long anyone waits on such a holder before giving up.
|
||||
pub give_up_after: Duration,
|
||||
/// How long waiting on a labelled job goes on before it's worth saying (it isn't cut off).
|
||||
pub slow_job_after: Duration,
|
||||
}
|
||||
|
||||
impl Default for Limits {
|
||||
fn default() -> Self {
|
||||
Limits { stuck_after: Duration::from_secs(120), give_up_after: Duration::from_secs(900), slow_job_after: Duration::from_secs(1800) }
|
||||
}
|
||||
}
|
||||
|
||||
/// How serious a [`Problem`] is.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ProblemKind {
|
||||
/// A deadlock that was broken: the thread that found it stopped, and the rest carried on.
|
||||
Broken,
|
||||
/// A deadlock that couldn't be broken from here: restarting the app clears it.
|
||||
Deadlock,
|
||||
/// Something holds a lock far longer than it should, and others are waiting.
|
||||
Stuck,
|
||||
/// Work waits on a long job. Normal for big jobs; it's said so nobody wonders.
|
||||
SlowJob,
|
||||
}
|
||||
|
||||
/// Something the watchdog or a waiting thread found.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Problem {
|
||||
pub kind: ProblemKind,
|
||||
pub summary: String,
|
||||
pub log: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
/// Why [`Mutex::lock_unless`] came back without the lock.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum GaveUp {
|
||||
Stopped,
|
||||
/// The holder looks stuck; says who holds it.
|
||||
Stuck(String),
|
||||
}
|
||||
|
||||
/// A mutual-exclusion lock whose waits are watched (see the module docs).
|
||||
pub struct Mutex<T: ?Sized> {
|
||||
inner: std::sync::Mutex<T>,
|
||||
}
|
||||
|
||||
/// Holds a [`Mutex`] until dropped.
|
||||
pub struct MutexGuard<'a, T: ?Sized + 'a> {
|
||||
inner: std::sync::MutexGuard<'a, T>,
|
||||
id: usize,
|
||||
}
|
||||
|
||||
impl<T> Mutex<T> {
|
||||
pub const fn new(value: T) -> Self {
|
||||
Mutex { inner: std::sync::Mutex::new(value) }
|
||||
}
|
||||
|
||||
pub fn into_inner(self) -> LockResult<T> {
|
||||
self.inner.into_inner()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized> Mutex<T> {
|
||||
fn id(&self) -> usize {
|
||||
(&raw const self.inner).cast::<()>() as usize
|
||||
}
|
||||
|
||||
fn wrap<'a>(id: usize, g: std::sync::MutexGuard<'a, T>, at: &'static Location<'static>) -> MutexGuard<'a, T> {
|
||||
tracker::held(id, at);
|
||||
MutexGuard { inner: g, id }
|
||||
}
|
||||
|
||||
/// Take the lock, waiting while it's busy. Comes back with the lock unless waiting could never
|
||||
/// end: then this thread's work unwinds, releasing what it holds, and its `catch_unwind`
|
||||
/// reports the error.
|
||||
#[track_caller]
|
||||
pub fn lock(&self) -> LockResult<MutexGuard<'_, T>> {
|
||||
let at = Location::caller();
|
||||
match self.wait(at, &|| false, Limits::default(), false) {
|
||||
Ok(r) => r,
|
||||
Err(GaveUp::Stuck(why)) => tracker::unwind(why),
|
||||
// `stopped` above never says stop.
|
||||
Err(GaveUp::Stopped) => tracker::unwind(format!("A wait at {at} stopped for no reason.")),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`Mutex::lock`] for work that can report an error: gives up when `stop` says so, or as soon
|
||||
/// as the holder looks stuck, instead of waiting on. A deadlock still unwinds.
|
||||
#[track_caller]
|
||||
pub fn lock_unless(&self, stop: &dyn Fn() -> bool) -> Result<LockResult<MutexGuard<'_, T>>, GaveUp> {
|
||||
self.wait(Location::caller(), stop, Limits::default(), true)
|
||||
}
|
||||
|
||||
fn wait(&self, at: &'static Location<'static>, stop: &dyn Fn() -> bool, limits: Limits, early: bool) -> Result<LockResult<MutexGuard<'_, T>>, GaveUp> {
|
||||
let id = self.id();
|
||||
let mut pause = Duration::from_millis(1);
|
||||
let mut waiting = false;
|
||||
loop {
|
||||
match self.inner.try_lock() {
|
||||
Ok(g) => {
|
||||
if waiting {
|
||||
tracker::end_wait();
|
||||
}
|
||||
return Ok(Ok(Self::wrap(id, g, at)));
|
||||
}
|
||||
Err(TryLockError::Poisoned(p)) => {
|
||||
if waiting {
|
||||
tracker::end_wait();
|
||||
}
|
||||
return Ok(Err(PoisonError::new(Self::wrap(id, p.into_inner(), at))));
|
||||
}
|
||||
Err(TryLockError::WouldBlock) => {}
|
||||
}
|
||||
if !waiting {
|
||||
tracker::begin_wait(id, at);
|
||||
waiting = true;
|
||||
}
|
||||
// Under one lock, so two threads in a cycle can't both decide to break it.
|
||||
if let Some(why) = tracker::must_give_up(id, limits, early) {
|
||||
match why {
|
||||
tracker::Verdict::Deadlock(text) => tracker::unwind(text),
|
||||
tracker::Verdict::Stuck(text) => {
|
||||
tracker::end_wait();
|
||||
return Err(GaveUp::Stuck(text));
|
||||
}
|
||||
}
|
||||
}
|
||||
if stop() {
|
||||
tracker::end_wait();
|
||||
return Err(GaveUp::Stopped);
|
||||
}
|
||||
tracker::pause(pause);
|
||||
pause = (pause * 2).min(Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
pub fn try_lock(&self) -> TryLockResult<MutexGuard<'_, T>> {
|
||||
let at = Location::caller();
|
||||
let id = self.id();
|
||||
match self.inner.try_lock() {
|
||||
Ok(g) => Ok(Self::wrap(id, g, at)),
|
||||
Err(TryLockError::Poisoned(p)) => Err(TryLockError::Poisoned(PoisonError::new(Self::wrap(id, p.into_inner(), at)))),
|
||||
Err(TryLockError::WouldBlock) => Err(TryLockError::WouldBlock),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_mut(&mut self) -> LockResult<&mut T> {
|
||||
self.inner.get_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Default> Default for Mutex<T> {
|
||||
fn default() -> Self {
|
||||
Mutex::new(T::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> From<T> for Mutex<T> {
|
||||
fn from(value: T) -> Self {
|
||||
Mutex::new(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized> fmt::Debug for Mutex<T> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str("Mutex")
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized> MutexGuard<'_, T> {
|
||||
/// Say this hold is a job that may take a long time, by name. It's then never treated as stuck.
|
||||
pub fn label(&self, job: &'static str) {
|
||||
tracker::label(self.id, job);
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized> Deref for MutexGuard<'_, T> {
|
||||
type Target = T;
|
||||
fn deref(&self) -> &T {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized> DerefMut for MutexGuard<'_, T> {
|
||||
fn deref_mut(&mut self) -> &mut T {
|
||||
&mut self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized + fmt::Debug> fmt::Debug for MutexGuard<'_, T> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
(**self).fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized + fmt::Display> fmt::Display for MutexGuard<'_, T> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
(**self).fmt(f)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: ?Sized> Drop for MutexGuard<'_, T> {
|
||||
fn drop(&mut self) {
|
||||
tracker::released(self.id);
|
||||
}
|
||||
}
|
||||
|
||||
/// What's wrong right now and what went wrong recently, for the app to show.
|
||||
pub fn problems() -> Vec<Problem> {
|
||||
tracker::problems()
|
||||
}
|
||||
|
||||
/// Add a problem found some other way, so it's logged and shown like the rest.
|
||||
pub fn report(kind: ProblemKind, summary: impl Into<String>) {
|
||||
tracker::report(kind, summary.into());
|
||||
}
|
||||
|
||||
/// Forget a problem the person has dismissed.
|
||||
pub fn dismiss(summary: &str) {
|
||||
tracker::dismiss(summary);
|
||||
}
|
||||
|
||||
/// Check every few seconds for locks in trouble, writing each new problem to a log in `log_dir`.
|
||||
/// Starts once per process; apps call it, tests don't.
|
||||
pub fn start_watchdog(log_dir: std::path::PathBuf) {
|
||||
tracker::start_watchdog(log_dir);
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
mod tracker {
|
||||
//! One thread and no clock: nothing to watch, and nothing that could wait.
|
||||
use super::*;
|
||||
pub enum Verdict {
|
||||
Deadlock(String),
|
||||
Stuck(String),
|
||||
}
|
||||
pub fn held(_: usize, _: &'static Location<'static>) {}
|
||||
pub fn released(_: usize) {}
|
||||
pub fn label(_: usize, _: &'static str) {}
|
||||
pub fn begin_wait(_: usize, _: &'static Location<'static>) {}
|
||||
pub fn end_wait() {}
|
||||
pub fn pause(_: Duration) {}
|
||||
pub fn must_give_up(id: usize, _: Limits, _: bool) -> Option<Verdict> {
|
||||
Some(Verdict::Deadlock(format!("Lock {id} is busy on the only thread.")))
|
||||
}
|
||||
pub fn unwind(text: String) -> ! {
|
||||
std::panic::resume_unwind(Box::new(text))
|
||||
}
|
||||
pub fn problems() -> Vec<Problem> {
|
||||
Vec::new()
|
||||
}
|
||||
pub fn dismiss(_: &str) {}
|
||||
pub fn report(_: ProblemKind, _: String) {}
|
||||
pub fn start_watchdog(_: std::path::PathBuf) {}
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod tracker {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
use std::thread::ThreadId;
|
||||
use std::time::Instant;
|
||||
|
||||
/// How often the watchdog looks.
|
||||
const WATCH_EVERY: Duration = Duration::from_secs(5);
|
||||
/// Problems remembered for the app.
|
||||
const KEEP_PROBLEMS: usize = 8;
|
||||
|
||||
pub enum Verdict {
|
||||
Deadlock(String),
|
||||
Stuck(String),
|
||||
}
|
||||
|
||||
struct Hold {
|
||||
thread: ThreadId,
|
||||
thread_name: String,
|
||||
at: &'static Location<'static>,
|
||||
since: Instant,
|
||||
job: Option<&'static str>,
|
||||
}
|
||||
|
||||
struct Wait {
|
||||
lock: usize,
|
||||
thread_name: String,
|
||||
at: &'static Location<'static>,
|
||||
since: Instant,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Registry {
|
||||
holds: HashMap<usize, Hold>,
|
||||
waits: HashMap<ThreadId, Wait>,
|
||||
problems: Vec<Problem>,
|
||||
dismissed: Vec<String>,
|
||||
}
|
||||
|
||||
/// Held only for short bookkeeping, never while taking another lock, so it can't deadlock.
|
||||
fn registry() -> std::sync::MutexGuard<'static, Registry> {
|
||||
static REGISTRY: OnceLock<std::sync::Mutex<Registry>> = OnceLock::new();
|
||||
REGISTRY.get_or_init(Default::default).lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn thread_name() -> String {
|
||||
std::thread::current().name().unwrap_or("unnamed").to_string()
|
||||
}
|
||||
|
||||
pub fn held(id: usize, at: &'static Location<'static>) {
|
||||
let hold = Hold { thread: std::thread::current().id(), thread_name: thread_name(), at, since: Instant::now(), job: None };
|
||||
registry().holds.insert(id, hold);
|
||||
}
|
||||
|
||||
pub fn released(id: usize) {
|
||||
registry().holds.remove(&id);
|
||||
}
|
||||
|
||||
pub fn label(id: usize, job: &'static str) {
|
||||
if let Some(h) = registry().holds.get_mut(&id) {
|
||||
h.job = Some(job);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn begin_wait(lock: usize, at: &'static Location<'static>) {
|
||||
let me = std::thread::current().id();
|
||||
registry().waits.insert(me, Wait { lock, thread_name: thread_name(), at, since: Instant::now() });
|
||||
}
|
||||
|
||||
pub fn end_wait() {
|
||||
let me = std::thread::current().id();
|
||||
registry().waits.remove(&me);
|
||||
}
|
||||
|
||||
pub fn pause(d: Duration) {
|
||||
std::thread::sleep(d);
|
||||
}
|
||||
|
||||
/// The chain of threads waiting on each other that leads from `start` back to it, if there is one.
|
||||
fn cycle(r: &Registry, start: ThreadId) -> Option<String> {
|
||||
let mut parts = Vec::new();
|
||||
let mut thread = start;
|
||||
// A chain can't be longer than the number of waiting threads.
|
||||
for _ in 0..=r.waits.len() {
|
||||
let wait = r.waits.get(&thread)?;
|
||||
let hold = r.holds.get(&wait.lock)?;
|
||||
parts.push(format!("thread '{}' at {} waits on a lock taken at {} by thread '{}'", wait.thread_name, wait.at, hold.at, hold.thread_name));
|
||||
if hold.thread == start {
|
||||
return Some(parts.join("; "));
|
||||
}
|
||||
thread = hold.thread;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn remember(r: &mut Registry, kind: ProblemKind, summary: String) {
|
||||
if r.dismissed.contains(&summary) || r.problems.iter().any(|p| p.summary == summary) {
|
||||
return;
|
||||
}
|
||||
log::error!("{summary}");
|
||||
r.problems.push(Problem { kind, summary, log: None });
|
||||
if r.problems.len() > KEEP_PROBLEMS {
|
||||
r.problems.remove(0);
|
||||
}
|
||||
}
|
||||
|
||||
/// Should the waiting thread stop? Removes its wait when it should, in the same step as the
|
||||
/// decision, so the next thread in a cycle no longer sees one and carries on.
|
||||
pub fn must_give_up(lock: usize, limits: Limits, early: bool) -> Option<Verdict> {
|
||||
let me = std::thread::current().id();
|
||||
let mut r = registry();
|
||||
if let Some(chain) = cycle(&r, me) {
|
||||
// Unwinding during an unwind would abort the process; left waiting, it's reported
|
||||
// by the watchdog as a deadlock to restart from.
|
||||
if std::thread::panicking() {
|
||||
return None;
|
||||
}
|
||||
r.waits.remove(&me);
|
||||
let text = format!("A deadlock was found and broken: {chain}.");
|
||||
remember(&mut r, ProblemKind::Broken, text.clone());
|
||||
return Some(Verdict::Deadlock(text));
|
||||
}
|
||||
let waited = r.waits.get(&me)?.since.elapsed();
|
||||
let hold = r.holds.get(&lock)?;
|
||||
if hold.job.is_some() {
|
||||
return None;
|
||||
}
|
||||
let limit = if early { limits.stuck_after } else { limits.give_up_after };
|
||||
if waited < limit || hold.since.elapsed() < limit {
|
||||
return None;
|
||||
}
|
||||
let text = format!(
|
||||
"A lock taken at {} by thread '{}' hasn't been released for {} s, so a wait was given up.",
|
||||
hold.at,
|
||||
hold.thread_name,
|
||||
hold.since.elapsed().as_secs()
|
||||
);
|
||||
if std::thread::panicking() {
|
||||
return None;
|
||||
}
|
||||
r.waits.remove(&me);
|
||||
remember(&mut r, ProblemKind::Stuck, text.clone());
|
||||
Some(if early { Verdict::Stuck(text) } else { Verdict::Deadlock(text) })
|
||||
}
|
||||
|
||||
pub fn unwind(text: String) -> ! {
|
||||
end_wait();
|
||||
// Not `panic!`: this is a caught, expected way out, and the panic hook isn't what reports it.
|
||||
std::panic::resume_unwind(Box::new(text))
|
||||
}
|
||||
|
||||
pub fn problems() -> Vec<Problem> {
|
||||
registry().problems.clone()
|
||||
}
|
||||
|
||||
pub fn report(kind: ProblemKind, summary: String) {
|
||||
remember(&mut registry(), kind, summary);
|
||||
}
|
||||
|
||||
pub fn dismiss(summary: &str) {
|
||||
let mut r = registry();
|
||||
r.problems.retain(|p| p.summary != summary);
|
||||
r.dismissed.push(summary.to_string());
|
||||
}
|
||||
|
||||
/// What's wrong right now, whole-app: the watchdog's look at everything.
|
||||
pub(super) fn scan(limits: Limits) {
|
||||
let mut r = registry();
|
||||
let mut found = Vec::new();
|
||||
for (thread, wait) in &r.waits {
|
||||
if let Some(chain) = cycle(&r, *thread) {
|
||||
found.push((ProblemKind::Deadlock, format!("A deadlock can't be broken from here, so restart the app: {chain}.")));
|
||||
continue;
|
||||
}
|
||||
let Some(hold) = r.holds.get(&wait.lock) else { continue };
|
||||
let waited = wait.since.elapsed();
|
||||
let (kind, limit, what) = match hold.job {
|
||||
Some(job) => (ProblemKind::SlowJob, limits.slow_job_after, format!("'{job}' is still running")),
|
||||
None => (ProblemKind::Stuck, limits.stuck_after, format!("a lock taken at {} by thread '{}' hasn't been released", hold.at, hold.thread_name)),
|
||||
};
|
||||
if waited >= limit && hold.since.elapsed() >= limit {
|
||||
found.push((kind, format!("Thread '{}' at {} is waiting because {what}.", wait.thread_name, wait.at)));
|
||||
}
|
||||
}
|
||||
for (kind, text) in found {
|
||||
remember(&mut r, kind, text);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_watchdog(log_dir: std::path::PathBuf) {
|
||||
static STARTED: OnceLock<()> = OnceLock::new();
|
||||
if STARTED.set(()).is_err() {
|
||||
return;
|
||||
}
|
||||
let spawned = std::thread::Builder::new().name("lock-watch".into()).spawn(move || {
|
||||
loop {
|
||||
std::thread::sleep(WATCH_EVERY);
|
||||
scan(Limits::default());
|
||||
write_logs(&log_dir);
|
||||
}
|
||||
});
|
||||
if let Err(e) = spawned {
|
||||
log::warn!("couldn't start the lock watchdog: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Give each new problem its log file. Files are written outside the registry's lock.
|
||||
pub(super) fn write_logs(dir: &std::path::Path) {
|
||||
let new: Vec<(usize, String)> = registry().problems.iter().enumerate().filter(|(_, p)| p.log.is_none()).map(|(i, p)| (i, p.summary.clone())).collect();
|
||||
for (n, summary) in new {
|
||||
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);
|
||||
let path = dir.join(format!("lock-{stamp}-{n}.log"));
|
||||
let text = format!("Craft Anything lock report\n\n{summary}\n\nEverything holding or waiting on a lock now:\n{}\n", snapshot());
|
||||
let saved = std::fs::create_dir_all(dir)
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|()| crate::write_atomic_private(&path, text.as_bytes()).map_err(|e| e.message));
|
||||
match saved {
|
||||
Ok(()) => {
|
||||
if let Some(p) = registry().problems.iter_mut().find(|p| p.summary == summary) {
|
||||
p.log = Some(path);
|
||||
}
|
||||
}
|
||||
Err(e) => log::warn!("couldn't save a lock report: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot() -> String {
|
||||
let r = registry();
|
||||
let holds = r.holds.values().map(|h| {
|
||||
let job = h.job.map(|j| format!(" for '{j}'")).unwrap_or_default();
|
||||
format!(" held by thread '{}' at {} for {} s{job}", h.thread_name, h.at, h.since.elapsed().as_secs())
|
||||
});
|
||||
let waits = r.waits.values().map(|w| format!(" waited on by thread '{}' at {} for {} s", w.thread_name, w.at, w.since.elapsed().as_secs()));
|
||||
holds.chain(waits).collect::<Vec<_>>().join("\n")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(target_arch = "wasm32")))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Arc;
|
||||
use std::sync::mpsc::channel;
|
||||
|
||||
const LIMIT: Duration = Duration::from_secs(20);
|
||||
|
||||
fn text(p: Box<dyn std::any::Any + Send>) -> String {
|
||||
p.downcast_ref::<String>().cloned().unwrap_or_default()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_thread_that_locks_twice_stops_instead_of_hanging() {
|
||||
let m = Arc::new(Mutex::new(1));
|
||||
let m2 = m.clone();
|
||||
let (done, finished) = channel();
|
||||
std::thread::spawn(move || {
|
||||
// Exactly the shape of the bug: both guards live until the statement ends.
|
||||
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| *m2.lock().unwrap() + *m2.lock().unwrap()));
|
||||
let _ = done.send(r.map_err(text));
|
||||
});
|
||||
let why = finished.recv_timeout(LIMIT).expect("it hung").unwrap_err();
|
||||
assert!(why.contains("deadlock") && why.contains("sync.rs"), "says where: {why}");
|
||||
assert!(problems().iter().any(|p| p.kind == ProblemKind::Broken && p.summary == why));
|
||||
// The lock is free again, poisoned by the unwind and still usable.
|
||||
assert_eq!(*m.lock().unwrap_or_else(PoisonError::into_inner), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_threads_in_a_cycle_are_untangled_and_one_finishes() {
|
||||
let (a, b) = (Arc::new(Mutex::new(0)), Arc::new(Mutex::new(0)));
|
||||
let (done, finished) = channel();
|
||||
for (first, second) in [(a.clone(), b.clone()), (b, a)] {
|
||||
let done = done.clone();
|
||||
std::thread::spawn(move || {
|
||||
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||
let _one = first.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
let _two = second.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
}));
|
||||
let _ = done.send(r.is_ok());
|
||||
});
|
||||
}
|
||||
let results = [finished.recv_timeout(LIMIT).expect("it hung"), finished.recv_timeout(LIMIT).expect("it hung")];
|
||||
assert_eq!(results.iter().filter(|ok| **ok).count(), 1, "one stops, so the other finishes: {results:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_job_is_waited_for_however_short_the_limits() {
|
||||
let m = Arc::new(Mutex::new(0));
|
||||
let g = m.lock().unwrap();
|
||||
g.label("big-job");
|
||||
let (m2, (done, finished)) = (m.clone(), channel());
|
||||
std::thread::spawn(move || {
|
||||
let limits = Limits { stuck_after: Duration::ZERO, give_up_after: Duration::ZERO, slow_job_after: Duration::ZERO };
|
||||
let got = m2.wait(Location::caller(), &|| false, limits, true).map(|_| ());
|
||||
let _ = done.send(got);
|
||||
});
|
||||
assert!(finished.recv_timeout(Duration::from_millis(300)).is_err(), "still waiting on the job");
|
||||
drop(g);
|
||||
assert_eq!(finished.recv_timeout(LIMIT).expect("it hung"), Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_holder_that_never_lets_go_is_given_up_on_and_stop_works_while_waiting() {
|
||||
let m = Arc::new(Mutex::new(0));
|
||||
let g = m.lock().unwrap();
|
||||
let (m2, (done, finished)) = (m.clone(), channel());
|
||||
std::thread::spawn(move || {
|
||||
let limits = Limits { stuck_after: Duration::from_millis(50), ..Limits::default() };
|
||||
let _ = done.send(m2.wait(Location::caller(), &|| false, limits, true).map(|_| ()));
|
||||
});
|
||||
match finished.recv_timeout(LIMIT).expect("it hung") {
|
||||
Err(GaveUp::Stuck(why)) => assert!(why.contains("hasn't been released"), "{why}"),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
let stopped = m.clone();
|
||||
let (done, finished) = channel();
|
||||
std::thread::spawn(move || {
|
||||
let _ = done.send(stopped.lock_unless(&|| true).map(|_| ()));
|
||||
});
|
||||
assert_eq!(finished.recv_timeout(LIMIT).expect("it hung"), Err(GaveUp::Stopped));
|
||||
drop(g);
|
||||
assert!(m.try_lock().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_watchdog_reports_a_stuck_holder_in_a_log_and_can_forget_it() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let m = Arc::new(Mutex::new(0));
|
||||
let g = m.lock().unwrap();
|
||||
let (m2, (done, finished)) = (m.clone(), channel());
|
||||
std::thread::spawn(move || {
|
||||
let _ = done.send(m2.lock_unless(&|| false).map(|_| ()));
|
||||
});
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
tracker::scan(Limits { stuck_after: Duration::ZERO, ..Limits::default() });
|
||||
tracker::write_logs(dir.path());
|
||||
let p = problems().into_iter().find(|p| p.kind == ProblemKind::Stuck && p.summary.contains("sync.rs")).expect("reported");
|
||||
let log = std::fs::read_to_string(p.log.clone().expect("has a log")).unwrap();
|
||||
assert!(log.contains(&p.summary) && log.contains("held by thread"), "{log}");
|
||||
dismiss(&p.summary);
|
||||
assert!(!problems().iter().any(|q| q.summary == p.summary));
|
||||
drop(g);
|
||||
assert_eq!(finished.recv_timeout(LIMIT).expect("it hung"), Ok(()));
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,9 @@
|
||||
//! a worker thread; the UI polls finished results once per frame and never waits.
|
||||
|
||||
use craft_anything_engine::{CancelToken, Progress};
|
||||
use craft_anything_platform::sync::Mutex;
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
use std::sync::{Arc, PoisonError};
|
||||
use std::time::Instant;
|
||||
|
||||
/// What a worker hands back. `T` is the app's result type.
|
||||
|
||||
@@ -836,6 +836,37 @@ impl CraftApp {
|
||||
}
|
||||
}
|
||||
|
||||
/// Says when a lock got stuck (crates/platform/src/sync.rs), with a way out.
|
||||
fn stuck_banner(&mut self, ui: &mut egui::Ui, ctx: &egui::Context) {
|
||||
use craft_anything_platform::sync::{self, ProblemKind};
|
||||
let Some(problem) = sync::problems().into_iter().find(|p| p.kind != ProblemKind::SlowJob) else { return };
|
||||
let t = theme::Tokens::get(ctx);
|
||||
let text = match problem.kind {
|
||||
ProblemKind::Broken => "Part of the app got stuck, so it was stopped and the rest carried on. Your work is kept.",
|
||||
_ => "Part of the app is stuck. Restarting clears it, and your work is kept.",
|
||||
};
|
||||
egui::Panel::top("stuck-banner").frame(egui::Frame::new().fill(t.card).inner_margin(egui::Margin::symmetric(14, 8))).show(ui, |ui| {
|
||||
ui.horizontal_wrapped(|ui| {
|
||||
icons::show(ui, "triangle-alert", 16.0, t.danger);
|
||||
ui.label(egui::RichText::new(i18n::tr(text)).color(t.text)).on_hover_text(problem.summary.clone());
|
||||
if let Some(path) = &problem.log
|
||||
&& widgets::secondary(ui, None, "Open log").on_hover_text(path.display().to_string()).clicked()
|
||||
{
|
||||
self.open_log(path);
|
||||
}
|
||||
if widgets::secondary(ui, None, "Restart").clicked() {
|
||||
match craft_anything_platform::process::start_another_copy() {
|
||||
Ok(_) => ctx.send_viewport_cmd(egui::ViewportCommand::Close),
|
||||
Err(e) => self.toast(e.message, true),
|
||||
}
|
||||
}
|
||||
if widgets::secondary(ui, None, "Dismiss").clicked() {
|
||||
sync::dismiss(&problem.summary);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// Show a failure log in the system's viewer for text files.
|
||||
pub(crate) fn open_log(&mut self, path: &std::path::Path) {
|
||||
if let Err(e) = craft_anything_platform::open_path(path) {
|
||||
@@ -1428,6 +1459,7 @@ impl CraftApp {
|
||||
egui::Panel::top("titlebar").exact_size(titlebar::HEIGHT).frame(egui::Frame::new().fill(t.chrome)).show(ui, |ui| {
|
||||
guard_ui(ui, "title bar", |ui| titlebar::show(self, ui));
|
||||
});
|
||||
self.stuck_banner(ui, &ctx);
|
||||
if !self.settings.first_run_done {
|
||||
egui::CentralPanel::default().frame(egui::Frame::new().fill(t.canvas)).show(ui, |ui| guard_ui(ui, "first run", |ui| first_run::show(self, ui)));
|
||||
titlebar::window_edge(&ctx);
|
||||
|
||||
@@ -129,6 +129,7 @@ const ACTIONS: &[Action] = &[
|
||||
/// `CraftApp` methods that aren't person-facing actions: callbacks, polling, drawing, helpers.
|
||||
#[rustfmt::skip]
|
||||
const PLUMBING: &[(&str, &str)] = &[
|
||||
("stuck_banner", "draws what the lock watchdog found; `sync::problems` and its log are the headless side"),
|
||||
("act", "runs a button's tool on a worker (play, preview, open folder)"),
|
||||
("waiting_question", "finds the question an answer is for"),
|
||||
("acted", "shows a button tool's result (toast, open the preview)"),
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
//! The stuck-lock banner. In a test program of its own: what it shows is the whole process's, and
|
||||
//! a banner appearing moves everything under it, which makes clicks in tests running beside it miss.
|
||||
|
||||
use craft_anything_engine::Mode;
|
||||
use craft_anything_platform::sync;
|
||||
use craft_anything_ui_egui::demo;
|
||||
use egui_kittest::Harness;
|
||||
use egui_kittest::kittest::Queryable;
|
||||
|
||||
#[test]
|
||||
fn a_stuck_lock_is_shown_with_a_way_out() {
|
||||
let mut h = Harness::builder().with_size(egui::vec2(1280.0, 820.0)).with_max_steps(32).build_eframe(move |cc| {
|
||||
craft_anything_ui_egui::CraftApp::setup_context(&cc.egui_ctx);
|
||||
demo::app(demo::engine(Mode::Beginner, true))
|
||||
});
|
||||
let summary = "a lock taken at x.rs:1 is stuck";
|
||||
h.run_steps(2);
|
||||
assert!(h.query_by_label("Restart").is_none(), "nothing wrong, nothing shown");
|
||||
sync::report(sync::ProblemKind::SlowJob, "a long job is still running");
|
||||
h.run_steps(2);
|
||||
assert!(h.query_by_label("Restart").is_none(), "a long job is no alarm");
|
||||
sync::report(sync::ProblemKind::Stuck, summary);
|
||||
h.run_steps(4);
|
||||
h.get_by_label("Restart");
|
||||
h.get_by_label("Dismiss").click();
|
||||
h.run_steps(4);
|
||||
assert!(!sync::problems().iter().any(|p| p.summary == summary));
|
||||
assert!(h.query_by_label("Restart").is_none());
|
||||
}
|
||||
@@ -43,6 +43,11 @@ against a stand-in server, not yet a real Cutter). Every tool is listed in `docs
|
||||
|
||||
## Known limits
|
||||
|
||||
- Locks: every mutex but the browser viewer's goes through `platform::sync`. A deadlock on one thread or across
|
||||
threads is broken when it forms, with an error and a report in `logs/`; a watchdog looks every 5 s; a holder
|
||||
that isn't a labelled job is reported after 2 minutes and waited on for 15 at most. Tools are labelled, so they
|
||||
may run as long as they need. Checked by unit tests of each case, not by a live deadlock in the desktop app.
|
||||
Waits on channels and `join` aren't watched; each child program has its own deadlines.
|
||||
- A failed agent CLI run says what ran and what it printed, and writes a private log to `logs/` in the
|
||||
config folder (the newest 50 are kept; the person's message is left out; **Open log** in the app, `log:`
|
||||
in the CLI). Checked live against a GitHub ssh alias from the CLI; the button by a UI test, not by
|
||||
|
||||
@@ -68,6 +68,10 @@ what happens without a program builds its session in the mode the tool runs in.
|
||||
`a`, so `b`'s tests never ran; the rerun was filtered to the failing test, and a broken engine id
|
||||
reached `main`. Run with `--no-fail-fast`, and after a fix rerun the whole crates, not a filter.
|
||||
|
||||
**A test that changes the whole process.** A UI test raised the process-wide stuck-lock banner; in
|
||||
CI the banner moved the layout under another test running beside it, and its click missed. It passed
|
||||
25 runs on a Mac. Anything a test sets for the whole process goes in a test program of its own.
|
||||
|
||||
**Fixtures that leak into each other.** A "nothing here" path sat beside a fake game folder in the
|
||||
same temp directory, and game detection walked up and found the game. Give each case its own
|
||||
`tempfile::tempdir()`.
|
||||
@@ -296,6 +300,13 @@ and run `cargo xtask secrets` before every commit (the `.githooks/pre-commit` ho
|
||||
Update STATUS, the roadmap and generated docs (`cargo xtask tools`, `cargo xtask schemas`) in the
|
||||
same PR as the change.
|
||||
|
||||
**A second lock in one statement.** `CliSpec { deep: !engine.session().clean_room(), logs: engine.session().config_dir(), … }`
|
||||
looked like two quick reads, but an unbound guard lives to the end of the statement, so the second
|
||||
waited on the first forever. Every build froze after studying its target while the card calmly
|
||||
said "This is a big one", and no test ran that line. Read the session once; `cargo xtask locks`
|
||||
and `platform::sync` now catch this. A wait message must not reassure about what it can't see:
|
||||
say how long it's been since anything happened. Find a hang with `sample <pid>`: it names the lock.
|
||||
|
||||
**An error that points at nothing.** "Check the output below; run Claude Code by hand" came with no
|
||||
output below (the event carried only message and remediation) and no command, and the program
|
||||
that failed was really ssh, talking to a git alias from `~/.ssh/config` that had been offered as a
|
||||
|
||||
@@ -8,3 +8,5 @@ publish = false
|
||||
[dependencies]
|
||||
craft-anything-engine = { workspace = true }
|
||||
toml = { workspace = true }
|
||||
syn = { version = "2", features = ["full", "visit"] }
|
||||
proc-macro2 = { version = "1", features = ["span-locations"] }
|
||||
@@ -0,0 +1,274 @@
|
||||
//! `cargo xtask locks`: finds code that would wait on the engine's session lock while already
|
||||
//! holding it. `.session()` hands back a guard, and a guard that isn't bound to a name still lives
|
||||
//! until the end of the whole statement, so a second `.session()` in that statement waits forever.
|
||||
//! At run time such a wait is caught and stopped (crates/platform/src/sync.rs); this finds it first.
|
||||
//!
|
||||
//! Flags, in non-test code:
|
||||
//! - two `.session()` calls whose guards are alive together in one statement;
|
||||
//! - `match`, `if let` or `while let` on a `.session()`, with another in its body;
|
||||
//! - `let g = ….session();` followed in that block by `.session()` or `engine.call(…)`, before `drop(g)`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use syn::visit::{self, Visit};
|
||||
|
||||
fn is_session(m: &syn::ExprMethodCall) -> bool {
|
||||
m.args.is_empty() && (m.method == "session" || m.method == "try_session")
|
||||
}
|
||||
|
||||
fn is_engine_call(m: &syn::ExprMethodCall) -> bool {
|
||||
if m.method != "call" {
|
||||
return false;
|
||||
}
|
||||
match &*m.receiver {
|
||||
syn::Expr::Field(f) => matches!(&f.member, syn::Member::Named(n) if n == "engine"),
|
||||
syn::Expr::Path(p) => p.path.is_ident("engine"),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// `.session()` calls anywhere under a node, closures aside (they run later, if at all).
|
||||
#[derive(Default)]
|
||||
struct Everywhere {
|
||||
lines: Vec<usize>,
|
||||
with_calls: bool,
|
||||
}
|
||||
|
||||
impl Visit<'_> for Everywhere {
|
||||
fn visit_expr_closure(&mut self, _: &syn::ExprClosure) {}
|
||||
fn visit_item(&mut self, _: &syn::Item) {}
|
||||
fn visit_expr_method_call(&mut self, m: &syn::ExprMethodCall) {
|
||||
if is_session(m) || (self.with_calls && is_engine_call(m)) {
|
||||
self.lines.push(m.method.span().start().line);
|
||||
}
|
||||
visit::visit_expr_method_call(self, m);
|
||||
}
|
||||
}
|
||||
|
||||
/// `.session()` calls whose guards live to the end of this statement: nested blocks have their own
|
||||
/// statements, and a plain `if`'s condition ends before its body.
|
||||
#[derive(Default)]
|
||||
struct SameStatement {
|
||||
lines: Vec<usize>,
|
||||
/// (first, second) lines of scrutinee problems found on the way.
|
||||
held_into_body: Vec<(usize, usize)>,
|
||||
}
|
||||
|
||||
impl SameStatement {
|
||||
fn scrutinee(&mut self, scrutinee: &syn::Expr, body: &dyn Fn(&mut Everywhere)) {
|
||||
let mut head = Everywhere::default();
|
||||
head.visit_expr(scrutinee);
|
||||
if let Some(first) = head.lines.first() {
|
||||
let mut rest = Everywhere::default();
|
||||
body(&mut rest);
|
||||
if let Some(second) = rest.lines.first() {
|
||||
self.held_into_body.push((*first, *second));
|
||||
}
|
||||
}
|
||||
self.visit_expr(scrutinee);
|
||||
}
|
||||
}
|
||||
|
||||
impl Visit<'_> for SameStatement {
|
||||
fn visit_expr_closure(&mut self, _: &syn::ExprClosure) {}
|
||||
fn visit_item(&mut self, _: &syn::Item) {}
|
||||
fn visit_block(&mut self, _: &syn::Block) {}
|
||||
fn visit_expr_method_call(&mut self, m: &syn::ExprMethodCall) {
|
||||
if is_session(m) {
|
||||
self.lines.push(m.method.span().start().line);
|
||||
}
|
||||
visit::visit_expr_method_call(self, m);
|
||||
}
|
||||
fn visit_expr_if(&mut self, e: &syn::ExprIf) {
|
||||
if let syn::Expr::Let(l) = &*e.cond {
|
||||
self.scrutinee(&l.expr, &|rest| rest.visit_block(&e.then_branch));
|
||||
}
|
||||
// A plain condition is its own scope, checked as if it were a statement of its own.
|
||||
}
|
||||
fn visit_expr_while(&mut self, e: &syn::ExprWhile) {
|
||||
if let syn::Expr::Let(l) = &*e.cond {
|
||||
self.scrutinee(&l.expr, &|rest| rest.visit_block(&e.body));
|
||||
}
|
||||
}
|
||||
fn visit_expr_match(&mut self, e: &syn::ExprMatch) {
|
||||
self.scrutinee(&e.expr, &|rest| {
|
||||
for arm in &e.arms {
|
||||
rest.visit_arm(arm);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// The name a statement binds a session guard to, when it does.
|
||||
fn bound_guard(stmt: &syn::Stmt) -> Option<(String, usize)> {
|
||||
let syn::Stmt::Local(local) = stmt else { return None };
|
||||
let syn::Pat::Ident(name) = &local.pat else { return None };
|
||||
let syn::Expr::MethodCall(m) = &*local.init.as_ref()?.expr else { return None };
|
||||
is_session(m).then(|| (name.ident.to_string(), m.method.span().start().line))
|
||||
}
|
||||
|
||||
fn drops(stmt: &syn::Stmt, name: &str) -> bool {
|
||||
let syn::Stmt::Expr(syn::Expr::Call(c), _) = stmt else { return false };
|
||||
let syn::Expr::Path(f) = &*c.func else { return false };
|
||||
f.path.is_ident("drop") && matches!(c.args.first(), Some(syn::Expr::Path(a)) if a.path.is_ident(name))
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Finder {
|
||||
found: Vec<String>,
|
||||
}
|
||||
|
||||
impl Finder {
|
||||
fn plain_conditions(&mut self, e: &syn::Expr) {
|
||||
let mut one = SameStatement::default();
|
||||
one.visit_expr(e);
|
||||
self.report(one);
|
||||
}
|
||||
|
||||
fn report(&mut self, s: SameStatement) {
|
||||
if let [first, second, ..] = s.lines[..] {
|
||||
self.found.push(format!("{first}: this statement takes the session lock again on line {second} while still holding it"));
|
||||
}
|
||||
for (first, second) in s.held_into_body {
|
||||
self.found.push(format!("{first}: the session lock taken here is held through its body, which takes it again on line {second}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Visit<'_> for Finder {
|
||||
fn visit_item_mod(&mut self, m: &syn::ItemMod) {
|
||||
let test = m.attrs.iter().any(|a| a.path().is_ident("cfg") && a.meta.to_token_stream_string().contains("test"));
|
||||
if !test {
|
||||
visit::visit_item_mod(self, m);
|
||||
}
|
||||
}
|
||||
fn visit_item_fn(&mut self, f: &syn::ItemFn) {
|
||||
if !f.attrs.iter().any(|a| a.path().is_ident("test")) {
|
||||
visit::visit_item_fn(self, f);
|
||||
}
|
||||
}
|
||||
fn visit_expr_if(&mut self, e: &syn::ExprIf) {
|
||||
if !matches!(&*e.cond, syn::Expr::Let(_)) {
|
||||
self.plain_conditions(&e.cond);
|
||||
}
|
||||
visit::visit_expr_if(self, e);
|
||||
}
|
||||
fn visit_block(&mut self, block: &syn::Block) {
|
||||
for (i, stmt) in block.stmts.iter().enumerate() {
|
||||
let mut one = SameStatement::default();
|
||||
one.visit_stmt(stmt);
|
||||
self.report(one);
|
||||
if let Some((name, line)) = bound_guard(stmt) {
|
||||
for later in block.stmts.iter().skip(i + 1) {
|
||||
if drops(later, &name) {
|
||||
break;
|
||||
}
|
||||
let mut rest = Everywhere { with_calls: true, ..Default::default() };
|
||||
rest.visit_stmt(later);
|
||||
if let Some(second) = rest.lines.first() {
|
||||
self.found.push(format!("{line}: `{name}` holds the session lock, and line {second} needs it again; drop it first"));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
visit::visit_block(self, block);
|
||||
}
|
||||
}
|
||||
|
||||
trait MetaText {
|
||||
fn to_token_stream_string(&self) -> String;
|
||||
}
|
||||
|
||||
impl MetaText for syn::Meta {
|
||||
fn to_token_stream_string(&self) -> String {
|
||||
match self {
|
||||
syn::Meta::List(l) => l.tokens.to_string(),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn problems_in(source: &str) -> Result<Vec<String>, String> {
|
||||
let file = syn::parse_file(source).map_err(|e| e.to_string())?;
|
||||
let mut f = Finder::default();
|
||||
f.visit_file(&file);
|
||||
Ok(f.found)
|
||||
}
|
||||
|
||||
fn rust_files(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else { return };
|
||||
for e in entries.flatten() {
|
||||
let p = e.path();
|
||||
let name = e.file_name().to_string_lossy().into_owned();
|
||||
if p.is_dir() {
|
||||
if !matches!(name.as_str(), "tests" | "examples" | "target") {
|
||||
rust_files(&p, out);
|
||||
}
|
||||
} else if name.ends_with(".rs") && name != "tests.rs" {
|
||||
out.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn check(root: &Path) -> bool {
|
||||
let mut files = Vec::new();
|
||||
rust_files(&root.join("crates"), &mut files);
|
||||
rust_files(&root.join("apps"), &mut files);
|
||||
let mut bad = 0;
|
||||
for file in &files {
|
||||
let Ok(source) = std::fs::read_to_string(file) else { continue };
|
||||
if !source.contains("session()") {
|
||||
continue;
|
||||
}
|
||||
let shown = file.strip_prefix(root).unwrap_or(file).display().to_string();
|
||||
match problems_in(&source) {
|
||||
Ok(found) => {
|
||||
for f in found {
|
||||
eprintln!("{shown}:{f}");
|
||||
bad += 1;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("{shown}: couldn't read it: {e}");
|
||||
bad += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if bad == 0 {
|
||||
println!("locks: ok ({} files)", files.len());
|
||||
} else {
|
||||
eprintln!("locks: {bad} place(s) would wait on a lock they already hold");
|
||||
}
|
||||
bad == 0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::problems_in;
|
||||
|
||||
fn flagged(body: &str) -> bool {
|
||||
!problems_in(&format!("fn f() {{ {body} }}")).unwrap().is_empty()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finds_a_lock_held_into_a_second() {
|
||||
// The bug that started this: both guards live until the struct expression ends.
|
||||
assert!(flagged("let s = Spec { a: !e.session().x(), b: e.session().y() };"));
|
||||
assert!(flagged("let n = e.session().a() + e.session().b();"));
|
||||
assert!(flagged("match e.session().x() { A => e.session().y(), B => 0 };"));
|
||||
assert!(flagged("if let Some(x) = e.session().x() { e.session().y(); }"));
|
||||
assert!(flagged("let g = e.session(); e.session().y();"));
|
||||
assert!(flagged("let g = self.engine.session(); self.engine.call(\"t\", p, o);"));
|
||||
assert!(flagged("if e.session().a() && e.session().b() { }"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn leaves_correct_code_alone() {
|
||||
assert!(!flagged("let a = e.session().x(); let b = e.session().y();"));
|
||||
assert!(!flagged("if e.session().a() { e.session().b(); }"), "a plain condition's guard ends before the body");
|
||||
assert!(!flagged("let g = e.session(); g.a(); drop(g); e.session().y();"));
|
||||
assert!(!flagged("let (a, b) = { let s = e.session(); (s.a(), s.b()) }; e.session().y();"));
|
||||
assert!(!flagged("let g = e.session(); let f = move || other.session();"), "a closure runs later");
|
||||
assert!(!flagged("match x { A => e.session().y(), B => e.session().z() };"), "only one arm runs");
|
||||
}
|
||||
}
|
||||
+5
-2
@@ -1,10 +1,11 @@
|
||||
//! `cargo xtask <layers | tools [--check] | wasm | licenses | schemas [--check] | pending [--count] | secrets [--history] | ci | version [set X.Y.Z] | ico <out.ico> <png>...>`
|
||||
//! `cargo xtask <layers | locks | tools [--check] | wasm | licenses | schemas [--check] | pending [--count] | secrets [--history] | ci | version [set X.Y.Z] | ico <out.ico> <png>...>`
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, ExitCode};
|
||||
|
||||
mod ico;
|
||||
mod locks;
|
||||
mod secrets;
|
||||
|
||||
/// Workspace crates and their layer. A crate may depend only on strictly lower layers
|
||||
@@ -35,6 +36,7 @@ fn main() -> ExitCode {
|
||||
let root = workspace_root();
|
||||
let ok = match args.first().map(String::as_str) {
|
||||
Some("layers") => layers(&root),
|
||||
Some("locks") => locks::check(&root),
|
||||
Some("tools") => tools(&root, args.iter().any(|a| a == "--check")),
|
||||
Some("schemas") => schemas(&root, args.iter().any(|a| a == "--check")),
|
||||
Some("wasm") => wasm(),
|
||||
@@ -48,6 +50,7 @@ fn main() -> ExitCode {
|
||||
}
|
||||
Some("ci") => {
|
||||
layers(&root)
|
||||
&& locks::check(&root)
|
||||
&& secrets::secrets(&root, false)
|
||||
&& run("cargo", &["fmt", "--all", "--check"])
|
||||
&& run("cargo", &["clippy", "--workspace", "--all-targets", "--", "-D", "warnings"])
|
||||
@@ -60,7 +63,7 @@ fn main() -> ExitCode {
|
||||
}
|
||||
_ => {
|
||||
eprintln!(
|
||||
"usage: cargo xtask <layers | tools [--check] | wasm | licenses | schemas [--check] | pending [--count] | secrets [--history] | ci | version [set X.Y.Z] | ico <out.ico> <png>...>"
|
||||
"usage: cargo xtask <layers | locks | tools [--check] | wasm | licenses | schemas [--check] | pending [--count] | secrets [--history] | ci | version [set X.Y.Z] | ico <out.ico> <png>...>"
|
||||
);
|
||||
false
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user