diff --git a/crates/ai/src/cli_agents/mod.rs b/crates/ai/src/cli_agents/mod.rs index 2e2d27f2..fba6f136 100644 --- a/crates/ai/src/cli_agents/mod.rs +++ b/crates/ai/src/cli_agents/mod.rs @@ -899,6 +899,13 @@ fn nothing_done(backend: &str) -> AgentEvent { AgentEvent::TurnDone { backend: backend.to_string(), input_tokens: 0, output_tokens: 0, cost_usd: None, cost_is_estimate: false } } +/// How long a turn's process may keep running after its stream already said the turn was done +/// (Claude Code's final `result` line, or another adapter's equivalent) before we stop waiting on +/// it. The turn is over once the stream says so; a shutdown hook or cleanup step that hangs after +/// that must not leave the build "Pausing" for as long as the idle timeout allows (hours, for a +/// builder turn, since a hung hook alone never touches stdout again to reset it). +const RESULT_GRACE: Duration = Duration::from_secs(5); + /// One message as one run of the agent's program (here or over SSH), its events sent on `tx` /// as they come. Returns the `TurnDone` that ends the turn, whatever happened, for the caller to /// send once the conversation can take the next message. The session it reports is kept for the @@ -1034,6 +1041,10 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender, sto let failed_tools = Arc::new(AtomicBool::new(false)); let (failed_tools2, failed_tools3) = (failed_tools.clone(), failed_tools.clone()); let stop2 = stop.clone(); + // Set once the stream's own terminal event is parsed, so the process is stopped `RESULT_GRACE` + // after rather than whenever it happens to exit (its own shutdown hooks aren't ours to wait on). + let result_seen: Arc>> = Arc::new(Mutex::new(None)); + let (result_seen2, result_seen3) = (result_seen.clone(), result_seen.clone()); let mut proc = (if remote.is_none() { adapter.process(&prog) } else { Process::new(&prog) }) .args(&argv) .timeout(Duration::from_secs(spec.options.timeout_secs.unwrap_or(turn))) @@ -1043,7 +1054,10 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender, sto } let mut proc = proc .stdout_cap(4 * 1024 * 1024) - .cancel_when(move || stop2.is_stopped() || failed_tools2.load(Ordering::SeqCst)) + .cancel_when(move || { + let seen: Option = *result_seen3.lock().unwrap_or_else(PoisonError::into_inner); + stop2.is_stopped() || failed_tools2.load(Ordering::SeqCst) || seen.is_some_and(|at| at.elapsed() >= RESULT_GRACE) + }) .on_stdout_line(move |l| { if failed_tools3.load(Ordering::SeqCst) { return; @@ -1066,6 +1080,12 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender, sto failed_tools3.store(true, Ordering::SeqCst); } } + if p.saw_result { + let mut seen = result_seen2.lock().unwrap_or_else(PoisonError::into_inner); + if seen.is_none() { + *seen = Some(std::time::Instant::now()); + } + } for e in events { let _ = tx2.send(e); } @@ -1090,10 +1110,16 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender, sto } else if let Some(id) = &p.session_id { *session.lock().unwrap_or_else(PoisonError::into_inner) = Some(id.clone()); } + // The stream already said the turn was done (its terminal `result` line): whatever ended the + // process after that — our own cancellation once `RESULT_GRACE` passed, or a wall/idle + // deadline that happened to fire in that same window — isn't a failure of the turn itself. A + // genuine protocol failure (`p.failure`, handled above) and a deadline hit before any result + // was parsed still are: this never excuses a turn that didn't finish. + let ended_after_result = p.saw_result && matches!(result.as_ref().err().map(|e| e.code), Some(ErrorCode::Cancelled) | Some(ErrorCode::TimedOut)); if p.failure.is_none() { let failed = match &result { Ok(out) if !out.success => Some(TurnFailure::process_exit(out.code)), - Err(e) if e.code != ErrorCode::Cancelled => Some(TurnFailure::from_process(e)), + Err(e) if e.code != ErrorCode::Cancelled && !ended_after_result => Some(TurnFailure::from_process(e)), _ => None, }; *failure.lock().unwrap_or_else(PoisonError::into_inner) = failed; @@ -1126,6 +1152,10 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender, sto } } Ok(_) => {} + // The stream already said the turn was done; it's that result, not this cancellation or + // deadline, that ends the turn. Whatever kept the process itself running past that (a + // shutdown hook, logging, cleanup) isn't ours to wait on or report as a stop or a failure. + Err(_) if ended_after_result => {} Err(e) if e.code == craft_anything_evidence::ErrorCode::Cancelled => { let _ = tx.send(AgentEvent::Status { text: "Stopped.".into() }); } @@ -2727,6 +2757,93 @@ echo '{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}' } } + /// A fake `claude` that prints the turn's `result` line, then hangs well past it (standing in + /// for a shutdown hook or cleanup step that got stuck after Claude Code's own turn was done). + #[cfg(unix)] + fn hangs_after_its_result(dir: &std::path::Path) -> PathBuf { + use std::os::unix::fs::PermissionsExt; + let program = dir.join("claude"); + let reply = r#"{"type":"result","subtype":"success","is_error":false,"num_turns":1,"session_id":"s1"}"#; + std::fs::write(&program, format!("#!/bin/sh\necho '{reply}'\nsleep 20\n")).unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap(); + program + } + + #[cfg(windows)] + fn hangs_after_its_result(dir: &std::path::Path) -> PathBuf { + let program = dir.join("claude.cmd"); + let reply = r#"{"type":"result","subtype":"success","is_error":false,"num_turns":1,"session_id":"s1"}"#; + std::fs::write(&program, format!("@echo off\r\necho {reply}\r\nping -n 21 127.0.0.1 >nul\r\n")).unwrap(); + program + } + + /// A turn whose stream already said it was done (the `result` line) must not stay "Building" + /// for as long as the process takes to actually exit: a shutdown hook or cleanup step that + /// hangs after that line isn't ours to wait on. Without `RESULT_GRACE`, this test would see no + /// `TurnDone` until the fake CLI's 20 s sleep ends; with it, `TurnDone` arrives within seconds. + #[test] + fn a_turn_already_said_done_ends_without_waiting_for_the_process_to_exit() { + let dir = tempfile::tempdir().unwrap(); + let program = hangs_after_its_result(dir.path()); + let config = + BackendConfig { name: "Claude".into(), kind: BackendKind::AgentCli, model: "claude-code".into(), base_url: None, ssh_host: None, enabled: true }; + let options = BackendOptions { command: Some(program.display().to_string()), ..Default::default() }; + let mut spec = CliSpec::new(config, options, false, None); + // A short, single-line preamble: on Windows, a `.cmd` stand-in can't take the real + // multi-line default (a batch file's arguments can't carry a newline), which is beside + // the point of this test either way. + spec.preamble = Some("P:".into()); + let mut c = CliConversation::new(spec); + c.send("go"); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut events = Vec::new(); + loop { + let got = c.poll(); + let done = got.iter().any(|e| matches!(e, AgentEvent::TurnDone { .. })); + events.extend(got); + if done { + break; + } + assert!(std::time::Instant::now() < deadline, "no TurnDone within {RESULT_GRACE:?} of the result line: {events:?}"); + std::thread::sleep(Duration::from_millis(10)); + } + let bad = |e: &AgentEvent| matches!(e, AgentEvent::Error { .. }) || matches!(e, AgentEvent::Status { text } if text == "Stopped."); + assert!(!events.iter().any(bad), "{events:?}"); + } + + /// PR #1426 review: a wall deadline shorter than `RESULT_GRACE` can fire before our own grace + /// period does, even though the stream already parsed the terminal result. That race must + /// still count as the turn completing, not as a retryable failure (a builder could otherwise + /// retry a turn that had already finished). + #[test] + fn a_wall_deadline_during_result_grace_still_counts_as_done() { + let dir = tempfile::tempdir().unwrap(); + let program = hangs_after_its_result(dir.path()); + let config = + BackendConfig { name: "Claude".into(), kind: BackendKind::AgentCli, model: "claude-code".into(), base_url: None, ssh_host: None, enabled: true }; + // Shorter than RESULT_GRACE: the result is parsed almost at once, but the process (asleep + // for 20 s) is still alive when this deadline, not our own grace period, ends the run. + let options = BackendOptions { command: Some(program.display().to_string()), timeout_secs: Some(2), ..Default::default() }; + let mut spec = CliSpec::new(config, options, false, None); + spec.preamble = Some("P:".into()); + let mut c = CliConversation::new(spec); + c.send("go"); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut events = Vec::new(); + loop { + let got = c.poll(); + let done = got.iter().any(|e| matches!(e, AgentEvent::TurnDone { .. })); + events.extend(got); + if done { + break; + } + assert!(std::time::Instant::now() < deadline, "no TurnDone: {events:?}"); + std::thread::sleep(Duration::from_millis(10)); + } + let bad = |e: &AgentEvent| matches!(e, AgentEvent::Error { .. }) || matches!(e, AgentEvent::Status { text } if text == "Stopped."); + assert!(!events.iter().any(bad), "a result parsed before the wall deadline must not be reported as a failure: {events:?}"); + } + /// #91: a Mac app with no CLI beside it reached Claude Code, which stopped with a stack trace. #[test] fn a_missing_or_broken_helper_is_said_plainly() { diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 7830de63..25d39b0b 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -349,6 +349,13 @@ Every round spends the person's money or quota. One that can't do anything shoul write the project and show its build's output, not to fix a build (`NO_PROJECT_PROMPT`). — claimed: fabox-os, branch bugfix/736-stop-empty-build-loop - [ ] Live-check #736 on Windows with gemini-cli: an empty app/ and a builder that claims `npm run build` stops after two turns (or 10 checks) with the "no files yet" note, and Try again doesn't restart it +- [x] A finished builder turn whose own CLI process lingers after its stream already said it was + done (e.g. a stuck shutdown hook) no longer leaves the build on "Building"/"Pausing" for as + long as the idle timeout allows (hours, for a builder turn): `run_cli` ends the turn + `RESULT_GRACE` after its terminal `result` line instead of waiting for the process itself to + exit. A wall or idle deadline that fires during that grace period (the result was already + parsed) counts as completion, not a failure; a deadline with no parsed result still stops the + turn as before — claimed: mytrade2909-design, branch bugfix/builder-stuck-after-turn-completes, PR #1426 ### R5.4 Local models can build Owner: open · Status: todo diff --git a/docs/pitfalls.md b/docs/pitfalls.md index 3335ec3e..5939ea42 100644 --- a/docs/pitfalls.md +++ b/docs/pitfalls.md @@ -242,6 +242,15 @@ the program it opened stayed open (and held the shell's pipe, so the command nev the caller may exit next, use `kill_and_wait`, and check after a live run that nothing was left running. +**A turn's own process outlived its answer.** `run_cli` only sent `AgentEvent::TurnDone` once the +whole coding-agent subprocess exited, never once its JSON stream said the turn was done (the +`result` line). Claude Code can print that line and then linger (its own shutdown hook, logging) +for as long as the idle timeout allows — hours, for a builder turn — so a finished build stayed on +"Building"/"Pausing" long after the real work and even a final report were done, with no crash and +no error to find. Once the stream's own terminal event is parsed, stop waiting on the process +itself: fold a `result_seen` flag into `cancel_when` with a short grace period (`RESULT_GRACE`), +and don't report that as "Stopped." A terminal stream event, not the OS exit, is what ends a turn. + **A console held open by a program started next door.** On Windows Server 2022 opening a pseudoconsole briefly makes inheritable handles (a Unix terminal too, until it's made close-on-exec), so a program another thread started at that moment kept the console open until it ended, and closing