mirror of
https://github.com/storytold/craft-anything.git
synced 2026-11-20 14:30:55 +01:00
Stop waiting on a builder's process once its stream says it's done (#1426)
A coding agent's turn ended only when its process exited, so when Claude Code printed its final result and then kept running (a shutdown hook, logging), a finished build stayed on "Building" or "Pausing" for as long as the idle limit allowed, hours for a builder turn. The turn now ends a short grace (5 s) after the stream's own result, without being reported as stopped or failed; a deadline hit before any result still stops the turn as before.
This commit is contained in:
1 parent
a4f1ee81b0
commit
c813f6bec5
3 files changed
+135
-2
No files matched your search
@@ -899,6 +899,13 @@ fn nothing_done(backend: &str) -> AgentEvent {
|
||||
AgentEvent::TurnDone { backend: backend.to_string(), input_tokens: 0, output_tokens: 0, cost_usd: None, cost_is_estimate: false }
|
||||
}
|
||||
|
||||
/// How long a turn's process may keep running after its stream already said the turn was done
|
||||
/// (Claude Code's final `result` line, or another adapter's equivalent) before we stop waiting on
|
||||
/// it. The turn is over once the stream says so; a shutdown hook or cleanup step that hangs after
|
||||
/// that must not leave the build "Pausing" for as long as the idle timeout allows (hours, for a
|
||||
/// builder turn, since a hung hook alone never touches stdout again to reset it).
|
||||
const RESULT_GRACE: Duration = Duration::from_secs(5);
|
||||
|
||||
/// One message as one run of the agent's program (here or over SSH), its events sent on `tx`
|
||||
/// as they come. Returns the `TurnDone` that ends the turn, whatever happened, for the caller to
|
||||
/// send once the conversation can take the next message. The session it reports is kept for the
|
||||
@@ -1034,6 +1041,10 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender<AgentEvent>, sto
|
||||
let failed_tools = Arc::new(AtomicBool::new(false));
|
||||
let (failed_tools2, failed_tools3) = (failed_tools.clone(), failed_tools.clone());
|
||||
let stop2 = stop.clone();
|
||||
// Set once the stream's own terminal event is parsed, so the process is stopped `RESULT_GRACE`
|
||||
// after rather than whenever it happens to exit (its own shutdown hooks aren't ours to wait on).
|
||||
let result_seen: Arc<Mutex<Option<std::time::Instant>>> = Arc::new(Mutex::new(None));
|
||||
let (result_seen2, result_seen3) = (result_seen.clone(), result_seen.clone());
|
||||
let mut proc = (if remote.is_none() { adapter.process(&prog) } else { Process::new(&prog) })
|
||||
.args(&argv)
|
||||
.timeout(Duration::from_secs(spec.options.timeout_secs.unwrap_or(turn)))
|
||||
@@ -1043,7 +1054,10 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender<AgentEvent>, sto
|
||||
}
|
||||
let mut proc = proc
|
||||
.stdout_cap(4 * 1024 * 1024)
|
||||
.cancel_when(move || stop2.is_stopped() || failed_tools2.load(Ordering::SeqCst))
|
||||
.cancel_when(move || {
|
||||
let seen: Option<std::time::Instant> = *result_seen3.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
stop2.is_stopped() || failed_tools2.load(Ordering::SeqCst) || seen.is_some_and(|at| at.elapsed() >= RESULT_GRACE)
|
||||
})
|
||||
.on_stdout_line(move |l| {
|
||||
if failed_tools3.load(Ordering::SeqCst) {
|
||||
return;
|
||||
@@ -1066,6 +1080,12 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender<AgentEvent>, sto
|
||||
failed_tools3.store(true, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
if p.saw_result {
|
||||
let mut seen = result_seen2.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
if seen.is_none() {
|
||||
*seen = Some(std::time::Instant::now());
|
||||
}
|
||||
}
|
||||
for e in events {
|
||||
let _ = tx2.send(e);
|
||||
}
|
||||
@@ -1090,10 +1110,16 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender<AgentEvent>, sto
|
||||
} else if let Some(id) = &p.session_id {
|
||||
*session.lock().unwrap_or_else(PoisonError::into_inner) = Some(id.clone());
|
||||
}
|
||||
// The stream already said the turn was done (its terminal `result` line): whatever ended the
|
||||
// process after that — our own cancellation once `RESULT_GRACE` passed, or a wall/idle
|
||||
// deadline that happened to fire in that same window — isn't a failure of the turn itself. A
|
||||
// genuine protocol failure (`p.failure`, handled above) and a deadline hit before any result
|
||||
// was parsed still are: this never excuses a turn that didn't finish.
|
||||
let ended_after_result = p.saw_result && matches!(result.as_ref().err().map(|e| e.code), Some(ErrorCode::Cancelled) | Some(ErrorCode::TimedOut));
|
||||
if p.failure.is_none() {
|
||||
let failed = match &result {
|
||||
Ok(out) if !out.success => Some(TurnFailure::process_exit(out.code)),
|
||||
Err(e) if e.code != ErrorCode::Cancelled => Some(TurnFailure::from_process(e)),
|
||||
Err(e) if e.code != ErrorCode::Cancelled && !ended_after_result => Some(TurnFailure::from_process(e)),
|
||||
_ => None,
|
||||
};
|
||||
*failure.lock().unwrap_or_else(PoisonError::into_inner) = failed;
|
||||
@@ -1126,6 +1152,10 @@ fn run_cli(spec: &CliSpec, text: &str, retry: bool, tx: &Sender<AgentEvent>, sto
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
// The stream already said the turn was done; it's that result, not this cancellation or
|
||||
// deadline, that ends the turn. Whatever kept the process itself running past that (a
|
||||
// shutdown hook, logging, cleanup) isn't ours to wait on or report as a stop or a failure.
|
||||
Err(_) if ended_after_result => {}
|
||||
Err(e) if e.code == craft_anything_evidence::ErrorCode::Cancelled => {
|
||||
let _ = tx.send(AgentEvent::Status { text: "Stopped.".into() });
|
||||
}
|
||||
@@ -2727,6 +2757,93 @@ echo '{"type":"turn.completed","usage":{"input_tokens":1,"output_tokens":1}}'
|
||||
}
|
||||
}
|
||||
|
||||
/// A fake `claude` that prints the turn's `result` line, then hangs well past it (standing in
|
||||
/// for a shutdown hook or cleanup step that got stuck after Claude Code's own turn was done).
|
||||
#[cfg(unix)]
|
||||
fn hangs_after_its_result(dir: &std::path::Path) -> PathBuf {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let program = dir.join("claude");
|
||||
let reply = r#"{"type":"result","subtype":"success","is_error":false,"num_turns":1,"session_id":"s1"}"#;
|
||||
std::fs::write(&program, format!("#!/bin/sh\necho '{reply}'\nsleep 20\n")).unwrap();
|
||||
std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
program
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn hangs_after_its_result(dir: &std::path::Path) -> PathBuf {
|
||||
let program = dir.join("claude.cmd");
|
||||
let reply = r#"{"type":"result","subtype":"success","is_error":false,"num_turns":1,"session_id":"s1"}"#;
|
||||
std::fs::write(&program, format!("@echo off\r\necho {reply}\r\nping -n 21 127.0.0.1 >nul\r\n")).unwrap();
|
||||
program
|
||||
}
|
||||
|
||||
/// A turn whose stream already said it was done (the `result` line) must not stay "Building"
|
||||
/// for as long as the process takes to actually exit: a shutdown hook or cleanup step that
|
||||
/// hangs after that line isn't ours to wait on. Without `RESULT_GRACE`, this test would see no
|
||||
/// `TurnDone` until the fake CLI's 20 s sleep ends; with it, `TurnDone` arrives within seconds.
|
||||
#[test]
|
||||
fn a_turn_already_said_done_ends_without_waiting_for_the_process_to_exit() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let program = hangs_after_its_result(dir.path());
|
||||
let config =
|
||||
BackendConfig { name: "Claude".into(), kind: BackendKind::AgentCli, model: "claude-code".into(), base_url: None, ssh_host: None, enabled: true };
|
||||
let options = BackendOptions { command: Some(program.display().to_string()), ..Default::default() };
|
||||
let mut spec = CliSpec::new(config, options, false, None);
|
||||
// A short, single-line preamble: on Windows, a `.cmd` stand-in can't take the real
|
||||
// multi-line default (a batch file's arguments can't carry a newline), which is beside
|
||||
// the point of this test either way.
|
||||
spec.preamble = Some("P:".into());
|
||||
let mut c = CliConversation::new(spec);
|
||||
c.send("go");
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut events = Vec::new();
|
||||
loop {
|
||||
let got = c.poll();
|
||||
let done = got.iter().any(|e| matches!(e, AgentEvent::TurnDone { .. }));
|
||||
events.extend(got);
|
||||
if done {
|
||||
break;
|
||||
}
|
||||
assert!(std::time::Instant::now() < deadline, "no TurnDone within {RESULT_GRACE:?} of the result line: {events:?}");
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
let bad = |e: &AgentEvent| matches!(e, AgentEvent::Error { .. }) || matches!(e, AgentEvent::Status { text } if text == "Stopped.");
|
||||
assert!(!events.iter().any(bad), "{events:?}");
|
||||
}
|
||||
|
||||
/// PR #1426 review: a wall deadline shorter than `RESULT_GRACE` can fire before our own grace
|
||||
/// period does, even though the stream already parsed the terminal result. That race must
|
||||
/// still count as the turn completing, not as a retryable failure (a builder could otherwise
|
||||
/// retry a turn that had already finished).
|
||||
#[test]
|
||||
fn a_wall_deadline_during_result_grace_still_counts_as_done() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let program = hangs_after_its_result(dir.path());
|
||||
let config =
|
||||
BackendConfig { name: "Claude".into(), kind: BackendKind::AgentCli, model: "claude-code".into(), base_url: None, ssh_host: None, enabled: true };
|
||||
// Shorter than RESULT_GRACE: the result is parsed almost at once, but the process (asleep
|
||||
// for 20 s) is still alive when this deadline, not our own grace period, ends the run.
|
||||
let options = BackendOptions { command: Some(program.display().to_string()), timeout_secs: Some(2), ..Default::default() };
|
||||
let mut spec = CliSpec::new(config, options, false, None);
|
||||
spec.preamble = Some("P:".into());
|
||||
let mut c = CliConversation::new(spec);
|
||||
c.send("go");
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(10);
|
||||
let mut events = Vec::new();
|
||||
loop {
|
||||
let got = c.poll();
|
||||
let done = got.iter().any(|e| matches!(e, AgentEvent::TurnDone { .. }));
|
||||
events.extend(got);
|
||||
if done {
|
||||
break;
|
||||
}
|
||||
assert!(std::time::Instant::now() < deadline, "no TurnDone: {events:?}");
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
let bad = |e: &AgentEvent| matches!(e, AgentEvent::Error { .. }) || matches!(e, AgentEvent::Status { text } if text == "Stopped.");
|
||||
assert!(!events.iter().any(bad), "a result parsed before the wall deadline must not be reported as a failure: {events:?}");
|
||||
}
|
||||
|
||||
/// #91: a Mac app with no CLI beside it reached Claude Code, which stopped with a stack trace.
|
||||
#[test]
|
||||
fn a_missing_or_broken_helper_is_said_plainly() {
|
||||
|
||||
@@ -349,6 +349,13 @@ Every round spends the person's money or quota. One that can't do anything shoul
|
||||
write the project and show its build's output, not to fix a build (`NO_PROJECT_PROMPT`). — claimed: fabox-os, branch bugfix/736-stop-empty-build-loop
|
||||
- [ ] Live-check #736 on Windows with gemini-cli: an empty app/ and a builder that claims `npm run build`
|
||||
stops after two turns (or 10 checks) with the "no files yet" note, and Try again doesn't restart it
|
||||
- [x] A finished builder turn whose own CLI process lingers after its stream already said it was
|
||||
done (e.g. a stuck shutdown hook) no longer leaves the build on "Building"/"Pausing" for as
|
||||
long as the idle timeout allows (hours, for a builder turn): `run_cli` ends the turn
|
||||
`RESULT_GRACE` after its terminal `result` line instead of waiting for the process itself to
|
||||
exit. A wall or idle deadline that fires during that grace period (the result was already
|
||||
parsed) counts as completion, not a failure; a deadline with no parsed result still stops the
|
||||
turn as before — claimed: mytrade2909-design, branch bugfix/builder-stuck-after-turn-completes, PR #1426
|
||||
|
||||
### R5.4 Local models can build
|
||||
Owner: open · Status: todo
|
||||
|
||||
@@ -242,6 +242,15 @@ the program it opened stayed open (and held the shell's pipe, so the command nev
|
||||
the caller may exit next, use `kill_and_wait`, and check after a live run that nothing was left
|
||||
running.
|
||||
|
||||
**A turn's own process outlived its answer.** `run_cli` only sent `AgentEvent::TurnDone` once the
|
||||
whole coding-agent subprocess exited, never once its JSON stream said the turn was done (the
|
||||
`result` line). Claude Code can print that line and then linger (its own shutdown hook, logging)
|
||||
for as long as the idle timeout allows — hours, for a builder turn — so a finished build stayed on
|
||||
"Building"/"Pausing" long after the real work and even a final report were done, with no crash and
|
||||
no error to find. Once the stream's own terminal event is parsed, stop waiting on the process
|
||||
itself: fold a `result_seen` flag into `cancel_when` with a short grace period (`RESULT_GRACE`),
|
||||
and don't report that as "Stopped." A terminal stream event, not the OS exit, is what ends a turn.
|
||||
|
||||
**A console held open by a program started next door.** On Windows Server 2022 opening a
|
||||
pseudoconsole briefly makes inheritable handles (a Unix terminal too, until it's made close-on-exec),
|
||||
so a program another thread started at that moment kept the console open until it ended, and closing
|
||||
|
||||
Reference in new issue
Block a user