diff --git a/crates/ai/src/agent.rs b/crates/ai/src/agent.rs index 61b78b48..30d28216 100644 --- a/crates/ai/src/agent.rs +++ b/crates/ai/src/agent.rs @@ -314,6 +314,9 @@ pub struct AgentLoop { failure: Arc>>, /// Build commands that failed in a row in this step (see [`MAX_FAILED_BUILDS`]). failed_builds: u32, + /// Context windows servers said they have, by chain position: a request that didn't fit + /// tells us the real size (#962), which may be smaller than the budget we assumed. + server_windows: BTreeMap, } impl AgentLoop { @@ -509,10 +512,15 @@ impl AgentLoop { fn context_budget(&self) -> u64 { let slot = self.chain.get(self.current); let configured = if self.cfg.context_tokens == 0 { slot.map(|s| s.context_tokens).unwrap_or(0) } else { self.cfg.context_tokens }; - match configured { - 0 if slot.is_some_and(|s| s.local) => context::LOCAL_CONTEXT_TOKENS, - 0 => context::CLOUD_CONTEXT_TOKENS, - n => n, + // A window the server reported replaces our guess (a budget the person set stays the most), + // with a quarter of it left free for the answer. + let reported = self.server_windows.get(&self.current).map(|window| window - window / 4); + match (configured, reported) { + (0, Some(room)) => room, + (n, Some(room)) => n.min(room), + (0, None) if slot.is_some_and(|s| s.local) => context::LOCAL_CONTEXT_TOKENS, + (0, None) => context::CLOUD_CONTEXT_TOKENS, + (n, None) => n, } } @@ -600,6 +608,7 @@ impl AgentLoop { msg_usage: &mut Usage, msg_cost: &mut Option, ) -> Option { + let mut refitted = false; loop { // A fallback can have a smaller window than the backend that just failed. if let Some((usage, cost)) = self.fit_context(system, tools) { @@ -633,6 +642,22 @@ impl AgentLoop { Err(e) => { let name = slot.name.clone(); self.host.journal("error", json!({"backend": name, "code": e.code, "message": e.message})); + // The request didn't fit the server's context window, and it said how large that + // is: make room to fit it, measured with the server's own count, and ask once more. + let detail = |k: &str| e.details.as_ref().and_then(|d| d.get(k)).and_then(Value::as_u64); + if !produced + && !refitted + && let Some(window) = detail("context_window") + { + refitted = true; + self.server_windows.insert(self.current, window); + if let Some(asked) = detail("prompt_tokens") { + self.last_input = asked; + self.last_history_estimate = context::estimate_tokens(&self.history); + } + self.emit(AgentEvent::Status { text: format!("{name} has room for {window} tokens; making room…") }); + continue; + } if !produced && self.current + 1 < self.chain.len() { self.current += 1; let next = self.chain.get(self.current).map(|s| s.name.clone()).unwrap_or_default(); @@ -819,6 +844,7 @@ impl ApiConversation { last_history_estimate: 0, failure: failure.clone(), failed_builds: 0, + server_windows: BTreeMap::new(), }; let busy2 = busy.clone(); std::thread::Builder::new() @@ -1373,9 +1399,19 @@ pub(crate) mod tests { last_history_estimate: 0, failure: Arc::default(), failed_builds: 0, + server_windows: BTreeMap::new(), }; let tools = [ToolDef { name: "look".into(), description: "x".repeat(4000), schema: json!({"type": "object"}) }]; + // A window the server reported (#962) replaces the local guess but not a smaller set budget. + agent.server_windows.insert(0, 65_536); + assert_eq!(agent.context_budget(), 1000, "a budget that was set stays the most"); + agent.chain[0].context_tokens = 0; + assert_eq!(agent.context_budget(), 49_152, "three quarters of the reported window"); + agent.server_windows.clear(); + assert_eq!(agent.context_budget(), context::LOCAL_CONTEXT_TOKENS); + agent.chain[0].context_tokens = 1000; + assert!(agent.fit_context("A short system prompt.", &[]).is_none()); assert!(host.journal.lock().unwrap().is_empty(), "a small prompt is not judged"); @@ -1538,6 +1574,64 @@ pub(crate) mod tests { assert!(trims[0].1["tokens_before"].as_u64().unwrap() > budget); } + #[test] + fn a_request_too_long_for_the_servers_window_is_fitted_and_asked_once_more() { + // #962: Ollama on its 64k default refused a 70k request; our 24k-local guess counts + // characters, so the server's own count is what the next fit is measured with. + let overflow = || { + let body = json!({"error": {"message": "request (69944 tokens) exceeds the available context size (65536 tokens), try increasing it"}}); + Err(crate::http::status_error("Ollama (local)", 400, &body.to_string())) + }; + let answer = || turn(vec![Block::Text { text: "done".into() }], StopReason::EndTurn); + for (turns, fits) in [(vec![overflow(), answer()], true), (vec![overflow(), overflow(), answer()], false)] { + let dir = tempfile::tempdir().unwrap(); + let transcript = dir.path().join("transcript.json"); + let mut history = vec![ + Msg::user_text("Build it"), + Msg { role: Speaker::Assistant, blocks: vec![Block::ToolUse { id: "old".into(), name: "look".into(), input: json!({}) }] }, + Msg { role: Speaker::User, blocks: vec![Block::ToolResult { tool_use_id: "old".into(), content: "x".repeat(8000), is_error: false }] }, + ]; + for _ in 0..context::KEEP_RECENT / 2 { + history.push(Msg { role: Speaker::Assistant, blocks: vec![Block::Text { text: "checked".into() }] }); + history.push(Msg::user_text("continue")); + } + context::save(&transcript, &history).unwrap(); + let host = Arc::new(FakeHost { calls: Mutex::default(), journal: Mutex::default() }); + let seen = Arc::new(Mutex::new(Vec::new())); + let model = Scripted { turns: VecDeque::from(turns), seen: seen.clone() }; + let chain = vec![ModelSlot { name: "Ollama (local)".into(), model: Box::new(model), local: true, effort: None, max_turns: 5, context_tokens: 0 }]; + let mut conversation = ApiConversation::start(host.clone(), chain, LoopConfig { transcript: Some(transcript), ..Default::default() }).unwrap(); + conversation.send("continue"); + let events = drain_until_done(&mut conversation, |_, _| {}); + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2, "asked once more, never again (fits={fits})"); + let old = |h: &[Msg]| { + h.iter() + .flat_map(|m| &m.blocks) + .find_map(|b| match b { + Block::ToolResult { tool_use_id, content, .. } if tool_use_id == "old" => Some(content.len()), + _ => None, + }) + .unwrap() + }; + assert_eq!(old(&seen[0]), 8000); + assert!(old(&seen[1]) < 8000, "room was made before asking again"); + assert!(events.iter().any(|e| matches!(e, AgentEvent::Status { text } if text.contains("65536"))), "{events:?}"); + let journal = host.journal.lock().unwrap(); + assert!(journal.iter().any(|(kind, entry)| kind == "context" && entry["budget"] == 65536 - 65536 / 4 && entry["tokens_before"] == 69944)); + let error = events.iter().find_map(|e| match e { + AgentEvent::Error { remediation, .. } => Some(remediation.clone().unwrap_or_default()), + _ => None, + }); + if fits { + assert!(error.is_none(), "{events:?}"); + assert!(events.iter().any(|e| matches!(e, AgentEvent::TextDelta { text } if text == "done"))); + } else { + assert!(error.is_some_and(|r| r.contains("OLLAMA_CONTEXT_LENGTH")), "{events:?}"); + } + } + } + #[test] fn fallback_chain_and_refusal() { let host = Arc::new(FakeHost { calls: Mutex::default(), journal: Mutex::default() }); diff --git a/crates/ai/src/http.rs b/crates/ai/src/http.rs index 412e6e05..ad07a0f6 100644 --- a/crates/ai/src/http.rs +++ b/crates/ai/src/http.rs @@ -60,9 +60,36 @@ pub fn status_error(what: &str, status: u16, body: &str) -> ToolError { let provider = parsed .as_ref() .and_then(|v| v.pointer("/error/code").and_then(Value::as_str).or_else(|| v.pointer("/error/type").and_then(Value::as_str)).map(str::to_string)); - ToolError::new(code, format!("{what} answered {status}: {detail}")) - .with_remediation(remediation) - .with_details(serde_json::json!({"status": status, "provider_error": provider})) + let overflow = context_overflow(&detail); + let remediation = match overflow { + Some((window, _)) => format!( + "The conversation is longer than the model's context window ({window} tokens). Give the model a larger context \ + in its server (Ollama: the Context length setting or OLLAMA_CONTEXT_LENGTH; LM Studio: the model's Context Length), \ + or set context_tokens for this AI in Settings → AI so less is sent." + ), + None => remediation.to_string(), + }; + ToolError::new(code, format!("{what} answered {status}: {detail}")).with_remediation(remediation).with_details(serde_json::json!({ + "status": status, + "provider_error": provider, + "context_window": overflow.map(|o| o.0), + "prompt_tokens": overflow.and_then(|o| o.1), + })) +} + +/// The context window a server says a request didn't fit in, and the request's size when it says +/// that too (#962). Local servers word it as llama.cpp does ("request (69944 tokens) exceeds the +/// available context size (65536 tokens)"), OpenAI-style ones as "maximum context length is 8192 +/// tokens ... you requested 9000 tokens". +pub fn context_overflow(message: &str) -> Option<(u64, Option)> { + let m = message.to_ascii_lowercase(); + let after = |marker: &str| -> Option { + let (_, rest) = m.split_once(marker)?; + let digits: String = rest.trim_start().chars().take_while(char::is_ascii_digit).collect(); + digits.parse().ok().filter(|n| *n > 0) + }; + let window = after("available context size (").or_else(|| after("maximum context length is"))?; + Some((window, after("request (").or_else(|| after("requested")))) } /// Read only the human-readable fields, never serialise an arbitrary provider payload. @@ -231,6 +258,26 @@ mod tests { v } + #[test] + fn a_full_context_window_is_read_from_the_servers_words() { + let ollama = "request (69944 tokens) exceeds the available context size (65536 tokens), try increasing it"; + assert_eq!(context_overflow(ollama), Some((65536, Some(69944)))); + let openai = "This model's maximum context length is 8192 tokens. However, you requested 9000 tokens (8000 in the messages)."; + assert_eq!(context_overflow(openai), Some((8192, Some(9000)))); + assert_eq!(context_overflow("maximum context length is 4096 tokens"), Some((4096, None))); + for other in ["Invalid request: the message at position 64 must not be empty", "context size (abc tokens)", "", "available context size ("] { + assert_eq!(context_overflow(other), None, "{other:?}"); + } + let body = serde_json::json!({"error": {"message": ollama}}).to_string(); + let e = status_error("Ollama (local)", 400, &body); + let details = e.details.unwrap(); + assert_eq!((details["context_window"].as_u64(), details["prompt_tokens"].as_u64()), (Some(65536), Some(69944))); + assert!(e.remediation.unwrap().contains("OLLAMA_CONTEXT_LENGTH")); + let plain = status_error("Ollama (local)", 400, r#"{"error":{"message":"bad"}}"#); + assert!(plain.details.unwrap()["context_window"].is_null()); + assert_eq!(plain.remediation.as_deref(), Some("The request was rejected; see the message.")); + } + #[test] fn sse_edge_cases() { let evs = collect(": comment\r\nevent: a\r\ndata: 1\r\ndata: 2\r\n\r\ndata:x\n\n\n\ndata: tail"); diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 4ac74b64..8e26cd19 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -403,6 +403,8 @@ been run live. - [ ] Find how each local server is told its context size and set it, or stop with a plain message when the prompt won't fit. We assume 24k and send no `num_ctx`, which an OpenAI-style endpoint may ignore anyway; LM Studio refused a 12.6k prompt on its 8k default (#73) +- [x] When a server refuses a request for its context window and says the size (Ollama, llama.cpp, + OpenAI-style), fit the conversation to that window and ask once more, else say how to raise it (#962) - [x] A per-backend context budget in `[[ai.backends]]` (`LoopConfig::context_tokens` exists), for services whose real window differs from 150k (#374) — implementation: `6bd94b84828ec89b805cb8a246f557d0a778c037` - [x] Find LM Studio on another port (#73): the port in its settings file (when it is there) and diff --git a/docs/STATUS.md b/docs/STATUS.md index 04ae9472..5e89f5d0 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -84,7 +84,7 @@ workspace clippy, layers, fmt and the secret scan of all history were clean. Chi | Rebuild workflow (Study → Build → Check → Refine) | `crates/ai/src/rebuild/` (kinds, checks and stacks are pluggable: docs/extending.md). Three kinds: website, app, game (a project whose stack is a game engine is always a game). The request decides how to build: a language, framework, engine or platform it names wins over the kind's default stack (Rust + egui for apps, Unity for games, the original's for websites), and the build check, the recorded stack and Play follow what was built (an Electron app opens in its own Electron; a plain static site has nothing to build and passes the build check as it is). Websites: Study also captures up to 4 other pages of the site; menus are opened on a computer and on a phone; the check serves the build on loopback and compares every page at 1440, 768 and 390 px (likeness = mean of layout and perceptual detail, SSIM, averaged over pages, leaving out the parts of the original that change by themselves, found by photographing it three times; the note adds text, page structure, the colours, font, text sizes and spacing the build misses, and per width a page that scrolls sideways, heading sizes and sections whose padding, items per row or content column differ, the errors, failed requests and accessibility gaps the build adds, and the opened menus and pop-ups). A stack or library named in the request (Next.js, shadcn/ui, Tailwind CSS…, `stacks.toml`) is what the builder uses; with none named, a website's stack is the builder's choice (told to match what the original is built with) and the stack it used is recorded from app/package.json. The builder leads: it studies, asks the tools follow-up questions, records what it can't find out as unknowns (`core_unknown_add`; the run reports the open ones), and tests its own build mid-turn with `rebuild_check` (the same checks, for every kind); a check with anything it couldn't compare, or any claim that fails (a website page that scrolls sideways, logs new errors, fails to load a file, lacks a landmark, or a menu that doesn't open; for games each self-test step, for Deep-mode sheets each original number in the code, for console apps the same exit and output as the original's runs), isn't good enough; apps and games by whether they build. Up to 6 refinement rounds, or until a website reaches 90% likeness. A turn cut off by a time limit, crash or dropped connection resumes (up to 3 times); builder turns may run 24 h. A project copied from another computer (or moved to another folder) keeps building with Claude Code: the project keeps a copy of the builder's session (`agent-session.jsonl`) after every turn and puts it back where Claude Code finds it on a computer without it, so the same conversation carries on (#451); when the session still isn't there, a new one starts from the brief, what's in app/ and the kept requests and replies (or the last summary), and questions about the build put the kept session back too, else are answered from the files and forget the missing session (#452). Live runs with Claude Code 2.1.316 on one Mac (2026-11-02), a second computer stood in for by a separate signed-in `CLAUDE_CONFIG_DIR`: a copied project's session was put back and resumed (the builder recalled its first reply from memory); a kept copy of another session was refused and the new session used the handoff; a real project synced from Windows whose session was missing started afresh and reached Ready. Not yet run between two real computers | Unit tests for prompts, stack choice, site detection, feed folding; live runs reached Build and decompiling (below) | | Context-summary usage | Successful API context summaries contribute to the message totals consumed by build usage and receive a `model_turn` journal entry with `purpose: context_summary`. Input tokens include cached ones, as coding agents report them (Ollama 0.40 sends them apart). Cached-token pricing is retained; an unknown summary price makes the total unknown, and a failed call adds no invented usage | Scripted-model tests exercise the real `ApiConversation`, transcript resumption, exact input/output and cache-price totals, journal entries, empty summaries, local zero cost, unknown prices and failed summaries. No live provider or end-to-end rebuild was used for this correction | | Local model builds | Regression coverage confirms that a fenced call followed by a summary or protocol marker is not run and the next model message says to retry it without trailing text (R5.4). Protocol markers between calls retain this retry condition even when later calls follow; ordinary words between calls remain supported in both ReAct modes. No live local-model build has been run after this change. Earlier, Ollama 0.40.7 with `qwen2.5-coder:14b` (32k context) wrote calls as JSON text, which ran whole or after a sentence; by check 3 its project compiled, but `main.rs` was still cargo's "Hello, world!", so the screenshot check failed at the round limit. A lean local builder's prompt asks for no memory tools it isn't offered, and a project one folder down is named in the check | Historical live evidence: four CLI builds of "a scientific calculator" on Windows (RTX 5090), 2026-11-04, one fix added per run | -| Backend context budgets | `context_tokens` in `[[ai.backends]]` controls when an API conversation is trimmed and summarised. Zero or omission retains local/cloud defaults; a nonzero conversation override takes priority. Before each attempt, including a fallback, fitting adds estimated history growth since the last successful request to its reported input size. Trimming resets that reference, and successful requests refresh it; summary usage remains in message totals | Regressions cover settings and model construction, defaults/overrides, smaller fallback budgets, large-budget arithmetic and new tool output growing beyond a fallback budget even when the history estimate alone is below the reported input. The latest tool results are preserved, and a short follow-up does not count already-sent history twice. This does not set server context parameters or guarantee that a complete request fits; no live provider was used | +| Backend context budgets | `context_tokens` in `[[ai.backends]]` controls when an API conversation is trimmed and summarised. Zero or omission retains local/cloud defaults; a nonzero conversation override takes priority. Before each attempt, including a fallback, fitting adds estimated history growth since the last successful request to its reported input size. Trimming resets that reference, and successful requests refresh it; summary usage remains in message totals. When a server refuses a request for its context window and says the size (Ollama and llama.cpp's "exceeds the available context size", OpenAI-style "maximum context length is"), the error carries it (`context_window`, `prompt_tokens`), the conversation is fitted to three quarters of it (it replaces the local/cloud guess; a set budget stays the most) measured with the server's own count, and the request is sent once more; if that fails too, the remediation says how to raise the window (#962) | Regressions cover settings and model construction, defaults/overrides, smaller fallback budgets, large-budget arithmetic and new tool output growing beyond a fallback budget even when the history estimate alone is below the reported input. The latest tool results are preserved, and a short follow-up does not count already-sent history twice. `a_request_too_long_for_the_servers_window_is_fitted_and_asked_once_more` covers a refit that then fits and one that doesn't (asked once more, never again); the server's two wordings are parsed in `http`. This does not set server context parameters, and LM Studio's refusal names no size, so it isn't refitted; no live provider was used | | Games | Engine detection for 12 engines (Unreal, Unity, Godot, Source, GameMaker, RPG Maker, Ren'Py, MonoGame, LÖVE, CryEngine, Cocos, Cube) with versions. Unreal and Godot games are rebuilt in the same engine, others in Unity; installed engines are found, Godot installs itself, others get guided installs. Picking an inner folder (Engine, Binaries, Paks…) finds the game's folder. Just look inside a game shows what it's made of (`core_target_triage` on its folder: engine and version, how its code ships, the libraries its Unity assembly names give away, its own code, scenes and size) and opens it in the chat to ask about it. The builder keeps `app/MECHANICS.md` and its structured twin `app/mechanics.json` (checked after each build), in Deep mode only, may reuse the original's readable files for private use (never decrypting; generated pictures and 3D models replace its art when the person turns them on; Clean room copies none of its files), lists them in `CREDITS.md` (checked, with the private-use line in `README.md`); a game is ready only when its `--self-test` shows the player moving, acting and the HUD drawn, and the result can be started from the project page | Unit tests; live runs on Windows | | Clean room and Deep mode | Clean room (the default) runs only the tools in `engine/src/clean_room.rs`: ones that touch no target or watch from outside (the website as a visitor sees it, pictures of a window, its controls, recorded runs, a phone's screen). Every other tool, a plugin's included, is refused in the app, CLI, MCP and agents, with a way to turn on Deep mode. A Clean room build studies an app or a game only from the person's pictures and words (and a picture of its window when runtime study is on); its builder's reads and commands go through an approval rule that keeps it out of the original's folder and site (with Claude Code, and with ACP agents for what they ask about; other agents aren't held to it, see roadmap R2.2). Deep mode (`deep_mode` in Settings, `--deep` for one CLI run) uses every tool: the builder sees them all, and a website's study also reads its JavaScript and watches its start page for ten seconds (capture/how-it-works.md). Ghidra adds its decompiler in Deep mode when it's installed: it is found, or installed after asking. Study starts one analysis of the main program, saved and shared with the builder's tool server through a lock file (up to 8 h, Java gets half the machine's memory or the amount set in Settings → Engines, 4 to 31 GB, four slow passes skipped). The builder must decompile before the build is checked; addresses the built-in analysis can't place go to Ghidra, which makes a function on demand (for this run only) where its analysis has none, e.g. code reached only through tables of pointers. Without Ghidra, Deep mode uses the built-in engines | Live runs with Ghidra 12.1.4 on Windows (a shared analysis, and a decompile in seconds on a large program). Live runs with Ghidra 12.2 on Linux x86-64 (decompiles by name in position-independent programs and shared libraries, whose addresses Ghidra moves). The guided install of Ghidra and its Java is offered on Linux and macOS too, but has only been tried on Windows. Clean room and Deep mode: engine, AI, MCP, CLI and app tests; no live Clean room build run yet | | Auto mode | Settings → Safety, on by default. Builder actions go ahead without asking, except a fixed set that is never done unasked (deleting outside the project, system changes, publishing…); questions get "decide yourself"; asset spending stops at the budget. Each project can choose allow, ask or refuse per kind of action instead (`project_approvals_set`, the Approvals menu under the prompt box, or `/approvals publish ask`; read-only `gh … view`/`list` count as commands, not publishing) | Unit tests, UI tests; not tried in a live build |