mirror of
https://github.com/storytold/autocraft.git
synced 2026-11-20 22:41:03 +01:00
194 lines
13 KiB
JavaScript
194 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
|
// Runs the robustness scenarios (section 4 of the plan) against a scratch instance and prints the acceptance report.
|
|
// node scripts/loadtest/run.mjs <smoke|1|2|3|4|5|6|7|all> [--dir <work dir>] [--db <db to copy>] [--time-factor F] [--time-scale S]
|
|
// [--users N] [--parallel K] [--calls M] [--max-minutes N] [--sequential]
|
|
// `all` runs 1 to 5 side by side, then 6 and 7 each alone at their own scale. See README.md.
|
|
import { spawn } from "node:child_process";
|
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { collect, evaluate, evaluateLongPoll } from "./report.mjs";
|
|
import { MIX, Rig, sleep } from "./rig.mjs";
|
|
|
|
const argv = process.argv.slice(2);
|
|
const opt = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 ? argv[i + 1] : d; };
|
|
const flag = (n) => argv.includes(`--${n}`);
|
|
const which = argv.find((a) => !a.startsWith("--") && !argv[argv.indexOf(a) - 1]?.startsWith("--")) ?? "smoke";
|
|
|
|
// Every user-local time window (leases, heartbeats, cool-offs) and every scenario timing is multiplied by the scale
|
|
// (AUTOCRAFT_REMOTE_TIME_SCALE, test only); 1 runs the production windows. A scenario may name its own default scale; the
|
|
// driver's server modules read the scale once per process, so `all` runs such a scenario in a process of its own.
|
|
const SCALE_GIVEN = argv.includes("--time-scale");
|
|
const TIME_SCALE = Number(opt("time-scale", 0.1));
|
|
if (!(TIME_SCALE >= 0.01 && TIME_SCALE <= 1)) { console.error("--time-scale must be a number from 0.01 to 1"); process.exit(64); }
|
|
const S = (ms) => Math.max(1, Math.round(ms * TIME_SCALE));
|
|
|
|
const FAULTS = { rate502: 0.10, rateReset: 0.05, rate429: 0.02, retryAfter: 2, latMin: 0, latMax: 3000 };
|
|
|
|
/** Each scenario: defaults for the rig, what to do while the queue drains, and which acceptance lines apply. */
|
|
const SCENARIOS = {
|
|
smoke: { title: "2 helpers, 20 calls, faults on", users: 2, parallel: 2, calls: 20, timeFactor: 0.05, faults: FAULTS, accept: ["noLoss", "idle"], maxMin: 8 },
|
|
1: { title: "10% 502, 5% resets, 2% 429, 0-3 s latency", users: 6, parallel: 2, calls: 60, timeFactor: 0.1, faults: FAULTS, accept: ["noLoss", "idle"], maxMin: 12 },
|
|
2: { title: "dashboard restarted mid-call, then stopped for 5 minutes", users: 6, parallel: 2, calls: 50, timeFactor: 0.5, accept: ["noLoss", "noCooloff", "idle"], maxMin: 16,
|
|
async during(rig) { await sleep(S(60_000)); await rig.restartDashboard(); await sleep(S(60_000)); await rig.dashboardOutage(300 * TIME_SCALE); } },
|
|
3: { title: "a helper frozen with SIGSTOP for 3 minutes", users: 4, parallel: 2, calls: 40, timeFactor: 0.5, accept: ["noLoss", "noDouble", "idle"], maxMin: 12,
|
|
async during(rig) { await sleep(S(40_000)); rig.helpers[0].frozen = true; await rig.freezeHelper(0, 180 * TIME_SCALE); rig.helpers[0].frozen = false; } },
|
|
4: { title: "a helper killed with SIGKILL mid-call and restarted", users: 4, parallel: 2, calls: 40, timeFactor: 0.5, accept: ["noLoss", "idle"], maxMin: 12,
|
|
async during(rig) { await sleep(S(45_000)); rig.killHelper(0); await sleep(S(20_000)); rig.restartHelper(0); } },
|
|
// The failing helper runs 5 slots and the fast lane, so its first claims fail 5 calls in a row (the cool-off threshold)
|
|
// before the others drain the queue; one slot each would back off after every failure and see too few calls.
|
|
5: { title: "a helper whose harness always fails", users: 5, parallel: 2, calls: 60, timeFactor: 0.1, failing: [4], failingParallel: 5, accept: ["oneCooloff", "idle"], maxMin: 10 },
|
|
// Claims, heartbeats and status reports come at the scaled intervals, so the scale sets the dashboard's idle load: 0.5 is
|
|
// about twice production's. The harness delays are cut instead (time factor) to keep the run short, and the calls arrive
|
|
// over 25 s, as jobs queue them, rather than in one burst.
|
|
6: { title: "50 helpers x 4 slots against 300 mixed calls", users: 50, parallel: 4, calls: 300, rampS: 25, timeFactor: 0.1, timeScale: 0.5, accept: ["noLoss", "latency", "idle"], maxMin: 25 },
|
|
// Long-polling claims with protocol 4 claimers in this process (no harness: a lease is submitted at once), so the runners
|
|
// sit in held claims nearly all the time. Production-scale times: calls over 120 s, the restart at 60 s.
|
|
7: { title: "20 idle runners long-polling, random enqueues, a dashboard restart", users: 20, parallel: 1, calls: 60, spreadS: 120, restartAtS: 60, timeFactor: 1, timeScale: 0.25, longPoll: true, maxMin: 8 },
|
|
};
|
|
|
|
/** The time scale scenario `key` runs at: --time-scale when given, else the scenario's own, else the default. */
|
|
const scaleOf = (key) => (SCALE_GIVEN ? TIME_SCALE : SCENARIOS[key].timeScale ?? TIME_SCALE);
|
|
|
|
async function run(key) {
|
|
const sp = SCENARIOS[key];
|
|
if (sp.longPoll) return runLongPoll(key);
|
|
const scale = scaleOf(key);
|
|
const spec = { ...sp, name: `scenario-${key}`, users: Number(opt("users", sp.users)), parallel: Number(opt("parallel", sp.parallel)), calls: Number(opt("calls", sp.calls)), timeFactor: Number(opt("time-factor", sp.timeFactor)) * scale, maxMin: Number(opt("max-minutes", sp.maxMin * Math.max(scale, 0.3))) };
|
|
const dir = opt("dir", join(tmpdir(), `autocraft-loadtest-${Date.now()}`));
|
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
const rig = new Rig({ name: spec.name, dir, users: spec.users, parallel: spec.parallel, timeFactor: spec.timeFactor, timeScale: scale, dbFrom: opt("db"), faults: sp.faults });
|
|
try {
|
|
await rig.init();
|
|
for (let i = 0; i < spec.users; i++) {
|
|
const failing = (sp.failing ?? []).includes(i);
|
|
rig.startHelper(i, { failing, ...(failing && sp.failingParallel && { parallel: sp.failingParallel }) });
|
|
}
|
|
rig.note(`${spec.users} helpers started, ${spec.parallel} slots each`);
|
|
await sleep(Math.min(S(20_000) + 3000, 4000 + spec.users * 300));
|
|
await rig.enqueueMix(spec.calls, { rampMs: (sp.rampS ?? 0) * 1000 });
|
|
rig.note(`${spec.calls} calls queued${sp.rampS ? ` over ${sp.rampS} s` : ""} (harness delay factor ${spec.timeFactor}, time scale ${scale})`);
|
|
const during = sp.during ? sp.during(rig).catch((e) => rig.note(`scenario step failed: ${e.message}`)) : Promise.resolve();
|
|
const drained = await rig.waitDrained(spec.maxMin * 60_000);
|
|
await during;
|
|
if (sp.failing) await sleep(S(5000));
|
|
rig.note(drained ? "queue drained" : `not drained after ${spec.maxMin} min: ${rig.openCalls()} calls still open`);
|
|
await rig.collectDone();
|
|
rig.sample();
|
|
const m = collect(rig);
|
|
const { lines, checks } = evaluate(rig, spec, m);
|
|
writeFileSync(join(rig.dir, "report.json"), JSON.stringify({ spec: { name: spec.name, title: spec.title }, metrics: m, checks, timeline: rig.events }, null, 1));
|
|
return { key, lines, checks, dir: rig.dir };
|
|
} finally {
|
|
await rig.stop();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* One long-polling runner slot: claims with `wait: 25` until `lp.stop`, submits each lease at once, and records every claim
|
|
* (start, end, HTTP status, lease, reason) in `lp.claims` and each lease's time in `lp.leases`.
|
|
*/
|
|
async function longPollRunner(rig, u, lp) {
|
|
const base = `http://127.0.0.1:${rig.proxyPort}`;
|
|
const tiers = { top: "claude-opus-4-5", mid: "claude-sonnet-4-5", small: "claude-haiku-4-5" };
|
|
const post = (path, bearer, body) => fetch(`${base}${path}`, { method: "POST", headers: { authorization: `Bearer ${bearer}`, "content-type": "application/json" }, body: JSON.stringify(body), signal: lp.abort.signal });
|
|
for (let n = 1; !lp.stop; n++) {
|
|
const start = Date.now();
|
|
let status = 0, reply = null;
|
|
try {
|
|
const r = await post("/api/remote/claim", u.token, { requestId: `${u.login}-${n}-${Math.random().toString(36).slice(2, 8)}`, protocol: 4, wait: 25, helperVersion: "1.2.0", harness: "claude", harnessVersion: "2.1.0", tiers });
|
|
status = r.status;
|
|
reply = status === 200 ? await r.json() : (await r.arrayBuffer(), null);
|
|
} catch { /* reset or aborted */ }
|
|
if (lp.stop) break;
|
|
const end = Date.now();
|
|
lp.claims.push({ start, end, status, lease: reply?.lease?.id ?? null, reason: reply?.reason ?? null });
|
|
if (reply?.lease) {
|
|
lp.leases.set(reply.lease.id, end);
|
|
const tier = rig.db.prepare("SELECT tier FROM main.remote_calls WHERE id = ?").get(reply.lease.id)?.tier ?? "small";
|
|
try { await (await post("/api/remote/submit", reply.lease.secret, { ok: true, text: "ok", model: tiers[tier], usage: { inputTokens: 1, outputTokens: 1 }, durationMs: 1 })).arrayBuffer(); } catch { /* the job emulator requeues a lost lease */ }
|
|
continue;
|
|
}
|
|
await sleep(status === 200 ? reply.retryAfter * 1000 : rig.sc(1000));
|
|
}
|
|
}
|
|
|
|
async function runLongPoll(key) {
|
|
const sp = SCENARIOS[key];
|
|
const scale = scaleOf(key);
|
|
const Sc = (ms) => Math.max(1, Math.round(ms * scale));
|
|
const spec = { ...sp, name: `scenario-${key}`, users: Number(opt("users", sp.users)), calls: Number(opt("calls", sp.calls)) };
|
|
const dir = opt("dir", join(tmpdir(), `autocraft-loadtest-${Date.now()}`));
|
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
const rig = new Rig({ name: spec.name, dir, users: spec.users, parallel: 1, timeFactor: 1, timeScale: scale, dbFrom: opt("db") });
|
|
const lp = { claims: [], leases: new Map(), stop: false, abort: new AbortController(), t0: 0, end: 0, stopAt: 0, backAt: 0 };
|
|
try {
|
|
await rig.init();
|
|
lp.t0 = Date.now();
|
|
const runners = rig.users.map((u) => longPollRunner(rig, u, lp));
|
|
rig.note(`${spec.users} long-polling runners started`);
|
|
await sleep(Sc(5000));
|
|
const during = (async () => {
|
|
await sleep(Sc(sp.restartAtS * 1000));
|
|
lp.stopAt = Date.now();
|
|
await rig.restartDashboard();
|
|
lp.backAt = Date.now();
|
|
})().catch((e) => rig.note(`scenario step failed: ${e.message}`));
|
|
// Random arrivals: exponential gaps around the mean, small and gating calls as jobs queue them between larger ones.
|
|
const mean = Sc(sp.spreadS * 1000) / spec.calls;
|
|
for (let i = 0; i < spec.calls; i++) {
|
|
await sleep(-Math.log(1 - Math.random()) * mean);
|
|
await rig.enqueue(MIX[i % MIX.length]);
|
|
if (i === 0) rig.startJobEmulator();
|
|
}
|
|
rig.note(`${spec.calls} calls queued over about ${sp.spreadS} production seconds (time scale ${scale})`);
|
|
await during;
|
|
const drained = await rig.waitDrained(Number(opt("max-minutes", sp.maxMin)) * 60_000);
|
|
rig.note(drained ? "queue drained" : `not drained: ${rig.openCalls()} calls still open`);
|
|
// One more hold, so the runners' empty replies at their deadlines are counted.
|
|
await sleep(Sc(30_000));
|
|
lp.end = Date.now();
|
|
lp.stop = true;
|
|
lp.abort.abort();
|
|
await Promise.all(runners);
|
|
const m = collect(rig);
|
|
const { lines, checks } = evaluate(rig, spec, m);
|
|
const lpOut = evaluateLongPoll(rig, spec, lp);
|
|
lines.push(...lpOut.lines);
|
|
checks.push(...lpOut.checks);
|
|
writeFileSync(join(rig.dir, "report.json"), JSON.stringify({ spec: { name: spec.name, title: spec.title }, metrics: { ...m, longPoll: lpOut.metrics }, checks, timeline: rig.events }, null, 1));
|
|
return { key, lines, checks, dir: rig.dir };
|
|
} finally {
|
|
lp.stop = true;
|
|
lp.abort.abort();
|
|
await rig.stop();
|
|
}
|
|
}
|
|
|
|
/** Runs scenario `key` in a child process (its own time scale) and returns its result. */
|
|
async function runApart(key) {
|
|
const out = join(opt("dir", tmpdir()), `result-${key}-${process.pid}.json`);
|
|
const args = argv.filter((a, i) => a !== which || argv[i - 1]?.startsWith("--"));
|
|
const code = await new Promise((r) => spawn(process.execPath, [process.argv[1], key, ...args, "--result", out], { stdio: ["ignore", "inherit", "inherit"] }).on("close", r));
|
|
try { return JSON.parse(readFileSync(out, "utf8")); } catch { return { key, lines: [`== scenario-${key}: the run ended without a result (exit ${code})`], checks: [{ ok: false, name: "the scenario ran", detail: `exit ${code}` }], dir: "" }; }
|
|
}
|
|
|
|
const started = Date.now();
|
|
const keys = which === "all" ? ["1", "2", "3", "4", "5", "6", "7"] : [which];
|
|
if (!keys.every((k) => SCENARIOS[k])) { console.error(`unknown scenario ${which}; use smoke, 1-7 or all`); process.exit(64); }
|
|
const results = [];
|
|
// Scenarios at another scale than the rest run apart; the rest share this process.
|
|
const apart = which === "all" ? keys.filter((k) => scaleOf(k) !== TIME_SCALE) : [];
|
|
const here = keys.filter((k) => !apart.includes(k));
|
|
if (which === "all" && !flag("sequential")) results.push(...await Promise.all(here.filter((k) => k !== "6" && k !== "7").map((k) => run(k))), ...(here.includes("6") ? [await run("6")] : []), ...(here.includes("7") ? [await run("7")] : []));
|
|
else for (const k of here) results.push(await run(k));
|
|
for (const k of apart) results.push(await runApart(k));
|
|
results.sort((a, b) => a.key.localeCompare(b.key));
|
|
if (opt("result")) { writeFileSync(opt("result"), JSON.stringify(results[0])); process.exit(0); }
|
|
console.log("\n==================== acceptance ====================");
|
|
for (const r of results) console.log(`${r.lines.join("\n")}\n(files: ${r.dir})\n`);
|
|
const fails = results.flatMap((r) => r.checks.filter((c) => !c.ok).map((c) => `scenario ${r.key}: ${c.name}`));
|
|
console.log(fails.length ? `FAILED ${fails.length}:\n ${fails.join("\n ")}` : "ALL PASSED");
|
|
console.log(`run time ${Math.round((Date.now() - started) / 1000)} s`);
|
|
process.exit(fails.length ? 1 : 0);
|