mirror of
https://github.com/storytold/autocraft.git
synced 2026-11-20 06:21:09 +01:00
109 lines
8.3 KiB
JavaScript
109 lines
8.3 KiB
JavaScript
#!/usr/bin/env node
|
|
// Runs the robustness scenarios (section 4 of the plan) against a scratch instance and prints the acceptance report.
|
|
// node scripts/loadtest/run.mjs <smoke|1|2|3|4|5|6|all> [--dir <work dir>] [--db <db to copy>] [--time-factor F] [--time-scale S]
|
|
// [--users N] [--parallel K] [--calls M] [--max-minutes N] [--sequential]
|
|
// `all` runs 1 to 5 side by side, then 6 alone (its latency numbers need the machine to itself). See README.md.
|
|
import { spawn } from "node:child_process";
|
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { collect, evaluate } from "./report.mjs";
|
|
import { Rig, sleep } from "./rig.mjs";
|
|
|
|
const argv = process.argv.slice(2);
|
|
const opt = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 ? argv[i + 1] : d; };
|
|
const flag = (n) => argv.includes(`--${n}`);
|
|
const which = argv.find((a) => !a.startsWith("--") && !argv[argv.indexOf(a) - 1]?.startsWith("--")) ?? "smoke";
|
|
|
|
// Every user-local time window (leases, heartbeats, cool-offs) and every scenario timing is multiplied by the scale
|
|
// (AUTOCRAFT_REMOTE_TIME_SCALE, test only); 1 runs the production windows. A scenario may name its own default scale; the
|
|
// driver's server modules read the scale once per process, so `all` runs such a scenario in a process of its own.
|
|
const SCALE_GIVEN = argv.includes("--time-scale");
|
|
const TIME_SCALE = Number(opt("time-scale", 0.1));
|
|
if (!(TIME_SCALE >= 0.01 && TIME_SCALE <= 1)) { console.error("--time-scale must be a number from 0.01 to 1"); process.exit(64); }
|
|
const S = (ms) => Math.max(1, Math.round(ms * TIME_SCALE));
|
|
|
|
const FAULTS = { rate502: 0.10, rateReset: 0.05, rate429: 0.02, retryAfter: 2, latMin: 0, latMax: 3000 };
|
|
|
|
/** Each scenario: defaults for the rig, what to do while the queue drains, and which acceptance lines apply. */
|
|
const SCENARIOS = {
|
|
smoke: { title: "2 helpers, 20 calls, faults on", users: 2, parallel: 2, calls: 20, timeFactor: 0.05, faults: FAULTS, accept: ["noLoss", "idle"], maxMin: 8 },
|
|
1: { title: "10% 502, 5% resets, 2% 429, 0-3 s latency", users: 6, parallel: 2, calls: 60, timeFactor: 0.1, faults: FAULTS, accept: ["noLoss", "idle"], maxMin: 12 },
|
|
2: { title: "dashboard restarted mid-call, then stopped for 5 minutes", users: 6, parallel: 2, calls: 50, timeFactor: 0.5, accept: ["noLoss", "noCooloff", "idle"], maxMin: 16,
|
|
async during(rig) { await sleep(S(60_000)); await rig.restartDashboard(); await sleep(S(60_000)); await rig.dashboardOutage(300 * TIME_SCALE); } },
|
|
3: { title: "a helper frozen with SIGSTOP for 3 minutes", users: 4, parallel: 2, calls: 40, timeFactor: 0.5, accept: ["noLoss", "noDouble", "idle"], maxMin: 12,
|
|
async during(rig) { await sleep(S(40_000)); rig.helpers[0].frozen = true; await rig.freezeHelper(0, 180 * TIME_SCALE); rig.helpers[0].frozen = false; } },
|
|
4: { title: "a helper killed with SIGKILL mid-call and restarted", users: 4, parallel: 2, calls: 40, timeFactor: 0.5, accept: ["noLoss", "idle"], maxMin: 12,
|
|
async during(rig) { await sleep(S(45_000)); rig.killHelper(0); await sleep(S(20_000)); rig.restartHelper(0); } },
|
|
// The failing helper runs 5 slots and the fast lane, so its first claims fail 5 calls in a row (the cool-off threshold)
|
|
// before the others drain the queue; one slot each would back off after every failure and see too few calls.
|
|
5: { title: "a helper whose harness always fails", users: 5, parallel: 2, calls: 60, timeFactor: 0.1, failing: [4], failingParallel: 5, accept: ["oneCooloff", "idle"], maxMin: 10 },
|
|
// Claims, heartbeats and status reports come at the scaled intervals, so the scale sets the dashboard's idle load: 0.5 is
|
|
// about twice production's. The harness delays are cut instead (time factor) to keep the run short, and the calls arrive
|
|
// over 25 s, as jobs queue them, rather than in one burst.
|
|
6: { title: "50 helpers x 4 slots against 300 mixed calls", users: 50, parallel: 4, calls: 300, rampS: 25, timeFactor: 0.1, timeScale: 0.5, accept: ["noLoss", "latency", "idle"], maxMin: 25 },
|
|
};
|
|
|
|
/** The time scale scenario `key` runs at: --time-scale when given, else the scenario's own, else the default. */
|
|
const scaleOf = (key) => (SCALE_GIVEN ? TIME_SCALE : SCENARIOS[key].timeScale ?? TIME_SCALE);
|
|
|
|
async function run(key) {
|
|
const sp = SCENARIOS[key];
|
|
const scale = scaleOf(key);
|
|
const spec = { ...sp, name: `scenario-${key}`, users: Number(opt("users", sp.users)), parallel: Number(opt("parallel", sp.parallel)), calls: Number(opt("calls", sp.calls)), timeFactor: Number(opt("time-factor", sp.timeFactor)) * scale, maxMin: Number(opt("max-minutes", sp.maxMin * Math.max(scale, 0.3))) };
|
|
const dir = opt("dir", join(tmpdir(), `autocraft-loadtest-${Date.now()}`));
|
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
const rig = new Rig({ name: spec.name, dir, users: spec.users, parallel: spec.parallel, timeFactor: spec.timeFactor, timeScale: scale, dbFrom: opt("db"), faults: sp.faults });
|
|
try {
|
|
await rig.init();
|
|
for (let i = 0; i < spec.users; i++) {
|
|
const failing = (sp.failing ?? []).includes(i);
|
|
rig.startHelper(i, { failing, ...(failing && sp.failingParallel && { parallel: sp.failingParallel }) });
|
|
}
|
|
rig.note(`${spec.users} helpers started, ${spec.parallel} slots each`);
|
|
await sleep(Math.min(S(20_000) + 3000, 4000 + spec.users * 300));
|
|
await rig.enqueueMix(spec.calls, { rampMs: (sp.rampS ?? 0) * 1000 });
|
|
rig.note(`${spec.calls} calls queued${sp.rampS ? ` over ${sp.rampS} s` : ""} (harness delay factor ${spec.timeFactor}, time scale ${scale})`);
|
|
const during = sp.during ? sp.during(rig).catch((e) => rig.note(`scenario step failed: ${e.message}`)) : Promise.resolve();
|
|
const drained = await rig.waitDrained(spec.maxMin * 60_000);
|
|
await during;
|
|
if (sp.failing) await sleep(S(5000));
|
|
rig.note(drained ? "queue drained" : `not drained after ${spec.maxMin} min: ${rig.openCalls()} calls still open`);
|
|
await rig.collectDone();
|
|
rig.sample();
|
|
const m = collect(rig);
|
|
const { lines, checks } = evaluate(rig, spec, m);
|
|
writeFileSync(join(rig.dir, "report.json"), JSON.stringify({ spec: { name: spec.name, title: spec.title }, metrics: m, checks, timeline: rig.events }, null, 1));
|
|
return { key, lines, checks, dir: rig.dir };
|
|
} finally {
|
|
await rig.stop();
|
|
}
|
|
}
|
|
|
|
/** Runs scenario `key` in a child process (its own time scale) and returns its result. */
|
|
async function runApart(key) {
|
|
const out = join(opt("dir", tmpdir()), `result-${key}-${process.pid}.json`);
|
|
const args = argv.filter((a, i) => a !== which || argv[i - 1]?.startsWith("--"));
|
|
const code = await new Promise((r) => spawn(process.execPath, [process.argv[1], key, ...args, "--result", out], { stdio: ["ignore", "inherit", "inherit"] }).on("close", r));
|
|
try { return JSON.parse(readFileSync(out, "utf8")); } catch { return { key, lines: [`== scenario-${key}: the run ended without a result (exit ${code})`], checks: [{ ok: false, name: "the scenario ran", detail: `exit ${code}` }], dir: "" }; }
|
|
}
|
|
|
|
const started = Date.now();
|
|
const keys = which === "all" ? ["1", "2", "3", "4", "5", "6"] : [which];
|
|
if (!keys.every((k) => SCENARIOS[k])) { console.error(`unknown scenario ${which}; use smoke, 1-6 or all`); process.exit(64); }
|
|
const results = [];
|
|
// Scenarios at another scale than the rest run apart; the rest share this process.
|
|
const apart = which === "all" ? keys.filter((k) => scaleOf(k) !== TIME_SCALE) : [];
|
|
const here = keys.filter((k) => !apart.includes(k));
|
|
if (which === "all" && !flag("sequential")) results.push(...await Promise.all(here.filter((k) => k !== "6").map((k) => run(k))), ...(here.includes("6") ? [await run("6")] : []));
|
|
else for (const k of here) results.push(await run(k));
|
|
for (const k of apart) results.push(await runApart(k));
|
|
results.sort((a, b) => a.key.localeCompare(b.key));
|
|
if (opt("result")) { writeFileSync(opt("result"), JSON.stringify(results[0])); process.exit(0); }
|
|
console.log("\n==================== acceptance ====================");
|
|
for (const r of results) console.log(`${r.lines.join("\n")}\n(files: ${r.dir})\n`);
|
|
const fails = results.flatMap((r) => r.checks.filter((c) => !c.ok).map((c) => `scenario ${r.key}: ${c.name}`));
|
|
console.log(fails.length ? `FAILED ${fails.length}:\n ${fails.join("\n ")}` : "ALL PASSED");
|
|
console.log(`run time ${Math.round((Date.now() - started) / 1000)} s`);
|
|
process.exit(fails.length ? 1 : 0);
|