mirror of
https://github.com/storytold/autocraft.git
synced 2026-11-21 15:00:54 +01:00
218 lines
14 KiB
TypeScript
218 lines
14 KiB
TypeScript
// Go-live gate: run on a COPY of the production database.
|
|
// node scripts/golive-gate.ts <db-file> [--decisions 300] [--keep]
|
|
// (a) plans every item with the current code and runs the liveness check (must find no stuck item);
|
|
// (b) replays a team-sized day on a scratch copy without GitHub or model calls: 12 maintainers and 1 operator answer open decisions in
|
|
// priority order through the dashboard path and the card path, with disputes and concurrent answers on one question; the Act step runs
|
|
// with a fake GitHub that checks the write invariants. (The merge step is not replayed.)
|
|
// Exit code 1 when an item is stuck or an invariant broke. Deterministic: nothing random, no clock in any choice.
|
|
import { copyFileSync, existsSync, rmSync } from "node:fs";
|
|
import type { DatabaseSync } from "node:sqlite";
|
|
import { runAct } from "../src/act/index.ts";
|
|
import { setCommentApproval } from "../src/act/approval.ts";
|
|
import { clearDispute, openDispute } from "../src/answers/hold.ts";
|
|
import { writeEffect } from "../src/act/index.ts";
|
|
import { doCard } from "../src/conversations/act.ts";
|
|
import { decisionStamp } from "../src/conversations/cards.ts";
|
|
import { ConversationError } from "../src/conversations/store.ts";
|
|
import { loadPlanSet } from "../src/core/plan-set.ts";
|
|
import { writePlanCache } from "../src/core/plan-cache.ts";
|
|
import { applyAnswer } from "../src/dash/api/decisions.ts";
|
|
import { HttpError } from "../src/dash/server/util.ts";
|
|
import { connect } from "../src/db.ts";
|
|
import { BACKLOG, getDecision, isBatch, optionsOf, type DecisionRow } from "../src/decisions.ts";
|
|
import { textsFor, textsHash } from "../src/dupes/texts.ts";
|
|
import { computeLiveness, printReport, refreshLiveness } from "../src/liveness.ts";
|
|
import type { Querier } from "../src/mirror.ts";
|
|
import { pipelineRepos } from "../src/pilot.ts";
|
|
import { MAINTAINER_WRITES_KEY, canSettleCached } from "../src/settlers.ts";
|
|
|
|
const args = process.argv.slice(2);
|
|
const src = args.find((a) => !a.startsWith("--") && args[args.indexOf(a) - 1] !== "--decisions");
|
|
const flag = (name: string, dflt: number): number => { const i = args.indexOf(name); return i >= 0 ? Number(args[i + 1]) : dflt; };
|
|
if (!src) { console.error("usage: golive-gate.ts <db-file> [--decisions N]"); process.exit(2); }
|
|
const WANT = flag("--decisions", 300);
|
|
const WRITES_BEFORE_LIMIT = 60;
|
|
|
|
const work = `${src}.gate`;
|
|
for (const ext of ["", "-wal", "-shm"]) rmSync(work + ext, { force: true });
|
|
for (const ext of ["", "-wal", "-shm"]) if (existsSync(src + ext)) copyFileSync(src + ext, work + ext);
|
|
const db: DatabaseSync = connect(work);
|
|
console.log(`gate copy: ${work}`);
|
|
|
|
const planPass = (): void => { writePlanCache(db, loadPlanSet(db)); };
|
|
|
|
// ---- (a) liveness on current data
|
|
planPass();
|
|
const before = computeLiveness(db);
|
|
console.log("\n== liveness before the replay ==");
|
|
printReport({ ...before, items: before.items.slice(0, 40) });
|
|
if (before.items.length > 40) console.log(` ... ${before.items.length - 40} more`);
|
|
|
|
// ---- (b) replay
|
|
const OPERATOR = "gate-op";
|
|
const users = [OPERATOR, ...Array.from({ length: 12 }, (_, i) => `gate-m${i + 1}`)];
|
|
for (const u of users) db.prepare("INSERT OR REPLACE INTO access (login, role) VALUES (?, ?)").run(u, u === OPERATOR ? "operator" : "maintainer");
|
|
db.prepare("INSERT OR REPLACE INTO settings (key, value) VALUES (?, '1')").run(MAINTAINER_WRITES_KEY);
|
|
|
|
const on = [...pipelineRepos(db)].sort();
|
|
for (const r of on) { try { setCommentApproval(db, OPERATOR, r, true); } catch { /* repo not tracked */ } }
|
|
|
|
// Fake GitHub: records writes and checks the invariants at the moment of each write.
|
|
type Mode = "card" | "act";
|
|
const ctx: { mode: Mode } = { mode: "act" };
|
|
const violations: string[] = [];
|
|
const writes: { kind: string; ref: string; mode: Mode }[] = [];
|
|
const refOf = (id: string): { repo: string; number: number } | null => { const m = /^(.+)#(\d+)$/.exec(id); return m ? { repo: m[1]!, number: Number(m[2]) } : null; };
|
|
function checkWrite(kind: string, id: string): void {
|
|
const r = refOf(id);
|
|
if (!r) return;
|
|
writes.push({ kind, ref: id, mode: ctx.mode });
|
|
const locked = !!db.prepare("SELECT 1 FROM decisions WHERE repo = ? AND number = ? AND exec_state = 'settling' AND answer IS NULL").get(r.repo, r.number);
|
|
const disputed = !!db.prepare("SELECT 1 FROM decisions WHERE repo = ? AND number = ? AND exec_state = 'disputed' AND answer IS NOT NULL AND applied_at IS NULL").get(r.repo, r.number);
|
|
const answered = !!db.prepare("SELECT 1 FROM decisions WHERE repo = ? AND number = ? AND answer IS NOT NULL AND exec_state IS NOT 'disputed'").get(r.repo, r.number);
|
|
if (!locked && disputed) violations.push(`write after dispute: ${kind} on ${id}`);
|
|
else if (!locked && !answered) violations.push(`write without lock or answer: ${kind} on ${id}`);
|
|
}
|
|
const querier = {
|
|
spent: 0, remaining: 5000,
|
|
async query(q: string, v: Record<string, unknown>) {
|
|
if (q.includes("collaborators")) return { repository: { collaborators: { edges: [{ permission: "WRITE", node: { login: String(v.login) } }] } } };
|
|
if (q.startsWith("mutation")) {
|
|
const id = String(v.id ?? "");
|
|
checkWrite(q.includes("addComment") ? "comment" : q.includes("close") ? "close" : "write", id);
|
|
return q.includes("addComment") ? { addComment: { commentEdge: { node: { url: `https://x.test/${id}` } } } } : {};
|
|
}
|
|
if (q.includes("issueOrPullRequest")) {
|
|
// GitHub agrees with the mirror: a PR keeps the head the database holds.
|
|
const it = db.prepare("SELECT kind, head_oid FROM items WHERE repo = ? AND number = ?").get(String(v.name), Number(v.n)) as { kind: string; head_oid: string | null } | undefined;
|
|
const id = `${v.name}#${v.n}`;
|
|
return { repository: { issueOrPullRequest: it?.kind === "pr" ? { __typename: "PullRequest", id, state: "OPEN", headRefOid: it.head_oid } : { __typename: "Issue", id, state: "OPEN" } } };
|
|
}
|
|
if (q.includes("viewer")) return { viewer: { login: "pipeline-bot", id: "U1" } };
|
|
return { repository: {} };
|
|
},
|
|
} as unknown as Querier;
|
|
const deps = { querier, now: () => new Date() };
|
|
|
|
// The day's decisions, highest priority first.
|
|
const LEVEL = `CASE WHEN p.score IS NULL THEN 0 ELSE p.score END`;
|
|
const open = db.prepare(`SELECT d.id FROM decisions d LEFT JOIN item_priority p ON p.repo = d.repo AND p.number = d.number
|
|
WHERE d.answer IS NULL AND d.number IS NOT NULL AND d.repo IN (${on.map(() => "?").join(",") || "''"}) AND d.id NOT LIKE '%~replaced-%'
|
|
ORDER BY ${LEVEL} DESC, d.asked_at, d.id LIMIT ?`).all(...on, WANT) as { id: string }[];
|
|
const pick = (d: DecisionRow): string => {
|
|
const keys = optionsOf(d).map((o) => o.key).filter((k) => k !== BACKLOG.key);
|
|
return d.recommended && keys.includes(d.recommended) ? d.recommended : keys[0] ?? BACKLOG.key;
|
|
};
|
|
|
|
const used = (db.prepare("SELECT COUNT(*) AS n FROM actions WHERE substr(created_at, 1, 10) = ?").get(new Date().toISOString().slice(0, 10)) as { n: number }).n;
|
|
db.prepare("INSERT OR REPLACE INTO stage_limits (step, per_day) VALUES ('act', ?)").run(used + WRITES_BEFORE_LIMIT);
|
|
|
|
const stat = { superseded: 0, operatorFallback: 0, dashboard: 0, card: 0, applied: 0, queuedLimit: 0, refusedLock: 0, conflicts: 0, disputes: 0, otherRefused: [] as string[] };
|
|
const isLock = (m: string) => /is applying their answer|waits for the daily GitHub write limit/.test(m);
|
|
const isTaken = (m: string) => /already answered|answered this a moment ago|no longer open|decision changed/.test(m);
|
|
const disputed: string[] = [];
|
|
|
|
function dashboard(d: DecisionRow, u: string, opt: string): void {
|
|
const body: Record<string, unknown> = { answer: opt };
|
|
if (d.kind === "duplicate" && d.id.includes(":dupegroup:")) { const t = textsFor(db, d.id, opt); if (t.length) body.textsHash = textsHash(t); }
|
|
try {
|
|
applyAnswer(db, d, u, body, "dashboard");
|
|
stat.applied++; stat.dashboard++;
|
|
} catch (e) {
|
|
const m = (e as Error).message;
|
|
if (e instanceof HttpError && isLock(m)) stat.refusedLock++;
|
|
else if (isTaken(m)) stat.conflicts++;
|
|
else stat.otherRefused.push(`${d.id}: ${m.slice(0, 100)}`);
|
|
}
|
|
}
|
|
|
|
for (const [i, row] of open.entries()) {
|
|
const d = getDecision(db, row.id)!;
|
|
// A maintainer answers only where the dashboard would let them settle; otherwise the operator does.
|
|
const want = users[i % users.length]!;
|
|
const u = want === OPERATOR || canSettleCached(db, want, "maintainer", [d.repo!]) ? want : OPERATOR;
|
|
if (u !== want) stat.operatorFallback++;
|
|
const other = users[(i + 5) % users.length]!;
|
|
const opt = pick(d);
|
|
const cardable = i % 3 === 0 && !isBatch(d) && !d.id.includes(":dupegroup:") && opt !== BACKLOG.key;
|
|
if (cardable) {
|
|
const params = { kind: "resolution", steps: [{ kind: "comment" }], settle: { kind: "answer", decision: d.id, option: opt, stamp: decisionStamp(db, d) } };
|
|
const card = Number(db.prepare("INSERT INTO conversation_cards (turn_id, repo, number, kind, params, text, description) VALUES (0, ?, ?, 'resolution', ?, 'Noted.', 'gate card')").run(d.repo, d.number, JSON.stringify(params)).lastInsertRowid);
|
|
let release!: () => void;
|
|
const gate = new Promise<boolean>((r) => { release = () => r(true); });
|
|
const concurrent = i % 9 === 0;
|
|
ctx.mode = "card";
|
|
const running = doCard(db, card, u, { ...deps, canWrite: () => (concurrent ? gate : Promise.resolve(true)), role: db.prepare("SELECT role FROM access WHERE login = ?").get(u) ? "maintainer" : "operator" } as never);
|
|
if (concurrent) {
|
|
// Let the press claim the decision first.
|
|
for (let k = 0; k < 400 && (db.prepare("SELECT state FROM conversation_cards WHERE id = ?").get(card) as { state: string }).state === "proposed"; k++) await new Promise((r) => setTimeout(r, 5));
|
|
// Another person answers the same question while the first press is in flight.
|
|
dashboard(getDecision(db, d.id)!, other, opt);
|
|
release();
|
|
}
|
|
try {
|
|
const r = await running;
|
|
stat.card++;
|
|
if (r.card.state === "done") stat.applied++;
|
|
else if (r.card.state === "queued") stat.queuedLimit++;
|
|
else if (r.card.state === "replaced" && concurrent) stat.superseded++;
|
|
else stat.otherRefused.push(`${d.id}: card ${r.card.state} ${r.card.error ?? ""} ${concurrent ? "(concurrent)" : ""} ${other}`.slice(0, 160));
|
|
} catch (e) {
|
|
const m = (e as Error).message;
|
|
if (e instanceof ConversationError && isLock(m)) stat.refusedLock++;
|
|
else if (isTaken(m)) stat.conflicts++;
|
|
else stat.otherRefused.push(`${d.id}: ${m.slice(0, 100)}`);
|
|
}
|
|
ctx.mode = "act";
|
|
} else {
|
|
dashboard(d, u, opt);
|
|
const now = getDecision(db, d.id)!;
|
|
if (i % 7 === 2 && now.answer && writeEffect(now.kind, now.answer) && openDispute(db, d.id, other, "gate dispute", new Date().toISOString())) { stat.disputes++; disputed.push(d.id); }
|
|
}
|
|
}
|
|
|
|
// Every second dispute is kept (cleared); the rest stay disputed through the Act step.
|
|
for (const [i, id] of disputed.entries()) if (i % 2 === 0) clearDispute(db, id);
|
|
|
|
// End of day: the limit is lifted and the Act step carries out queued presses and answered decisions, per repo, with the fake GitHub.
|
|
db.prepare("UPDATE stage_limits SET per_day = NULL WHERE step = 'act'").run();
|
|
ctx.mode = "act";
|
|
const actFailures: string[] = [];
|
|
for (const repo of on) {
|
|
try {
|
|
const s = await runAct(db, { repo, dryRun: false }, () => {}, deps as never);
|
|
for (const f of s.failed ?? []) actFailures.push(`${repo}: ${JSON.stringify(f).slice(0, 120)}`);
|
|
} catch (e) { actFailures.push(`${repo}: ${(e as Error).message.slice(0, 120)}`); }
|
|
}
|
|
|
|
// Invariants on the stored result.
|
|
const dupAnswers = db.prepare("SELECT target, COUNT(*) AS n FROM audit WHERE action = 'answer' AND login LIKE 'gate-%' GROUP BY target HAVING n > 1").all() as { target: string; n: number }[];
|
|
for (const r of dupAnswers) violations.push(`two answers on one question: ${r.target} (${r.n})`);
|
|
const stillQueued = (db.prepare("SELECT COUNT(*) AS n FROM conversation_cards WHERE state = 'queued'").get() as { n: number }).n;
|
|
const stillLocked = (db.prepare("SELECT COUNT(*) AS n FROM decisions WHERE exec_state = 'settling' AND answer IS NULL").get() as { n: number }).n;
|
|
if (stillLocked > 0 && stillQueued === 0) violations.push(`${stillLocked} decisions still locked with no card running or queued`);
|
|
const heldDisputed = (db.prepare("SELECT COUNT(*) AS n FROM decisions WHERE exec_state = 'disputed' AND answer IS NOT NULL AND applied_at IS NULL").get() as { n: number }).n;
|
|
|
|
// Plan again with the answers in and check liveness once more.
|
|
planPass();
|
|
const after = computeLiveness(db);
|
|
refreshLiveness(db, Date.now(), true);
|
|
|
|
console.log("\n== replay ==");
|
|
console.log(`decisions in the day: ${open.length}; users: ${users.length} (1 operator, 12 maintainers)`);
|
|
console.log(`answers by the operator because the maintainer may not settle that repo: ${stat.operatorFallback}`);
|
|
console.log(`answers applied: ${stat.applied} (dashboard ${stat.dashboard}, card ${stat.card - stat.queuedLimit - 0})`);
|
|
console.log(`queued behind the daily limit: ${stat.queuedLimit} (still queued after the Act step: ${stillQueued})`);
|
|
console.log(`refused by the lock (concurrent answer): ${stat.refusedLock}; answered by someone else first: ${stat.conflicts}; press superseded by the concurrent answer: ${stat.superseded}`);
|
|
console.log(`disputes opened: ${stat.disputes}; still disputed at the end: ${heldDisputed}`);
|
|
console.log(`fake GitHub writes: ${writes.length} (${writes.filter((w) => w.mode === "card").length} from cards)`);
|
|
if (stat.otherRefused.length) { console.log(`other refusals: ${stat.otherRefused.length}`); for (const m of stat.otherRefused.slice(0, 15)) console.log(` ${m}`); }
|
|
if (actFailures.length) { console.log(`act failures: ${actFailures.length}`); for (const m of actFailures.slice(0, 10)) console.log(` ${m}`); }
|
|
console.log(`invariants broken: ${violations.length}`);
|
|
for (const v of violations.slice(0, 20)) console.log(` ${v}`);
|
|
console.log("\n== liveness after the replay ==");
|
|
printReport({ ...after, items: after.items.slice(0, 40) });
|
|
if (after.items.length > 40) console.log(` ... ${after.items.length - 40} more`);
|
|
|
|
process.exit(before.stuck > 0 || after.stuck > 0 || violations.length > 0 ? 1 : 0);
|