Review fixes: gated proposals reserve their step, batch merge and unassign respect PR leases, fallback proposals per repo, lane gate by capacity

This commit is contained in:
aiongg committed 2026-11-07 00:31:40 +00:00
1 parent 40e8595cb1
commit 7f92251ea4
6 files changed
+73 -33

No files matched your search

+1 -1
View File
@@ -29,7 +29,7 @@ export async function nextPage(db: DatabaseSync, visibleNames: string[], opts: {
const kept = stored[lane.name]?.items ?? [];
const items: NextItem[] = kept.filter((i) => visible.has(i.repo)).map((i) => ({ ...i, ...shown(i.repo, i.number), why: why(i.repo, i.number, i.action) }));
const total = (stored[lane.name]?.total ?? 0) - (kept.length - items.length);
const gate = laneGate(db, lane, ms);
const gate = laneGate(db, lane, stepDefs, ms);
const open = items.filter((i) => !i.blocked);
// Every item blocked by a repo approval: the step cannot do its work although it could start.
const refused = !open.length && items.length ? items[0]!.blocked : null;
+31 -18
View File
@@ -37,6 +37,7 @@ import { PIPELINE_LOGIN_KEY, buildQueue, type QueueEntry } from "./queue.ts";
import { planned } from "../core/select.ts";
import { envVar } from "../env.ts";
import { jobItem } from "../runner/job-items.ts";
import { leaseItem } from "../runner/leases.ts";
import { ownerOf, slugOf } from "../owners.ts";
const ACTOR = "batch";
@@ -351,25 +352,37 @@ export async function runBatches(db: DatabaseSync, params: BatchParams, log: (li
// Holds and controls may have changed during the calls above: look again right before merging.
if (getBatch(db, b.id).state !== "open") { log(` ${b.branch}: ended meanwhile; not merged`); return; }
if (members.some((m) => mergeHeld(db, repo, m.number))) { log(` ${b.branch}: a PR in the batch was held meanwhile; not merged`); s.waiting.push(b.branch); return; }
let after: Awaited<ReturnType<typeof refreshChecks>>[number] | undefined;
// Every member must still hold at its batched head, and the batch PR at the batch tip.
const g = await guardBatchMerge(db, { repo, number: b.pr_number!, head: b.commit_sha }, members.map((m) => ({ number: m.number, head: m.head_oid })), by, liveRead);
if ("refused" in g) { log(` ${b.branch}: not merged: ${g.refused}`); s.waiting.push(b.branch); return; }
const r = await mergeViaCommand(w, d.run, d.mergeCommand, g, { method: "merge", extra: allowFailedArgs(db, repo, d.mergeCommand) }, { run: runId, head: b.commit_sha, batch: b.branch }, async () => {
after = (await refreshChecks(db, d.querier, repo, [b.pr_number!]))[0];
return after?.state === "MERGED";
});
if (r.code !== 0) {
// The merge may have gone through although the command failed afterwards.
if (after?.state === "MERGED") return recordLanded(b, members, q);
const why = oneLine(r.stderr) || oneLine(r.stdout) || `exit code ${r.code}`;
updateBatch(db, b.id, at(), { summary: `${d.mergeCommand}: ${why}`.slice(0, 400) });
log(` ${b.branch}: refused: ${why}`);
s.waiting.push(b.branch);
return;
// A fix-up or CI fix pushing to a member would move its head under the merge: hold the members' leases until it ends.
const leases: (() => void)[] = [];
for (const m of members) {
const release = leaseItem(db, `pr:${repo}#${m.number}`, log);
if (!release) { for (const f of leases) f(); log(` ${b.branch}: a PR in the batch is being worked; not merged this pass`); s.waiting.push(b.branch); return; }
leases.push(release);
}
try {
let after: Awaited<ReturnType<typeof refreshChecks>>[number] | undefined;
// Every member must still hold at its batched head, and the batch PR at the batch tip.
const g = await guardBatchMerge(db, { repo, number: b.pr_number!, head: b.commit_sha }, members.map((m) => ({ number: m.number, head: m.head_oid })), by, liveRead);
if ("refused" in g) { log(` ${b.branch}: not merged: ${g.refused}`); s.waiting.push(b.branch); return; }
const r = await mergeViaCommand(w, d.run, d.mergeCommand, g, { method: "merge", extra: allowFailedArgs(db, repo, d.mergeCommand) }, { run: runId, head: b.commit_sha, batch: b.branch }, async () => {
after = (await refreshChecks(db, d.querier, repo, [b.pr_number!]))[0];
return after?.state === "MERGED";
});
if (r.code !== 0) {
// The merge may have gone through although the command failed afterwards.
if (after?.state === "MERGED") return recordLanded(b, members, q);
const why = oneLine(r.stderr) || oneLine(r.stdout) || `exit code ${r.code}`;
updateBatch(db, b.id, at(), { summary: `${d.mergeCommand}: ${why}`.slice(0, 400) });
log(` ${b.branch}: refused: ${why}`);
s.waiting.push(b.branch);
return;
}
// The merge command prints `<repo>#<pr> MERGED <merge commit>`.
recordLanded(b, members, q, /\bMERGED ([0-9a-f]{7,40})\b/.exec(r.stdout)?.[1] ?? null);
} finally {
for (const f of leases) f();
}
// The merge command prints `<repo>#<pr> MERGED <merge commit>`.
recordLanded(b, members, q, /\bMERGED ([0-9a-f]{7,40})\b/.exec(r.stdout)?.[1] ?? null);
}
async function advance(id: number): Promise<void> {
+14 -7
View File
@@ -47,7 +47,7 @@ import { claudePauseReason } from "../agents/accounts.ts";
import { tripReasonOfStep } from "../push-outcomes.ts";
import { STEP_STAGE } from "../core/push-outcome.ts";
import { jobBlock, stepParallel, type Active } from "./concurrency.ts";
import { knownStep, loadSteps, modelsOf, usesClaude, type Params, type StepDef } from "./steps.ts";
import { knownStep, loadSteps, STEPS, modelsOf, usesClaude, type Params, type StepDef } from "./steps.ts";
import { pilotItems, scopeRepos } from "../pilot.ts";
import { JUDGEMENT_MODELS, judgementModel } from "../decide/judgement.ts";
import { FAST_FLOOR, fastFirst, fastScoreOf } from "../fasttrack.ts";
@@ -782,7 +782,7 @@ export type Detected = {
timing?: { loadMs: number; lanesMs: number; cacheMs: number; written: number; deleted: number };
};
/** The fewest proposals one lane hands to apply, in case the first ones are refused at enqueue; the proposals of a lane are for different repos. */
/** The fewest proposals one lane hands to apply: one per repo, in order, so that when the first is refused at enqueue (or the step is full at step_parallel) the next repo's goes in. */
const PROPOSALS_PER_LANE = 3;
const nextUtcMidnight = (ms: number): number => Date.parse(dayStart(ms)) + 86_400_000;
@@ -924,6 +924,9 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op
const lanes = opts.lanes ?? LANES, log = opts.log ?? (() => {});
const queued: number[] = [];
const done = new Set<string>();
// Jobs queued in this pass and the steps their proposals gate: they hold a repo of the step, so with step_parallel 1 no other repo takes it.
const reserved: Active[] = [];
const asActive = (step: string, params: Params, state: string): Active => ({ id: 0, step, state, repo: typeof params.repo === "string" ? params.repo : null, number: typeof params.number === "number" ? params.number : null });
const steps = await loadSteps();
try { applyUsedAnswers(db); } catch (e) { log(`autopilot: answers: ${(e as Error).message}`); }
// Marks the next planner pass wants: when a wait ends, and why stages are not starting.
@@ -935,7 +938,8 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op
if (done.has(slot)) continue;
try {
// The detection ran a moment ago: a job of the same step and repo may have been queued since.
if (jobBlock(db, steps, { step: p.step, repo: typeof p.params.repo === "string" ? p.params.repo : null, number: typeof p.params.number === "number" ? p.params.number : null })) continue;
if (jobBlock(db, steps, { step: p.step, repo: typeof p.params.repo === "string" ? p.params.repo : null, number: typeof p.params.number === "number" ? p.params.number : null }, { reserved })) continue;
if (p.gate && jobBlock(db, steps, { step: p.gate.step, repo: typeof p.gate.params.repo === "string" ? p.gate.params.repo : null }, { reserved })) continue;
// The limit is what is left now, so two proposals of one step cannot both carry the whole remainder.
const job = await enqueue(db, p.step, p.params, AUTOPILOT_USER, (step, params) => {
if (typeof params.limit !== "number") return params;
@@ -943,6 +947,8 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op
return lim ? { ...params, ...lim } : null;
});
queued.push(job.id);
reserved.push(asActive(p.step, p.params, "queued"));
if (p.gate) reserved.push(asActive(p.gate.step, p.gate.params, "gated"));
done.add(slot);
if (p.fp !== undefined) setSetting(db, fpKey(p.step, typeof p.params.repo === "string" ? p.params.repo : null), JSON.stringify({ job: job.id, fp: p.fp } satisfies FpRecord));
const c: Candidate = { step: p.step, params: Object.fromEntries(Object.entries(p.params).filter(([k]) => k !== "limit")) };
@@ -1100,7 +1106,7 @@ export function heldLookup(db: DatabaseSync): (repo: string, number: number, dec
const laneHold = (name: string): Held | null => {
if (!gateOf.has(name)) {
const lane = LANES.find((l) => l.name === name);
const g = lane ? laneGate(db, lane) : null;
const g = lane ? laneGate(db, lane, new Map(STEPS.map((x) => [x.name, x]))) : null;
gateOf.set(name, g && !g.canRun && g.kind !== "running" && g.why ? { reason: plain(g.why), approval: false } : null);
}
return gateOf.get(name)!;
@@ -1124,13 +1130,14 @@ export function heldLookup(db: DatabaseSync): (repo: string, number: number, dec
export type LaneGate = { canRun: boolean; kind: "ok" | "running" | "off" | "breaker" | "limit" | "account" | "backoff"; why: string | null };
/** Whether a job of the lane can start now, from the same checks the planner's gate() makes. */
export function laneGate(db: DatabaseSync, lane: Pick<Lane, "steps">, ms: number = Date.now()): LaneGate {
export function laneGate(db: DatabaseSync, lane: Pick<Lane, "steps">, allSteps: Map<string, StepDef>, ms: number = Date.now()): LaneGate {
if (!isEnabled(db)) return { canRun: false, kind: "off", why: isDraining(db) ? "The pipeline is draining" : "The pipeline is off" };
const steps = lane.steps ?? [];
let blocked: LaneGate | null = null;
for (const s of steps) {
const j = db.prepare("SELECT id, state FROM jobs WHERE step = ? AND state IN ('queued', 'running') ORDER BY state = 'running' DESC LIMIT 1").get(s) as { id: number; state: string } | undefined;
if (j) { blocked ??= { canRun: false, kind: "running", why: `Job ${j.id} of ${s} is ${j.state}` }; continue; }
// Blocked only when no repo can start: the step already works its most repos.
const full = jobBlock(db, allSteps, { step: s, repo: null }, { precheck: true });
if (full) { blocked ??= { canRun: false, kind: "running", why: full.replace(/ \(step_parallel\)$/, "") }; continue; }
const why = pauseReason(db, s, ms);
if (!why) return { canRun: true, kind: "ok", why: null };
const kind = /^tripped/.test(why) ? "breaker" : /daily limit/.test(why) ? "limit" : /^failed/.test(why) ? "backoff" : "account";
+7 -7
View File
@@ -52,25 +52,25 @@ export type Want = { step: string; repo: string | null; number?: number | null }
/**
* Why the job cannot be queued or started now, or null. `states` are the jobs that count: queued and running for
* queueing, running alone for starting. `extra` are jobs about to be queued. `mutex` also checks StepDef.exclusive
* queueing, running alone for starting. `extra` are proposals of the same pass: they only count as the same work (a duplicate), so a fallback proposal for another repo can still be made. `reserved` are jobs this pass already queued or gated: they count for the same work and for the repo capacity. `mutex` also checks StepDef.exclusive
* against running jobs. With `precheck` (a planner check before any repo is known) only the repo capacity is checked.
*/
export function jobBlock(db: DatabaseSync, steps: Map<string, StepDef>, want: Want, o: { states?: ("queued" | "running")[]; extra?: Active[]; mutex?: boolean; precheck?: boolean } = {}): string | null {
const jobs = [...activeJobs(db, o.states), ...(o.extra ?? [])];
const same = jobs.filter((j) => j.step === want.step);
export function jobBlock(db: DatabaseSync, steps: Map<string, StepDef>, want: Want, o: { states?: ("queued" | "running")[]; extra?: Active[]; reserved?: Active[]; mutex?: boolean; precheck?: boolean } = {}): string | null {
const held = [...activeJobs(db, o.states), ...(o.reserved ?? [])].filter((j) => j.step === want.step);
const same = [...held, ...(o.extra ?? []).filter((j) => j.step === want.step)];
if (!o.precheck) {
const clash = same.find((j) => sameWork(want.step, j, { repo: want.repo, number: want.number ?? null }));
if (clash) return `job ${clash.id} of ${want.step} is ${clash.state}`;
}
const cap = stepParallel(db);
if (cap !== Infinity) {
const repos = new Set(same.map((j) => j.repo ?? ""));
const repos = new Set(held.map((j) => j.repo ?? ""));
if (repos.size >= cap && (o.precheck || !repos.has(want.repo ?? ""))) return `${want.step} is working ${repos.size} repo${repos.size === 1 ? "" : "s"}, the most at once (step_parallel)`;
}
const mutex = steps.get(want.step)?.exclusive;
if (o.mutex && mutex) {
const held = activeJobs(db, ["running"]).find((j) => steps.get(j.step)?.exclusive === mutex);
if (held) return `job ${held.id} (${held.step}) is running with the same mutex (${mutex})`;
const other = activeJobs(db, ["running"]).find((j) => steps.get(j.step)?.exclusive === mutex);
if (other) return `job ${other.id} (${other.step}) is running with the same mutex (${mutex})`;
}
return null;
}
+6
View File
@@ -12,6 +12,7 @@ import type { Writer } from "../github/write.ts";
import { viewerOf } from "../merge/index.ts";
import { Client, type Querier } from "../mirror.ts";
import { jobItem } from "../runner/job-items.ts";
import { leaseItem } from "../runner/leases.ts";
import { ownerOf } from "../owners.ts";
export const ACTOR = "unassign";
@@ -40,12 +41,17 @@ export async function runUnassign(db: DatabaseSync, p: UnassignParams): Promise<
db.prepare("INSERT INTO main.settings (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value").run(PIPELINE_LOGIN_KEY, me.login);
for (const d of due) {
const ref = `${p.repo}#${d.number}`;
// A running job works the item (it may have just claimed it): leave the claim alone.
const release = leaseItem(db, `${d.pr ? "pr" : "issue"}:${ref}`, log);
if (!release) continue;
try {
const info = await (d.pr ? pullRequestInfo(q, ownerOf(p.repo), p.repo, d.number) : issueInfo(q, ownerOf(p.repo), p.repo, d.number));
if (info && await releaseItem(w, g, q, p.repo, d.number, info, me, { run: p.runId })) { out.unassigned.push(ref); jobItem(db, p.repo, d.number, `unassigned from ${me.login}`, log); }
} catch (e) {
out.failed.push(ref);
jobItem(db, p.repo, d.number, `could not unassign: ${oneLine(String((e as Error).message ?? e))}`, log);
} finally {
release();
}
}
return out;
+14
View File
@@ -62,3 +62,17 @@ test("at the process cap a queued security-class job starts before an older norm
await settle(r, [busy.id, normal.id, urgent.id]);
assert.ok(getJob(db, urgent.id).startedAt! <= getJob(db, normal.id).startedAt!, "the security job started first");
});
test("a proposal reserves the step it is gated on: at step_parallel 1 another repo's job of that step waits", async () => {
const { applyProposals, detectAutopilot } = await import("../src/runner/autopilot.ts");
db.exec("DELETE FROM jobs");
setStepParallel(db, 1, "test");
const lanes = [
{ name: "first", steps: ["test-repo"], detect: () => [{ step: "test-repo", params: { repo: "a" }, gate: { step: "test-urgent", params: { repo: "a" } } }] },
{ name: "second", steps: ["test-urgent"], detect: () => [{ step: "test-urgent", params: { repo: "b" } }] },
];
const found = await detectAutopilot(db, { lanes });
const ids = await applyProposals(db, found.proposals, { lanes });
assert.deepEqual((db.prepare("SELECT step, repo FROM jobs WHERE id IN (" + ids.join(",") + ")").all() as { step: string; repo: string }[]).map((j) => `${j.step}:${j.repo}`), ["test-repo:a"]);
assert.equal(db.prepare("SELECT COUNT(*) AS n FROM jobs").get()!.n, 1);
});