From 7f92251ea4601e350ffb972779d69cbeb7e3885b Mon Sep 17 00:00:00 2001 From: aiongg <67543387+aiongg@users.noreply.github.com> Date: Sat, 7 Nov 2026 00:31:40 +0000 Subject: [PATCH] Review fixes: gated proposals reserve their step, batch merge and unassign respect PR leases, fallback proposals per repo, lane gate by capacity --- src/dash/api/next.ts | 2 +- src/merge/batch.ts | 49 +++++++++++++++++++++++++-------------- src/runner/autopilot.ts | 21 +++++++++++------ src/runner/concurrency.ts | 14 +++++------ src/unassign/index.ts | 6 +++++ test/step-repo.test.ts | 14 +++++++++++ 6 files changed, 73 insertions(+), 33 deletions(-) diff --git a/src/dash/api/next.ts b/src/dash/api/next.ts index 41cb78ee..556973a5 100644 --- a/src/dash/api/next.ts +++ b/src/dash/api/next.ts @@ -29,7 +29,7 @@ export async function nextPage(db: DatabaseSync, visibleNames: string[], opts: { const kept = stored[lane.name]?.items ?? []; const items: NextItem[] = kept.filter((i) => visible.has(i.repo)).map((i) => ({ ...i, ...shown(i.repo, i.number), why: why(i.repo, i.number, i.action) })); const total = (stored[lane.name]?.total ?? 0) - (kept.length - items.length); - const gate = laneGate(db, lane, ms); + const gate = laneGate(db, lane, stepDefs, ms); const open = items.filter((i) => !i.blocked); // Every item blocked by a repo approval: the step cannot do its work although it could start. const refused = !open.length && items.length ? items[0]!.blocked : null; diff --git a/src/merge/batch.ts b/src/merge/batch.ts index 71568db6..88053219 100644 --- a/src/merge/batch.ts +++ b/src/merge/batch.ts @@ -37,6 +37,7 @@ import { PIPELINE_LOGIN_KEY, buildQueue, type QueueEntry } from "./queue.ts"; import { planned } from "../core/select.ts"; import { envVar } from "../env.ts"; import { jobItem } from "../runner/job-items.ts"; +import { leaseItem } from "../runner/leases.ts"; import { ownerOf, slugOf } from "../owners.ts"; const ACTOR = "batch"; @@ -351,25 +352,37 @@ export async function runBatches(db: DatabaseSync, params: BatchParams, log: (li // Holds and controls may have changed during the calls above: look again right before merging. if (getBatch(db, b.id).state !== "open") { log(` ${b.branch}: ended meanwhile; not merged`); return; } if (members.some((m) => mergeHeld(db, repo, m.number))) { log(` ${b.branch}: a PR in the batch was held meanwhile; not merged`); s.waiting.push(b.branch); return; } - let after: Awaited>[number] | undefined; - // Every member must still hold at its batched head, and the batch PR at the batch tip. - const g = await guardBatchMerge(db, { repo, number: b.pr_number!, head: b.commit_sha }, members.map((m) => ({ number: m.number, head: m.head_oid })), by, liveRead); - if ("refused" in g) { log(` ${b.branch}: not merged: ${g.refused}`); s.waiting.push(b.branch); return; } - const r = await mergeViaCommand(w, d.run, d.mergeCommand, g, { method: "merge", extra: allowFailedArgs(db, repo, d.mergeCommand) }, { run: runId, head: b.commit_sha, batch: b.branch }, async () => { - after = (await refreshChecks(db, d.querier, repo, [b.pr_number!]))[0]; - return after?.state === "MERGED"; - }); - if (r.code !== 0) { - // The merge may have gone through although the command failed afterwards. - if (after?.state === "MERGED") return recordLanded(b, members, q); - const why = oneLine(r.stderr) || oneLine(r.stdout) || `exit code ${r.code}`; - updateBatch(db, b.id, at(), { summary: `${d.mergeCommand}: ${why}`.slice(0, 400) }); - log(` ${b.branch}: refused: ${why}`); - s.waiting.push(b.branch); - return; + // A fix-up or CI fix pushing to a member would move its head under the merge: hold the members' leases until it ends. + const leases: (() => void)[] = []; + for (const m of members) { + const release = leaseItem(db, `pr:${repo}#${m.number}`, log); + if (!release) { for (const f of leases) f(); log(` ${b.branch}: a PR in the batch is being worked; not merged this pass`); s.waiting.push(b.branch); return; } + leases.push(release); + } + try { + let after: Awaited>[number] | undefined; + // Every member must still hold at its batched head, and the batch PR at the batch tip. + const g = await guardBatchMerge(db, { repo, number: b.pr_number!, head: b.commit_sha }, members.map((m) => ({ number: m.number, head: m.head_oid })), by, liveRead); + if ("refused" in g) { log(` ${b.branch}: not merged: ${g.refused}`); s.waiting.push(b.branch); return; } + const r = await mergeViaCommand(w, d.run, d.mergeCommand, g, { method: "merge", extra: allowFailedArgs(db, repo, d.mergeCommand) }, { run: runId, head: b.commit_sha, batch: b.branch }, async () => { + after = (await refreshChecks(db, d.querier, repo, [b.pr_number!]))[0]; + return after?.state === "MERGED"; + }); + if (r.code !== 0) { + // The merge may have gone through although the command failed afterwards. + if (after?.state === "MERGED") return recordLanded(b, members, q); + const why = oneLine(r.stderr) || oneLine(r.stdout) || `exit code ${r.code}`; + updateBatch(db, b.id, at(), { summary: `${d.mergeCommand}: ${why}`.slice(0, 400) }); + log(` ${b.branch}: refused: ${why}`); + s.waiting.push(b.branch); + return; + } + // The merge command prints `# MERGED `. + recordLanded(b, members, q, /\bMERGED ([0-9a-f]{7,40})\b/.exec(r.stdout)?.[1] ?? null); + + } finally { + for (const f of leases) f(); } - // The merge command prints `# MERGED `. - recordLanded(b, members, q, /\bMERGED ([0-9a-f]{7,40})\b/.exec(r.stdout)?.[1] ?? null); } async function advance(id: number): Promise { diff --git a/src/runner/autopilot.ts b/src/runner/autopilot.ts index 68ce0f81..cafb5f0a 100644 --- a/src/runner/autopilot.ts +++ b/src/runner/autopilot.ts @@ -47,7 +47,7 @@ import { claudePauseReason } from "../agents/accounts.ts"; import { tripReasonOfStep } from "../push-outcomes.ts"; import { STEP_STAGE } from "../core/push-outcome.ts"; import { jobBlock, stepParallel, type Active } from "./concurrency.ts"; -import { knownStep, loadSteps, modelsOf, usesClaude, type Params, type StepDef } from "./steps.ts"; +import { knownStep, loadSteps, STEPS, modelsOf, usesClaude, type Params, type StepDef } from "./steps.ts"; import { pilotItems, scopeRepos } from "../pilot.ts"; import { JUDGEMENT_MODELS, judgementModel } from "../decide/judgement.ts"; import { FAST_FLOOR, fastFirst, fastScoreOf } from "../fasttrack.ts"; @@ -782,7 +782,7 @@ export type Detected = { timing?: { loadMs: number; lanesMs: number; cacheMs: number; written: number; deleted: number }; }; -/** The fewest proposals one lane hands to apply, in case the first ones are refused at enqueue; the proposals of a lane are for different repos. */ +/** The fewest proposals one lane hands to apply: one per repo, in order, so that when the first is refused at enqueue (or the step is full at step_parallel) the next repo's goes in. */ const PROPOSALS_PER_LANE = 3; const nextUtcMidnight = (ms: number): number => Date.parse(dayStart(ms)) + 86_400_000; @@ -924,6 +924,9 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op const lanes = opts.lanes ?? LANES, log = opts.log ?? (() => {}); const queued: number[] = []; const done = new Set(); + // Jobs queued in this pass and the steps their proposals gate: they hold a repo of the step, so with step_parallel 1 no other repo takes it. + const reserved: Active[] = []; + const asActive = (step: string, params: Params, state: string): Active => ({ id: 0, step, state, repo: typeof params.repo === "string" ? params.repo : null, number: typeof params.number === "number" ? params.number : null }); const steps = await loadSteps(); try { applyUsedAnswers(db); } catch (e) { log(`autopilot: answers: ${(e as Error).message}`); } // Marks the next planner pass wants: when a wait ends, and why stages are not starting. @@ -935,7 +938,8 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op if (done.has(slot)) continue; try { // The detection ran a moment ago: a job of the same step and repo may have been queued since. - if (jobBlock(db, steps, { step: p.step, repo: typeof p.params.repo === "string" ? p.params.repo : null, number: typeof p.params.number === "number" ? p.params.number : null })) continue; + if (jobBlock(db, steps, { step: p.step, repo: typeof p.params.repo === "string" ? p.params.repo : null, number: typeof p.params.number === "number" ? p.params.number : null }, { reserved })) continue; + if (p.gate && jobBlock(db, steps, { step: p.gate.step, repo: typeof p.gate.params.repo === "string" ? p.gate.params.repo : null }, { reserved })) continue; // The limit is what is left now, so two proposals of one step cannot both carry the whole remainder. const job = await enqueue(db, p.step, p.params, AUTOPILOT_USER, (step, params) => { if (typeof params.limit !== "number") return params; @@ -943,6 +947,8 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op return lim ? { ...params, ...lim } : null; }); queued.push(job.id); + reserved.push(asActive(p.step, p.params, "queued")); + if (p.gate) reserved.push(asActive(p.gate.step, p.gate.params, "gated")); done.add(slot); if (p.fp !== undefined) setSetting(db, fpKey(p.step, typeof p.params.repo === "string" ? p.params.repo : null), JSON.stringify({ job: job.id, fp: p.fp } satisfies FpRecord)); const c: Candidate = { step: p.step, params: Object.fromEntries(Object.entries(p.params).filter(([k]) => k !== "limit")) }; @@ -1100,7 +1106,7 @@ export function heldLookup(db: DatabaseSync): (repo: string, number: number, dec const laneHold = (name: string): Held | null => { if (!gateOf.has(name)) { const lane = LANES.find((l) => l.name === name); - const g = lane ? laneGate(db, lane) : null; + const g = lane ? laneGate(db, lane, new Map(STEPS.map((x) => [x.name, x]))) : null; gateOf.set(name, g && !g.canRun && g.kind !== "running" && g.why ? { reason: plain(g.why), approval: false } : null); } return gateOf.get(name)!; @@ -1124,13 +1130,14 @@ export function heldLookup(db: DatabaseSync): (repo: string, number: number, dec export type LaneGate = { canRun: boolean; kind: "ok" | "running" | "off" | "breaker" | "limit" | "account" | "backoff"; why: string | null }; /** Whether a job of the lane can start now, from the same checks the planner's gate() makes. */ -export function laneGate(db: DatabaseSync, lane: Pick, ms: number = Date.now()): LaneGate { +export function laneGate(db: DatabaseSync, lane: Pick, allSteps: Map, ms: number = Date.now()): LaneGate { if (!isEnabled(db)) return { canRun: false, kind: "off", why: isDraining(db) ? "The pipeline is draining" : "The pipeline is off" }; const steps = lane.steps ?? []; let blocked: LaneGate | null = null; for (const s of steps) { - const j = db.prepare("SELECT id, state FROM jobs WHERE step = ? AND state IN ('queued', 'running') ORDER BY state = 'running' DESC LIMIT 1").get(s) as { id: number; state: string } | undefined; - if (j) { blocked ??= { canRun: false, kind: "running", why: `Job ${j.id} of ${s} is ${j.state}` }; continue; } + // Blocked only when no repo can start: the step already works its most repos. + const full = jobBlock(db, allSteps, { step: s, repo: null }, { precheck: true }); + if (full) { blocked ??= { canRun: false, kind: "running", why: full.replace(/ \(step_parallel\)$/, "") }; continue; } const why = pauseReason(db, s, ms); if (!why) return { canRun: true, kind: "ok", why: null }; const kind = /^tripped/.test(why) ? "breaker" : /daily limit/.test(why) ? "limit" : /^failed/.test(why) ? "backoff" : "account"; diff --git a/src/runner/concurrency.ts b/src/runner/concurrency.ts index 1363fa88..31ff4a70 100644 --- a/src/runner/concurrency.ts +++ b/src/runner/concurrency.ts @@ -52,25 +52,25 @@ export type Want = { step: string; repo: string | null; number?: number | null } /** * Why the job cannot be queued or started now, or null. `states` are the jobs that count: queued and running for - * queueing, running alone for starting. `extra` are jobs about to be queued. `mutex` also checks StepDef.exclusive + * queueing, running alone for starting. `extra` are proposals of the same pass: they only count as the same work (a duplicate), so a fallback proposal for another repo can still be made. `reserved` are jobs this pass already queued or gated: they count for the same work and for the repo capacity. `mutex` also checks StepDef.exclusive * against running jobs. With `precheck` (a planner check before any repo is known) only the repo capacity is checked. */ -export function jobBlock(db: DatabaseSync, steps: Map, want: Want, o: { states?: ("queued" | "running")[]; extra?: Active[]; mutex?: boolean; precheck?: boolean } = {}): string | null { - const jobs = [...activeJobs(db, o.states), ...(o.extra ?? [])]; - const same = jobs.filter((j) => j.step === want.step); +export function jobBlock(db: DatabaseSync, steps: Map, want: Want, o: { states?: ("queued" | "running")[]; extra?: Active[]; reserved?: Active[]; mutex?: boolean; precheck?: boolean } = {}): string | null { + const held = [...activeJobs(db, o.states), ...(o.reserved ?? [])].filter((j) => j.step === want.step); + const same = [...held, ...(o.extra ?? []).filter((j) => j.step === want.step)]; if (!o.precheck) { const clash = same.find((j) => sameWork(want.step, j, { repo: want.repo, number: want.number ?? null })); if (clash) return `job ${clash.id} of ${want.step} is ${clash.state}`; } const cap = stepParallel(db); if (cap !== Infinity) { - const repos = new Set(same.map((j) => j.repo ?? "")); + const repos = new Set(held.map((j) => j.repo ?? "")); if (repos.size >= cap && (o.precheck || !repos.has(want.repo ?? ""))) return `${want.step} is working ${repos.size} repo${repos.size === 1 ? "" : "s"}, the most at once (step_parallel)`; } const mutex = steps.get(want.step)?.exclusive; if (o.mutex && mutex) { - const held = activeJobs(db, ["running"]).find((j) => steps.get(j.step)?.exclusive === mutex); - if (held) return `job ${held.id} (${held.step}) is running with the same mutex (${mutex})`; + const other = activeJobs(db, ["running"]).find((j) => steps.get(j.step)?.exclusive === mutex); + if (other) return `job ${other.id} (${other.step}) is running with the same mutex (${mutex})`; } return null; } diff --git a/src/unassign/index.ts b/src/unassign/index.ts index 477cd071..d9f5ef9e 100644 --- a/src/unassign/index.ts +++ b/src/unassign/index.ts @@ -12,6 +12,7 @@ import type { Writer } from "../github/write.ts"; import { viewerOf } from "../merge/index.ts"; import { Client, type Querier } from "../mirror.ts"; import { jobItem } from "../runner/job-items.ts"; +import { leaseItem } from "../runner/leases.ts"; import { ownerOf } from "../owners.ts"; export const ACTOR = "unassign"; @@ -40,12 +41,17 @@ export async function runUnassign(db: DatabaseSync, p: UnassignParams): Promise< db.prepare("INSERT INTO main.settings (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value").run(PIPELINE_LOGIN_KEY, me.login); for (const d of due) { const ref = `${p.repo}#${d.number}`; + // A running job works the item (it may have just claimed it): leave the claim alone. + const release = leaseItem(db, `${d.pr ? "pr" : "issue"}:${ref}`, log); + if (!release) continue; try { const info = await (d.pr ? pullRequestInfo(q, ownerOf(p.repo), p.repo, d.number) : issueInfo(q, ownerOf(p.repo), p.repo, d.number)); if (info && await releaseItem(w, g, q, p.repo, d.number, info, me, { run: p.runId })) { out.unassigned.push(ref); jobItem(db, p.repo, d.number, `unassigned from ${me.login}`, log); } } catch (e) { out.failed.push(ref); jobItem(db, p.repo, d.number, `could not unassign: ${oneLine(String((e as Error).message ?? e))}`, log); + } finally { + release(); } } return out; diff --git a/test/step-repo.test.ts b/test/step-repo.test.ts index eec283a8..560ff6ca 100644 --- a/test/step-repo.test.ts +++ b/test/step-repo.test.ts @@ -62,3 +62,17 @@ test("at the process cap a queued security-class job starts before an older norm await settle(r, [busy.id, normal.id, urgent.id]); assert.ok(getJob(db, urgent.id).startedAt! <= getJob(db, normal.id).startedAt!, "the security job started first"); }); + +test("a proposal reserves the step it is gated on: at step_parallel 1 another repo's job of that step waits", async () => { + const { applyProposals, detectAutopilot } = await import("../src/runner/autopilot.ts"); + db.exec("DELETE FROM jobs"); + setStepParallel(db, 1, "test"); + const lanes = [ + { name: "first", steps: ["test-repo"], detect: () => [{ step: "test-repo", params: { repo: "a" }, gate: { step: "test-urgent", params: { repo: "a" } } }] }, + { name: "second", steps: ["test-urgent"], detect: () => [{ step: "test-urgent", params: { repo: "b" } }] }, + ]; + const found = await detectAutopilot(db, { lanes }); + const ids = await applyProposals(db, found.proposals, { lanes }); + assert.deepEqual((db.prepare("SELECT step, repo FROM jobs WHERE id IN (" + ids.join(",") + ")").all() as { step: string; repo: string }[]).map((j) => `${j.step}:${j.repo}`), ["test-repo:a"]); + assert.equal(db.prepare("SELECT COUNT(*) AS n FROM jobs").get()!.n, 1); +});