mirror of
https://github.com/storytold/autocraft.git
synced 2026-11-20 22:41:03 +01:00
Review fixes: gated proposals reserve their step, batch merge and unassign respect PR leases, fallback proposals per repo, lane gate by capacity
This commit is contained in:
1 parent
40e8595cb1
commit
7f92251ea4
6 files changed
+73
-33
No files matched your search
@@ -29,7 +29,7 @@ export async function nextPage(db: DatabaseSync, visibleNames: string[], opts: {
|
||||
const kept = stored[lane.name]?.items ?? [];
|
||||
const items: NextItem[] = kept.filter((i) => visible.has(i.repo)).map((i) => ({ ...i, ...shown(i.repo, i.number), why: why(i.repo, i.number, i.action) }));
|
||||
const total = (stored[lane.name]?.total ?? 0) - (kept.length - items.length);
|
||||
const gate = laneGate(db, lane, ms);
|
||||
const gate = laneGate(db, lane, stepDefs, ms);
|
||||
const open = items.filter((i) => !i.blocked);
|
||||
// Every item blocked by a repo approval: the step cannot do its work although it could start.
|
||||
const refused = !open.length && items.length ? items[0]!.blocked : null;
|
||||
|
||||
+31
-18
@@ -37,6 +37,7 @@ import { PIPELINE_LOGIN_KEY, buildQueue, type QueueEntry } from "./queue.ts";
|
||||
import { planned } from "../core/select.ts";
|
||||
import { envVar } from "../env.ts";
|
||||
import { jobItem } from "../runner/job-items.ts";
|
||||
import { leaseItem } from "../runner/leases.ts";
|
||||
import { ownerOf, slugOf } from "../owners.ts";
|
||||
|
||||
const ACTOR = "batch";
|
||||
@@ -351,25 +352,37 @@ export async function runBatches(db: DatabaseSync, params: BatchParams, log: (li
|
||||
// Holds and controls may have changed during the calls above: look again right before merging.
|
||||
if (getBatch(db, b.id).state !== "open") { log(` ${b.branch}: ended meanwhile; not merged`); return; }
|
||||
if (members.some((m) => mergeHeld(db, repo, m.number))) { log(` ${b.branch}: a PR in the batch was held meanwhile; not merged`); s.waiting.push(b.branch); return; }
|
||||
let after: Awaited<ReturnType<typeof refreshChecks>>[number] | undefined;
|
||||
// Every member must still hold at its batched head, and the batch PR at the batch tip.
|
||||
const g = await guardBatchMerge(db, { repo, number: b.pr_number!, head: b.commit_sha }, members.map((m) => ({ number: m.number, head: m.head_oid })), by, liveRead);
|
||||
if ("refused" in g) { log(` ${b.branch}: not merged: ${g.refused}`); s.waiting.push(b.branch); return; }
|
||||
const r = await mergeViaCommand(w, d.run, d.mergeCommand, g, { method: "merge", extra: allowFailedArgs(db, repo, d.mergeCommand) }, { run: runId, head: b.commit_sha, batch: b.branch }, async () => {
|
||||
after = (await refreshChecks(db, d.querier, repo, [b.pr_number!]))[0];
|
||||
return after?.state === "MERGED";
|
||||
});
|
||||
if (r.code !== 0) {
|
||||
// The merge may have gone through although the command failed afterwards.
|
||||
if (after?.state === "MERGED") return recordLanded(b, members, q);
|
||||
const why = oneLine(r.stderr) || oneLine(r.stdout) || `exit code ${r.code}`;
|
||||
updateBatch(db, b.id, at(), { summary: `${d.mergeCommand}: ${why}`.slice(0, 400) });
|
||||
log(` ${b.branch}: refused: ${why}`);
|
||||
s.waiting.push(b.branch);
|
||||
return;
|
||||
// A fix-up or CI fix pushing to a member would move its head under the merge: hold the members' leases until it ends.
|
||||
const leases: (() => void)[] = [];
|
||||
for (const m of members) {
|
||||
const release = leaseItem(db, `pr:${repo}#${m.number}`, log);
|
||||
if (!release) { for (const f of leases) f(); log(` ${b.branch}: a PR in the batch is being worked; not merged this pass`); s.waiting.push(b.branch); return; }
|
||||
leases.push(release);
|
||||
}
|
||||
try {
|
||||
let after: Awaited<ReturnType<typeof refreshChecks>>[number] | undefined;
|
||||
// Every member must still hold at its batched head, and the batch PR at the batch tip.
|
||||
const g = await guardBatchMerge(db, { repo, number: b.pr_number!, head: b.commit_sha }, members.map((m) => ({ number: m.number, head: m.head_oid })), by, liveRead);
|
||||
if ("refused" in g) { log(` ${b.branch}: not merged: ${g.refused}`); s.waiting.push(b.branch); return; }
|
||||
const r = await mergeViaCommand(w, d.run, d.mergeCommand, g, { method: "merge", extra: allowFailedArgs(db, repo, d.mergeCommand) }, { run: runId, head: b.commit_sha, batch: b.branch }, async () => {
|
||||
after = (await refreshChecks(db, d.querier, repo, [b.pr_number!]))[0];
|
||||
return after?.state === "MERGED";
|
||||
});
|
||||
if (r.code !== 0) {
|
||||
// The merge may have gone through although the command failed afterwards.
|
||||
if (after?.state === "MERGED") return recordLanded(b, members, q);
|
||||
const why = oneLine(r.stderr) || oneLine(r.stdout) || `exit code ${r.code}`;
|
||||
updateBatch(db, b.id, at(), { summary: `${d.mergeCommand}: ${why}`.slice(0, 400) });
|
||||
log(` ${b.branch}: refused: ${why}`);
|
||||
s.waiting.push(b.branch);
|
||||
return;
|
||||
}
|
||||
// The merge command prints `<repo>#<pr> MERGED <merge commit>`.
|
||||
recordLanded(b, members, q, /\bMERGED ([0-9a-f]{7,40})\b/.exec(r.stdout)?.[1] ?? null);
|
||||
|
||||
} finally {
|
||||
for (const f of leases) f();
|
||||
}
|
||||
// The merge command prints `<repo>#<pr> MERGED <merge commit>`.
|
||||
recordLanded(b, members, q, /\bMERGED ([0-9a-f]{7,40})\b/.exec(r.stdout)?.[1] ?? null);
|
||||
}
|
||||
|
||||
async function advance(id: number): Promise<void> {
|
||||
|
||||
+14
-7
@@ -47,7 +47,7 @@ import { claudePauseReason } from "../agents/accounts.ts";
|
||||
import { tripReasonOfStep } from "../push-outcomes.ts";
|
||||
import { STEP_STAGE } from "../core/push-outcome.ts";
|
||||
import { jobBlock, stepParallel, type Active } from "./concurrency.ts";
|
||||
import { knownStep, loadSteps, modelsOf, usesClaude, type Params, type StepDef } from "./steps.ts";
|
||||
import { knownStep, loadSteps, STEPS, modelsOf, usesClaude, type Params, type StepDef } from "./steps.ts";
|
||||
import { pilotItems, scopeRepos } from "../pilot.ts";
|
||||
import { JUDGEMENT_MODELS, judgementModel } from "../decide/judgement.ts";
|
||||
import { FAST_FLOOR, fastFirst, fastScoreOf } from "../fasttrack.ts";
|
||||
@@ -782,7 +782,7 @@ export type Detected = {
|
||||
timing?: { loadMs: number; lanesMs: number; cacheMs: number; written: number; deleted: number };
|
||||
};
|
||||
|
||||
/** The fewest proposals one lane hands to apply, in case the first ones are refused at enqueue; the proposals of a lane are for different repos. */
|
||||
/** The fewest proposals one lane hands to apply: one per repo, in order, so that when the first is refused at enqueue (or the step is full at step_parallel) the next repo's goes in. */
|
||||
const PROPOSALS_PER_LANE = 3;
|
||||
|
||||
const nextUtcMidnight = (ms: number): number => Date.parse(dayStart(ms)) + 86_400_000;
|
||||
@@ -924,6 +924,9 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op
|
||||
const lanes = opts.lanes ?? LANES, log = opts.log ?? (() => {});
|
||||
const queued: number[] = [];
|
||||
const done = new Set<string>();
|
||||
// Jobs queued in this pass and the steps their proposals gate: they hold a repo of the step, so with step_parallel 1 no other repo takes it.
|
||||
const reserved: Active[] = [];
|
||||
const asActive = (step: string, params: Params, state: string): Active => ({ id: 0, step, state, repo: typeof params.repo === "string" ? params.repo : null, number: typeof params.number === "number" ? params.number : null });
|
||||
const steps = await loadSteps();
|
||||
try { applyUsedAnswers(db); } catch (e) { log(`autopilot: answers: ${(e as Error).message}`); }
|
||||
// Marks the next planner pass wants: when a wait ends, and why stages are not starting.
|
||||
@@ -935,7 +938,8 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op
|
||||
if (done.has(slot)) continue;
|
||||
try {
|
||||
// The detection ran a moment ago: a job of the same step and repo may have been queued since.
|
||||
if (jobBlock(db, steps, { step: p.step, repo: typeof p.params.repo === "string" ? p.params.repo : null, number: typeof p.params.number === "number" ? p.params.number : null })) continue;
|
||||
if (jobBlock(db, steps, { step: p.step, repo: typeof p.params.repo === "string" ? p.params.repo : null, number: typeof p.params.number === "number" ? p.params.number : null }, { reserved })) continue;
|
||||
if (p.gate && jobBlock(db, steps, { step: p.gate.step, repo: typeof p.gate.params.repo === "string" ? p.gate.params.repo : null }, { reserved })) continue;
|
||||
// The limit is what is left now, so two proposals of one step cannot both carry the whole remainder.
|
||||
const job = await enqueue(db, p.step, p.params, AUTOPILOT_USER, (step, params) => {
|
||||
if (typeof params.limit !== "number") return params;
|
||||
@@ -943,6 +947,8 @@ export async function applyProposals(db: DatabaseSync, proposals: Proposal[], op
|
||||
return lim ? { ...params, ...lim } : null;
|
||||
});
|
||||
queued.push(job.id);
|
||||
reserved.push(asActive(p.step, p.params, "queued"));
|
||||
if (p.gate) reserved.push(asActive(p.gate.step, p.gate.params, "gated"));
|
||||
done.add(slot);
|
||||
if (p.fp !== undefined) setSetting(db, fpKey(p.step, typeof p.params.repo === "string" ? p.params.repo : null), JSON.stringify({ job: job.id, fp: p.fp } satisfies FpRecord));
|
||||
const c: Candidate = { step: p.step, params: Object.fromEntries(Object.entries(p.params).filter(([k]) => k !== "limit")) };
|
||||
@@ -1100,7 +1106,7 @@ export function heldLookup(db: DatabaseSync): (repo: string, number: number, dec
|
||||
const laneHold = (name: string): Held | null => {
|
||||
if (!gateOf.has(name)) {
|
||||
const lane = LANES.find((l) => l.name === name);
|
||||
const g = lane ? laneGate(db, lane) : null;
|
||||
const g = lane ? laneGate(db, lane, new Map(STEPS.map((x) => [x.name, x]))) : null;
|
||||
gateOf.set(name, g && !g.canRun && g.kind !== "running" && g.why ? { reason: plain(g.why), approval: false } : null);
|
||||
}
|
||||
return gateOf.get(name)!;
|
||||
@@ -1124,13 +1130,14 @@ export function heldLookup(db: DatabaseSync): (repo: string, number: number, dec
|
||||
export type LaneGate = { canRun: boolean; kind: "ok" | "running" | "off" | "breaker" | "limit" | "account" | "backoff"; why: string | null };
|
||||
|
||||
/** Whether a job of the lane can start now, from the same checks the planner's gate() makes. */
|
||||
export function laneGate(db: DatabaseSync, lane: Pick<Lane, "steps">, ms: number = Date.now()): LaneGate {
|
||||
export function laneGate(db: DatabaseSync, lane: Pick<Lane, "steps">, allSteps: Map<string, StepDef>, ms: number = Date.now()): LaneGate {
|
||||
if (!isEnabled(db)) return { canRun: false, kind: "off", why: isDraining(db) ? "The pipeline is draining" : "The pipeline is off" };
|
||||
const steps = lane.steps ?? [];
|
||||
let blocked: LaneGate | null = null;
|
||||
for (const s of steps) {
|
||||
const j = db.prepare("SELECT id, state FROM jobs WHERE step = ? AND state IN ('queued', 'running') ORDER BY state = 'running' DESC LIMIT 1").get(s) as { id: number; state: string } | undefined;
|
||||
if (j) { blocked ??= { canRun: false, kind: "running", why: `Job ${j.id} of ${s} is ${j.state}` }; continue; }
|
||||
// Blocked only when no repo can start: the step already works its most repos.
|
||||
const full = jobBlock(db, allSteps, { step: s, repo: null }, { precheck: true });
|
||||
if (full) { blocked ??= { canRun: false, kind: "running", why: full.replace(/ \(step_parallel\)$/, "") }; continue; }
|
||||
const why = pauseReason(db, s, ms);
|
||||
if (!why) return { canRun: true, kind: "ok", why: null };
|
||||
const kind = /^tripped/.test(why) ? "breaker" : /daily limit/.test(why) ? "limit" : /^failed/.test(why) ? "backoff" : "account";
|
||||
|
||||
@@ -52,25 +52,25 @@ export type Want = { step: string; repo: string | null; number?: number | null }
|
||||
|
||||
/**
|
||||
* Why the job cannot be queued or started now, or null. `states` are the jobs that count: queued and running for
|
||||
* queueing, running alone for starting. `extra` are jobs about to be queued. `mutex` also checks StepDef.exclusive
|
||||
* queueing, running alone for starting. `extra` are proposals of the same pass: they only count as the same work (a duplicate), so a fallback proposal for another repo can still be made. `reserved` are jobs this pass already queued or gated: they count for the same work and for the repo capacity. `mutex` also checks StepDef.exclusive
|
||||
* against running jobs. With `precheck` (a planner check before any repo is known) only the repo capacity is checked.
|
||||
*/
|
||||
export function jobBlock(db: DatabaseSync, steps: Map<string, StepDef>, want: Want, o: { states?: ("queued" | "running")[]; extra?: Active[]; mutex?: boolean; precheck?: boolean } = {}): string | null {
|
||||
const jobs = [...activeJobs(db, o.states), ...(o.extra ?? [])];
|
||||
const same = jobs.filter((j) => j.step === want.step);
|
||||
export function jobBlock(db: DatabaseSync, steps: Map<string, StepDef>, want: Want, o: { states?: ("queued" | "running")[]; extra?: Active[]; reserved?: Active[]; mutex?: boolean; precheck?: boolean } = {}): string | null {
|
||||
const held = [...activeJobs(db, o.states), ...(o.reserved ?? [])].filter((j) => j.step === want.step);
|
||||
const same = [...held, ...(o.extra ?? []).filter((j) => j.step === want.step)];
|
||||
if (!o.precheck) {
|
||||
const clash = same.find((j) => sameWork(want.step, j, { repo: want.repo, number: want.number ?? null }));
|
||||
if (clash) return `job ${clash.id} of ${want.step} is ${clash.state}`;
|
||||
}
|
||||
const cap = stepParallel(db);
|
||||
if (cap !== Infinity) {
|
||||
const repos = new Set(same.map((j) => j.repo ?? ""));
|
||||
const repos = new Set(held.map((j) => j.repo ?? ""));
|
||||
if (repos.size >= cap && (o.precheck || !repos.has(want.repo ?? ""))) return `${want.step} is working ${repos.size} repo${repos.size === 1 ? "" : "s"}, the most at once (step_parallel)`;
|
||||
}
|
||||
const mutex = steps.get(want.step)?.exclusive;
|
||||
if (o.mutex && mutex) {
|
||||
const held = activeJobs(db, ["running"]).find((j) => steps.get(j.step)?.exclusive === mutex);
|
||||
if (held) return `job ${held.id} (${held.step}) is running with the same mutex (${mutex})`;
|
||||
const other = activeJobs(db, ["running"]).find((j) => steps.get(j.step)?.exclusive === mutex);
|
||||
if (other) return `job ${other.id} (${other.step}) is running with the same mutex (${mutex})`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import type { Writer } from "../github/write.ts";
|
||||
import { viewerOf } from "../merge/index.ts";
|
||||
import { Client, type Querier } from "../mirror.ts";
|
||||
import { jobItem } from "../runner/job-items.ts";
|
||||
import { leaseItem } from "../runner/leases.ts";
|
||||
import { ownerOf } from "../owners.ts";
|
||||
|
||||
export const ACTOR = "unassign";
|
||||
@@ -40,12 +41,17 @@ export async function runUnassign(db: DatabaseSync, p: UnassignParams): Promise<
|
||||
db.prepare("INSERT INTO main.settings (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value").run(PIPELINE_LOGIN_KEY, me.login);
|
||||
for (const d of due) {
|
||||
const ref = `${p.repo}#${d.number}`;
|
||||
// A running job works the item (it may have just claimed it): leave the claim alone.
|
||||
const release = leaseItem(db, `${d.pr ? "pr" : "issue"}:${ref}`, log);
|
||||
if (!release) continue;
|
||||
try {
|
||||
const info = await (d.pr ? pullRequestInfo(q, ownerOf(p.repo), p.repo, d.number) : issueInfo(q, ownerOf(p.repo), p.repo, d.number));
|
||||
if (info && await releaseItem(w, g, q, p.repo, d.number, info, me, { run: p.runId })) { out.unassigned.push(ref); jobItem(db, p.repo, d.number, `unassigned from ${me.login}`, log); }
|
||||
} catch (e) {
|
||||
out.failed.push(ref);
|
||||
jobItem(db, p.repo, d.number, `could not unassign: ${oneLine(String((e as Error).message ?? e))}`, log);
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
return out;
|
||||
|
||||
@@ -62,3 +62,17 @@ test("at the process cap a queued security-class job starts before an older norm
|
||||
await settle(r, [busy.id, normal.id, urgent.id]);
|
||||
assert.ok(getJob(db, urgent.id).startedAt! <= getJob(db, normal.id).startedAt!, "the security job started first");
|
||||
});
|
||||
|
||||
test("a proposal reserves the step it is gated on: at step_parallel 1 another repo's job of that step waits", async () => {
|
||||
const { applyProposals, detectAutopilot } = await import("../src/runner/autopilot.ts");
|
||||
db.exec("DELETE FROM jobs");
|
||||
setStepParallel(db, 1, "test");
|
||||
const lanes = [
|
||||
{ name: "first", steps: ["test-repo"], detect: () => [{ step: "test-repo", params: { repo: "a" }, gate: { step: "test-urgent", params: { repo: "a" } } }] },
|
||||
{ name: "second", steps: ["test-urgent"], detect: () => [{ step: "test-urgent", params: { repo: "b" } }] },
|
||||
];
|
||||
const found = await detectAutopilot(db, { lanes });
|
||||
const ids = await applyProposals(db, found.proposals, { lanes });
|
||||
assert.deepEqual((db.prepare("SELECT step, repo FROM jobs WHERE id IN (" + ids.join(",") + ")").all() as { step: string; repo: string }[]).map((j) => `${j.step}:${j.repo}`), ["test-repo:a"]);
|
||||
assert.equal(db.prepare("SELECT COUNT(*) AS n FROM jobs").get()!.n, 1);
|
||||
});
|
||||
Reference in new issue
Block a user