mirror of
https://github.com/storytold/autocraft.git
synced 2026-11-21 23:10:57 +01:00
76 lines
3.9 KiB
TypeScript
76 lines
3.9 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { describe, test } from "node:test";
|
|
import { renderJev } from "../src/prompts/index.ts";
|
|
import { settle } from "../src/classify/settled.ts";
|
|
import { itemInput, jevVerdict, parseHaikuReply, parseRefereeReply, type Item } from "../src/classify/parse.ts";
|
|
|
|
const refs = ["a#1", "a#2", "a#3"];
|
|
|
|
describe("haiku reply", () => {
|
|
test("accepts fenced JSON and reports items that are missing or invalid", () => {
|
|
const reply = "```json\n" + JSON.stringify([
|
|
{ ref: "a#1", type_label: "bugfix", security: true, needs_decision: false, reason: "fixes a crash" },
|
|
{ ref: "a#2", type_label: "feature", security: false, needs_decision: false, reason: "bad label" },
|
|
{ ref: "zzz#9", type_label: "infra", security: false, needs_decision: false, reason: "not asked" },
|
|
]) + "\n```";
|
|
const { verdicts, missing } = parseHaikuReply(reply, refs);
|
|
assert.deepEqual(verdicts.map((v) => v.ref), ["a#1"]);
|
|
assert.deepEqual(missing, ["a#2", "a#3"]);
|
|
});
|
|
|
|
test("rejects text that is not JSON", () => {
|
|
assert.deepEqual(parseHaikuReply("Sorry, I cannot.", refs).verdicts, []);
|
|
assert.deepEqual(parseRefereeReply("[oops", refs), []);
|
|
});
|
|
});
|
|
|
|
describe("jev", () => {
|
|
const answers = {
|
|
type: { choice: "performance", confidence: 0.8, probabilities: { performance: 0.8, bugfix: 0.2 } },
|
|
security: { noul: 0.95 }, needs_decision: { noul: 0.1 },
|
|
};
|
|
test("maps an answer to a classification row with probabilities kept", () => {
|
|
const v = jevVerdict("a#1", answers)!;
|
|
assert.equal(v.type_label, "performance");
|
|
assert.equal(v.security, true);
|
|
assert.equal(v.needs_decision, false);
|
|
assert.equal(JSON.parse(v.probs!).top, 0.8);
|
|
});
|
|
test("Jev's flags are yes from 0.9 (security) and 0.8 (needs decision)", () => {
|
|
assert.equal(jevVerdict("a#1", { ...answers, security: { noul: 0.89 } })!.security, false);
|
|
assert.equal(jevVerdict("a#1", { ...answers, security: { noul: 0.9 } })!.security, true);
|
|
assert.equal(jevVerdict("a#1", { ...answers, needs_decision: { noul: 0.79 } })!.needs_decision, false);
|
|
assert.equal(jevVerdict("a#1", { ...answers, needs_decision: { noul: 0.8 } })!.needs_decision, true);
|
|
});
|
|
test("a flag settles by Jev's band: sure yes sets it, sure no beats Haiku, unsure takes Haiku's answer or stays unset", () => {
|
|
const jev = (sec: number) => ({ label: "bugfix", flags: [], top: 0.95, sec, nd: 0 });
|
|
const yes = { label: "bugfix", flags: ["security"] };
|
|
assert.deepEqual(settle(jev(0.95), null).flags, ["security"]);
|
|
assert.deepEqual(settle(jev(0.1), yes).flags, []);
|
|
assert.deepEqual(settle(jev(0.5), yes).flags, ["security"]);
|
|
assert.deepEqual(settle(jev(0.5), { label: "bugfix", flags: [] }).flags, []);
|
|
assert.deepEqual(settle(jev(0.5), null).flags, []);
|
|
});
|
|
test("drops an answer with an unknown label or a missing question", () => {
|
|
assert.equal(jevVerdict("a#1", { ...answers, type: { choice: "other" } }), null);
|
|
assert.equal(jevVerdict("a#1", { type: answers.type }), null);
|
|
});
|
|
test("item text goes in the state as data, cut to 4000 characters", () => {
|
|
const item: Item = { repo: "a", number: 1, kind: "pr", title: "t", body: "x".repeat(5000), lines_changed: 10, files_changed: 2, author_association: "NONE" };
|
|
assert.equal((itemInput(item).body as string).length, 4000);
|
|
const req = JSON.parse(renderJev("classify.jev", item)) as { state: string };
|
|
assert.equal(JSON.parse(req.state).files_changed, 2);
|
|
});
|
|
});
|
|
|
|
describe("settle without Jev", () => {
|
|
test("a Haiku verdict alone settles the item with Haiku's label and flags", () => {
|
|
const s = settle(null, { label: "bugfix", flags: ["security"] });
|
|
assert.equal(s.status, "settled");
|
|
assert.equal(s.source, "haiku");
|
|
assert.equal(s.label, "bugfix");
|
|
assert.deepEqual(s.flags, ["security"]);
|
|
assert.equal(settle(null, null).status, "unclassified");
|
|
});
|
|
});
|