Files
autocraft/test/classify.test.ts
T

76 lines
3.9 KiB
TypeScript

import assert from "node:assert/strict";
import { describe, test } from "node:test";
import { renderJev } from "../src/prompts/index.ts";
import { settle } from "../src/classify/settled.ts";
import { itemInput, jevVerdict, parseHaikuReply, parseRefereeReply, type Item } from "../src/classify/parse.ts";
const refs = ["a#1", "a#2", "a#3"];
describe("haiku reply", () => {
test("accepts fenced JSON and reports items that are missing or invalid", () => {
const reply = "```json\n" + JSON.stringify([
{ ref: "a#1", type_label: "bugfix", security: true, needs_decision: false, reason: "fixes a crash" },
{ ref: "a#2", type_label: "feature", security: false, needs_decision: false, reason: "bad label" },
{ ref: "zzz#9", type_label: "infra", security: false, needs_decision: false, reason: "not asked" },
]) + "\n```";
const { verdicts, missing } = parseHaikuReply(reply, refs);
assert.deepEqual(verdicts.map((v) => v.ref), ["a#1"]);
assert.deepEqual(missing, ["a#2", "a#3"]);
});
test("rejects text that is not JSON", () => {
assert.deepEqual(parseHaikuReply("Sorry, I cannot.", refs).verdicts, []);
assert.deepEqual(parseRefereeReply("[oops", refs), []);
});
});
describe("jev", () => {
const answers = {
type: { choice: "performance", confidence: 0.8, probabilities: { performance: 0.8, bugfix: 0.2 } },
security: { noul: 0.95 }, needs_decision: { noul: 0.1 },
};
test("maps an answer to a classification row with probabilities kept", () => {
const v = jevVerdict("a#1", answers)!;
assert.equal(v.type_label, "performance");
assert.equal(v.security, true);
assert.equal(v.needs_decision, false);
assert.equal(JSON.parse(v.probs!).top, 0.8);
});
test("Jev's flags are yes from 0.9 (security) and 0.8 (needs decision)", () => {
assert.equal(jevVerdict("a#1", { ...answers, security: { noul: 0.89 } })!.security, false);
assert.equal(jevVerdict("a#1", { ...answers, security: { noul: 0.9 } })!.security, true);
assert.equal(jevVerdict("a#1", { ...answers, needs_decision: { noul: 0.79 } })!.needs_decision, false);
assert.equal(jevVerdict("a#1", { ...answers, needs_decision: { noul: 0.8 } })!.needs_decision, true);
});
test("a flag settles by Jev's band: sure yes sets it, sure no beats Haiku, unsure takes Haiku's answer or stays unset", () => {
const jev = (sec: number) => ({ label: "bugfix", flags: [], top: 0.95, sec, nd: 0 });
const yes = { label: "bugfix", flags: ["security"] };
assert.deepEqual(settle(jev(0.95), null).flags, ["security"]);
assert.deepEqual(settle(jev(0.1), yes).flags, []);
assert.deepEqual(settle(jev(0.5), yes).flags, ["security"]);
assert.deepEqual(settle(jev(0.5), { label: "bugfix", flags: [] }).flags, []);
assert.deepEqual(settle(jev(0.5), null).flags, []);
});
test("drops an answer with an unknown label or a missing question", () => {
assert.equal(jevVerdict("a#1", { ...answers, type: { choice: "other" } }), null);
assert.equal(jevVerdict("a#1", { type: answers.type }), null);
});
test("item text goes in the state as data, cut to 4000 characters", () => {
const item: Item = { repo: "a", number: 1, kind: "pr", title: "t", body: "x".repeat(5000), lines_changed: 10, files_changed: 2, author_association: "NONE" };
assert.equal((itemInput(item).body as string).length, 4000);
const req = JSON.parse(renderJev("classify.jev", item)) as { state: string };
assert.equal(JSON.parse(req.state).files_changed, 2);
});
});
describe("settle without Jev", () => {
test("a Haiku verdict alone settles the item with Haiku's label and flags", () => {
const s = settle(null, { label: "bugfix", flags: ["security"] });
assert.equal(s.status, "settled");
assert.equal(s.source, "haiku");
assert.equal(s.label, "bugfix");
assert.deepEqual(s.flags, ["security"]);
assert.equal(settle(null, null).status, "unclassified");
});
});