Files
discord-github-ticket-relay/test/pull-requests.test.ts

314 lines
19 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { afterAll, beforeEach, describe, expect, it } from 'vitest';
import { IssueIndexer, setDuplicateOf } from '../src/domain/issue-index.js';
import { mentionedIssues, releaseOf } from '../src/domain/pull-index.js';
import { acceptSubmission } from '../src/domain/submissions.js';
import { getTriageSettings, parseTriageConfig, saveTriageConfig, setApiKey, setTriageEnabled } from '../src/triage/settings.js';
import { form, get, makeApp, makeWorkspace, resetDb, testDb, type ConfiguredWorkspace, type TestEnv } from './helpers.js';
/**
* Pull requests: kept from the issue sync, compared with tickets by AI triage (Being worked on,
* Already fixed), and listed in Repo status › Pull requests.
*/
const KEY = 'sk-ant-test-key-0123456789abcdef';
let env: TestEnv;
let ws: ConfiguredWorkspace;
let n = 0;
const wid = () => ws.workspace.id;
const w = () => `/w/${ws.workspace.id}`;
const ago = (days: number) => new Date(Date.now() - days * 86_400_000).toISOString();
const repo = (i = 0) => env.github.repos.get(ws.repoIds[i]!)!;
const issue = (num: number) => repo().issues.find((i) => i.number === num)!;
const result = async (id: string) => (await env.db.query('SELECT * FROM triage_results WHERE submission_id = $1', [id])).rows[0];
const lastSystem = () => env.anthropic.requests.at(-1)!.body.system[0].text as string;
const lastUser = () => env.anthropic.requests.at(-1)!.body.messages[0].content as string;
const pulls = async () => (await env.db.query('SELECT * FROM pull_index WHERE repo_id = $1 ORDER BY number', [repo().id])).rows;
const sync = async () => {
await env.db.query('UPDATE issue_index_repos SET sync_requested_at = now_ms()');
await new IssueIndexer(env.deps).tick();
};
beforeEach(async () => {
await resetDb();
env = await makeApp();
ws = await makeWorkspace(env, { apps: [{ name: 'PhotoCraft', repoIndex: 0 }] });
});
afterAll(async () => {
await testDb().end();
});
async function file(title: string, description: string) {
n++;
const r = await acceptSubmission(env.deps, {
workspace: ws.workspace, app: ws.apps[0]!, installationId: ws.installationId, source: 'discord', interactionId: `int-pr${n}`, interactionToken: null,
guildId: ws.guildId, guildName: 'G', channelId: ws.channelId, reporter: { id: `r${n}`, displayName: `Member ${n}`, username: `member${n}` },
title, description: `${description} (${n})`, ticketType: null,
});
if (r.kind !== 'accepted') throw new Error(r.kind);
await env.worker.drain();
return (await env.db.query('SELECT id, issue_number FROM submissions WHERE id = $1', [r.submission.id])).rows[0] as { id: string; issue_number: number };
}
async function enable(config: Record<string, unknown>) {
await setApiKey(env.db, env.deps.box, wid(), KEY, ws.owner.user.id);
await saveTriageConfig(env.db, wid(), parseTriageConfig(config));
await setTriageEnabled(env.db, wid(), true);
}
/** The issue #3592 asked for it, PR #3597 builds it; #3601 fixed a crash after the latest release; #3500 before it. */
function seed() {
const r = repo();
r.issues.push({ number: 3592, title: 'Free Transform across several selected layers', body: 'Let me transform several layers at once.', labels: ['enhancement'], created_at: ago(30), user: 'someone' });
r.issues.push({ number: 3590, title: 'Crash when exporting a spread to PDF', body: 'The app closes.', labels: ['bug'], created_at: ago(40), user: 'someone', state: 'closed', state_reason: 'completed', closed_at: ago(2) });
r.pulls = [
{ number: 3597, title: 'Free Transform for several layers', body: 'Implements the request in #3592.', user: 'FredApps', created_at: ago(5), updated_at: ago(2) },
{ number: 3598, title: 'WIP: faster brush engine', body: 'Not ready.', user: 'ana', created_at: ago(3), updated_at: ago(1), draft: true },
{ number: 3601, title: 'Fix the crash when exporting spreads to PDF', body: 'Fixes #3590', user: 'ben', created_at: ago(4), merged_at: ago(2), updated_at: ago(2) },
{ number: 3500, title: 'Fix ruler units in preferences', body: 'Closes #3400', user: 'cy', created_at: ago(20), merged_at: ago(15), updated_at: ago(15) },
{ number: 3599, title: 'Bump lodash from 4.17.20 to 4.17.21', body: '', user: 'dependabot[bot]', created_at: ago(2), updated_at: ago(2) },
{ number: 3200, title: 'Old merged change', body: '', user: 'cy', created_at: ago(200), merged_at: ago(150), updated_at: ago(150) },
{ number: 3100, title: 'Abandoned idea', body: '', user: 'cy', created_at: ago(220), merged_at: null, updated_at: ago(160) },
];
r.releases = [{ tag: 'v1.4.3', published_at: ago(10) }, { tag: 'v1.4.2', published_at: ago(30) }];
}
describe('Pull requests: read from GitHub', () => {
it('keeps open and recently merged pull requests, what they’re for, and the releases', async () => {
seed();
await sync();
const got = await pulls();
// Merged ones are kept for good (contributor roles count them); ones closed without merging only for 90 days.
expect(got.map((p) => p.number)).toEqual([3200, 3500, 3597, 3598, 3599, 3601]);
const p = Object.fromEntries(got.map((x) => [x.number, x]));
expect(p[3597]).toMatchObject({ state: 'open', draft: false, author: 'FredApps', closes: [], mentions: [3592], noise: null });
expect(p[3598]).toMatchObject({ draft: true });
expect(p[3601]).toMatchObject({ state: 'closed', closes: [3590], merged_at: expect.any(Date) });
expect(p[3599]!.noise).toBe('Bot');
// The issues stay issues only.
const issues = (await env.db.query('SELECT number FROM issue_index WHERE repo_id = $1', [repo().id])).rows.map((r) => r.number);
expect(issues).not.toContain(3597);
const releases = JSON.parse((await env.db.query('SELECT releases FROM issue_index_repos WHERE repo_id = $1', [repo().id])).rows[0].releases);
expect(releases.map((x: { tag: string }) => x.tag)).toEqual(['v1.4.3', 'v1.4.2']);
});
it('reads them from the pull request list: without Pull requests: Read for a public repository, and says so for a private one', async () => {
// Like the GitHub App in production: issues and metadata only, so the issue list has no pull requests.
env.github.installationPermissions = { issues: 'write', metadata: 'read' };
seed();
repo().private = false;
await sync();
expect((await pulls()).map((p) => p.number)).toEqual([3200, 3500, 3597, 3598, 3599, 3601]);
const row = async () => (await env.db.query('SELECT * FROM issue_index_repos WHERE repo_id = $1', [repo().id])).rows[0];
expect(await row()).toMatchObject({ pulls_error: null, pulls_loaded_at: expect.any(Date) });
// A private repository's pull requests can't be listed without the permission: the tab says so.
repo().private = true;
await env.db.query('DELETE FROM pull_index');
await env.db.query('UPDATE issue_index_repos SET pulls_loaded_at = NULL, pulls_checked_at = NULL');
await sync();
expect(await pulls()).toEqual([]);
expect((await row()).pulls_error).toBe('GitHub didn’t let the app read this repository’s pull requests. A private repository needs Pull requests: Read.');
// The issue sync isn't affected.
expect(await row()).toMatchObject({ status: 'ready', last_error: null });
});
it('after the catch-up, reads only what changed, every 10 minutes', async () => {
seed();
await sync();
const calls = env.github.pullListCalls;
repo().pulls!.push({ number: 3610, title: 'Snap guides to the pixel grid', body: '', user: 'eli', created_at: ago(0), updated_at: ago(0) });
await sync();
expect(env.github.pullListCalls).toBe(calls);
await env.db.query('UPDATE issue_index_repos SET pulls_checked_at = $1', [new Date(Date.now() - 11 * 60_000)]);
await sync();
expect(env.github.pullListCalls - calls).toBe(1);
expect((await pulls()).map((p) => p.number)).toContain(3610);
});
it('tries a failed catch-up again two minutes later, without breaking the issue sync', async () => {
seed();
env.github.failRequests.push({ match: /^GET \/repos\/[^/]+\/mobile\/pulls$/, status: 500, message: 'Server error' });
await sync();
let row = (await env.db.query('SELECT * FROM issue_index_repos WHERE repo_id = $1', [repo().id])).rows[0];
expect(row).toMatchObject({ status: 'ready', last_error: null, pulls_loaded_at: null, pulls_error: 'GitHub returned 500. Trying again shortly.' });
await sync();
expect(await pulls()).toEqual([]);
await env.db.query('UPDATE issue_index_repos SET pulls_checked_at = $1', [new Date(Date.now() - 3 * 60_000)]);
await sync();
row = (await env.db.query('SELECT * FROM issue_index_repos WHERE repo_id = $1', [repo().id])).rows[0];
expect(row).toMatchObject({ pulls_error: null, pulls_loaded_at: expect.any(Date) });
expect((await pulls()).length).toBe(6);
});
it('reads which issues a pull request names, and which release shipped a fix', () => {
expect(mentionedIssues('Free Transform for several layers\nImplements #3592, see also owner/repo#9 and x.com/a#5', 3597, [])).toEqual([3592]);
const releases = [{ tag: 'v3', at: '2026-10-15T00:00:00Z' }, { tag: 'v2', at: '2026-10-12T00:00:00Z' }];
expect(releaseOf(new Date('2026-10-10'), releases)?.tag).toBe('v2');
expect(releaseOf(new Date('2026-10-16'), releases)).toBeNull();
});
});
describe('AI triage: Being worked on and Already fixed', () => {
it('is off until turned on: Claude isn’t shown pull requests', async () => {
seed();
await sync();
await enable({ duplicate: { mode: 'suggest' } });
await file('Free transform on many layers', 'I want to transform several layers at once.');
await env.triage.drain();
expect(lastUser()).not.toContain('Pull requests and commits');
expect(lastSystem()).not.toContain('in_progress:');
expect(JSON.stringify(env.anthropic.requests.at(-1)!.body.output_config)).not.toContain('in_progress');
});
it('a duplicate whose issue a pull request is for: labels both and says which pull request', async () => {
seed();
await sync();
await enable({ duplicate: { mode: 'auto' }, inProgress: { mode: 'auto' }, fixed: { mode: 'suggest' } });
env.anthropic.queue.push({ ...env.anthropic.verdict, duplicates: [{ number: 3592, confidence: 95 }], in_progress: '#3597', in_progress_confidence: 93 });
const t = await file('Free transform on many layers', 'I want to transform several selected layers at once.');
await env.triage.drain();
// Claude saw the pull request, with the issue it's for, and was told the rules.
expect(lastUser()).toContain('"ref": "#3597"');
expect(lastUser()).toContain('"for": [\n 3592\n ]');
expect(lastUser()).not.toContain('#3599');
expect(lastSystem()).toContain('in_progress: the ref');
const i = issue(t.issue_number);
expect(i.labels.sort()).toEqual(['duplicate', 'in progress']);
expect(i.comments?.[0]).toContain('Possible duplicate of #3592.');
expect(i.comments?.[0]).toContain('This is being worked on in #3597.');
expect((await result(t.id)).verdict.in_progress_work).toMatchObject({ ref: '#3597', kind: 'pull', number: 3597 });
// The ticket page says so.
const page = (await env.server.inject({ method: 'GET', url: `${w()}/tickets/${t.id}`, ...get(ws.owner) })).body;
expect(page).toContain('Being worked on in <a href="https://github.com/');
});
it('a fix merged after the latest release: “it will be in the next release”, and no asking for details', async () => {
seed();
await sync();
await enable({ fixed: { mode: 'auto' }, needsInfo: { mode: 'auto' } });
env.anthropic.queue.push({ ...env.anthropic.verdict, fixed_by: '#3601', fixed_confidence: 94, needs_info: true, needs_info_confidence: 95, missing: ['steps'], duplicates: [{ number: 3590, confidence: 90 }] });
const t = await file('PDF export of a spread crashes', 'The app closes when I export a spread to PDF.');
await env.triage.drain();
const i = issue(t.issue_number);
expect(i.labels).toEqual(['fixed']);
expect(i.comments?.[0]).toBe('This was already fixed in #3601: it will be in the next release.');
expect(lastUser()).toContain('"release": "not released yet"');
});
it('a fix in a release names it; Claude can only pick what it was shown, in the right state', async () => {
seed();
repo().issues.push({ number: 3400, title: 'Ruler units reset after restart', body: 'Units go back to inches.', labels: ['bug'], created_at: ago(30), user: 'someone', state: 'closed', state_reason: 'completed', closed_at: ago(15) });
await sync();
await enable({ fixed: { mode: 'auto' }, inProgress: { mode: 'auto' } });
env.anthropic.queue.push({ ...env.anthropic.verdict, fixed_by: '3500', fixed_confidence: 92, duplicates: [{ number: 3400, confidence: 90 }] });
const t = await file('Ruler units reset after restart', 'Units go back to inches after restarting.');
await env.triage.drain();
expect(issue(t.issue_number).comments?.[0]).toBe('This was already fixed in #3500: it’s in v1.4.3.');
// An open pull request isn't a fix, a merged one isn't in progress, and one it wasn't shown is nothing.
env.anthropic.queue.push({ ...env.anthropic.verdict, fixed_by: '#3597', fixed_confidence: 95, in_progress: '#3601', in_progress_confidence: 95 });
const u = await file('Free transform on many layers', 'I want to transform several layers.');
await env.triage.drain();
env.anthropic.queue.push({ ...env.anthropic.verdict, in_progress: '#9999', in_progress_confidence: 95 });
const v = await file('Something else entirely', 'Unrelated.');
await env.triage.drain();
for (const x of [u, v]) {
expect(issue(x.issue_number).labels).toEqual([]);
expect((await result(x.id)).verdict).toMatchObject({ in_progress: null, fixed_by: null });
}
});
it('a regression is never “already fixed”', async () => {
seed();
await sync();
await enable({ fixed: { mode: 'auto' }, regression: { mode: 'auto' } });
env.anthropic.queue.push({ ...env.anthropic.verdict, regression_of: 3590, regression_confidence: 95, fixed_by: '#3601', fixed_confidence: 95, duplicates: [{ number: 3590, confidence: 90 }] });
const t = await file('PDF export still crashes after updating', 'Still closes on export in the latest build.');
await env.triage.drain();
expect(issue(t.issue_number).labels).toEqual(['regression']);
});
it('is on the Actions page, off, with its placeholders', async () => {
const body = (await env.server.inject({ method: 'GET', url: `${w()}/triage/actions`, ...get(ws.owner) })).body;
expect(body).toContain('Being worked on');
expect(body).toContain('Already fixed');
expect(body).toContain('{fix}, {release}, {app}');
expect((await getTriageSettings(env.db, wid())).config).toMatchObject({ inProgress: { mode: 'off', label: 'in progress' }, fixed: { mode: 'off', label: 'fixed' } });
// Saving turns it on, with its own label and comment.
await setApiKey(env.db, env.deps.box, wid(), KEY, ws.owner.user.id);
const res = await env.server.inject({
method: 'POST', url: `${w()}/triage/actions`,
...form(ws.owner, {
model: 'claude-haiku-5-5', threshold: '90', monthly_limit: '500', priority_label_critical: 'a', priority_label_high: 'b', priority_label_medium: 'c', priority_label_low: 'd',
app_ids: ws.apps[0]!.id, inProgress_mode: 'suggest', inProgress_label: 'wip', inProgress_comment: 'Someone is on it: {pull}.', fixed_mode: 'off',
}),
});
expect(res.statusCode).toBe(302);
expect((await getTriageSettings(env.db, wid())).config.inProgress).toMatchObject({ mode: 'suggest', label: 'wip', comment: 'Someone is on it: {pull}.', close: false });
});
});
describe('Repo status › Pull requests', () => {
async function turnOn() {
expect((await env.server.inject({ method: 'POST', url: `${w()}/repos/settings`, ...form(ws.owner, { enabled: '1', keep: '1' }) })).statusCode).toBe(302);
}
it('lists open ones (most reported first), merged ones waiting for a release, and merged ones', async () => {
seed();
await sync();
// Two Discord tickets about #3592: its own, and a duplicate.
const a = await file('Free transform for several layers please', 'Several layers at once.');
const b = await file('Transform multiple layers', 'Select many layers and transform them.');
await sync();
await setDuplicateOf(env.db, repo().id, a.issue_number, 3592);
await setDuplicateOf(env.db, repo().id, b.issue_number, 3592);
await turnOn();
const page = (url: string) => env.server.inject({ method: 'GET', url: `${w()}${url}`, ...get(ws.owner) }).then((r) => r.body);
const open = await page('/repos?view=pulls');
expect(open).toMatch(/Pull requests<span class="n">2<\/span>/);
expect(open).toContain('Free Transform for several layers');
expect(open).toContain('Draft');
expect(open).not.toContain('Bump lodash');
expect(open).toMatch(/For<\/span> <a href="[^"]+\/issues\/3592"[^>]*>#3592<\/a> <span class="rs-pr-it">Free Transform across several selected layers<\/span>/);
expect(open).toContain('2 Discord reports</span>');
expect(open.indexOf('Free Transform for several layers')).toBeLessThan(open.indexOf('faster brush engine'));
const waiting = await page('/repos?view=pulls&list=waiting');
expect(waiting).toContain('Fix the crash when exporting spreads to PDF');
expect(waiting).toContain('not released yet');
expect(waiting).not.toContain('Fix ruler units');
const merged = await page('/repos?view=pulls&list=merged');
expect(merged).toContain('Fix ruler units in preferences');
expect(merged).toContain('in <span class="mono">v1.4.3</span>');
// The list shows the last 90 days; older merged ones are kept for contributor roles only.
expect(merged).not.toContain('Old merged change');
// Most reported shows that someone's on it.
expect(await page('/repos?view=reported')).toContain('Open PR #3597');
});
it('says how far reading got, what went wrong, and when the background process is out of date', async () => {
seed();
await turnOn();
const page = () => env.server.inject({ method: 'GET', url: `${w()}/repos?view=pulls`, ...get(ws.owner) }).then((r) => r.body);
// Not read yet.
let body = await page();
expect(body).toContain('Reading pull requests');
expect(body).toContain('0 of 1 repositories done');
// Issues sync, but nothing has started on pull requests long after this version arrived: an old process.
await new IssueIndexer(env.deps).sync(repo().id, { pages: 5 }).catch(() => null);
await env.db.query("INSERT INTO issue_index_repos (repo_id, owner, name, installation_id, last_synced_at) VALUES ($1, $2, $3, $4, now_ms()) ON CONFLICT (repo_id) DO UPDATE SET last_synced_at = now_ms()", [repo().id, repo().owner, repo().name, ws.installationId]);
await env.db.query("UPDATE schema_migrations SET applied_at = $1 WHERE name = '034_pull_request_list.sql'", [new Date(Date.now() - 20 * 60_000)]);
body = await page();
expect(body).toContain('Pull requests aren’t being read');
expect(body).toContain('ticket-relay-triage');
// Read, with a problem.
await env.db.query('UPDATE issue_index_repos SET pulls_checked_at = now_ms(), pulls_error = $1', ['GitHub returned 502. Trying again shortly.']);
body = await page();
expect(body).not.toContain('Pull requests aren’t being read');
expect(body).toContain(`Couldn’t read the pull requests of ${repo().owner}/mobile`);
expect(body).toContain('GitHub returned 502');
// Done.
await env.db.query('UPDATE issue_index_repos SET pulls_loaded_at = now_ms(), pulls_error = NULL');
body = await page();
expect(body).not.toContain('Reading pull requests');
expect(body).not.toContain('Couldn’t read');
});
});