mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 19:44:48 +01:00
Merge remote-tracking branch 'origin/garrytan/opus55-wip-E2' into garrytan/opus-5-5-prompt-audit
This commit is contained in:
commit
6ba9fa30e4
16 files changed
+200
-187
No files matched your search
@@ -1,19 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { auqMachineFormatProblems, scoreAuqFormat } from './helpers/auq-sdk-capture';
|
||||
|
||||
const briefs = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/auq-matrix/briefs.json'), 'utf8')) as
|
||||
{ known_good: Record<string, string>; known_bad: Record<string, string> };
|
||||
|
||||
describe('AUQ matrix format gate', () => {
|
||||
test.each(Object.entries(briefs.known_good))('passes %s', (_name, text) => {
|
||||
expect(auqMachineFormatProblems(text)).toEqual([]);
|
||||
});
|
||||
test.each(Object.entries(briefs.known_bad))('fails %s', (_name, text) => {
|
||||
expect(auqMachineFormatProblems(text).length).toBeGreaterThan(0);
|
||||
});
|
||||
test('a brief missing only presentation elements no longer fails, but they are still reported', () => {
|
||||
expect(scoreAuqFormat(briefs.known_good['no-emoji-or-net']!).missing).toEqual(['Pros / cons:', '✅', '❌', 'Net:']);
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { serializeNativeAuq, displayedNativeAuq, nativeAuqPublicError, nativeAuqViewport, NATIVE_AUQ_CAPTURE_MS } from './helpers/auq-native-capture';
|
||||
import { scoreAuqFormat } from './helpers/auq-sdk-capture';
|
||||
import { auqMachineFormatProblems, scoreAuqFormat } from './helpers/auq-sdk-capture';
|
||||
import { createPendingQuestionRecorder, recordPendingQuestion, readFirstPendingQuestionForDisplay } from './helpers/plan-count-pending-question';
|
||||
import type { NativePlanQuestion, NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
|
||||
@@ -632,3 +632,21 @@ fs.writeFileSync(path.join(root,'results.json'),JSON.stringify({results,privateR
|
||||
expect(out).toContain('outcome=question_captured workflowCompleted=false');
|
||||
} finally {fs.rmSync(dir,{recursive:true,force:true});}
|
||||
},30_000);
|
||||
|
||||
// Stored first-AUQ briefs for the matrix's machine-read format gate.
|
||||
{
|
||||
const briefs = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/auq-matrix/briefs.json'), 'utf8')) as
|
||||
{ known_good: Record<string, string>; known_bad: Record<string, string> };
|
||||
|
||||
describe('AUQ matrix format gate', () => {
|
||||
test.each(Object.entries(briefs.known_good))('passes %s', (_name, text) => {
|
||||
expect(auqMachineFormatProblems(text)).toEqual([]);
|
||||
});
|
||||
test.each(Object.entries(briefs.known_bad))('fails %s', (_name, text) => {
|
||||
expect(auqMachineFormatProblems(text).length).toBeGreaterThan(0);
|
||||
});
|
||||
test('a brief missing only presentation elements no longer fails, but they are still reported', () => {
|
||||
expect(scoreAuqFormat(briefs.known_good['no-emoji-or-net']!).missing).toEqual(['Pros / cons:', '✅', '❌', 'Net:']);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { browseJudgeFloorsMet } from './helpers/workflow-judge-cache';
|
||||
|
||||
type Scores = { clarity: number; completeness: number; actionability: number };
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/browse-judge/panel-means.json'), 'utf8')) as
|
||||
{ known_good: Record<string, Scores>; known_bad: Record<string, Scores> };
|
||||
|
||||
describe('browse judge floors', () => {
|
||||
test.each(Object.entries(stored.known_good))('passes %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(true));
|
||||
test.each(Object.entries(stored.known_bad))('fails %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(false));
|
||||
});
|
||||
@@ -1,49 +0,0 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import type { AskUserQuestionFingerprint } from './helpers/claude-pty-runner';
|
||||
import type { NativeQuestion } from './helpers/plan-skill-questions';
|
||||
import { CEO_SCOPE_CANDIDATES } from './helpers/plan-review-cases';
|
||||
import { CEO_SPLIT_CALL_FLOOR } from './helpers/ceo-split-question-policy';
|
||||
import { validatePlanReviewDecisionResponse, type PlanReviewDecision, type PlanReviewDecisionInput } from './helpers/plan-review-decisions';
|
||||
|
||||
const PLATFORMS = ['Slack', 'Discord', 'Microsoft Teams', 'Telegram', 'Mattermost'];
|
||||
const question = (platform: string, n: number): NativeQuestion => ({
|
||||
header: platform, multiSelect: false,
|
||||
question: `D3.${n} — ${platform} integration\nELI10: Decide this integration independently. Recommendation: Include.`,
|
||||
options: ['Include', 'Defer', 'Cut', 'Hold'].map(label => ({ label, description: `Choose ${label} for ${platform}.` })),
|
||||
});
|
||||
type NativeCall = AskUserQuestionFingerprint & { toolUseId: string; questions: NativeQuestion[] };
|
||||
const call = (id: string, questions: NativeQuestion[]) => ({
|
||||
toolUseId: id, questions, selectedOptions: questions.map(() => 1), signature: id,
|
||||
promptSnippet: 'diagnostic', options: [], observedAtMs: 1, preReview: true,
|
||||
}) as unknown as NativeCall;
|
||||
const row = (fp: NativeCall, tab: number, target: string): PlanReviewDecision => ({
|
||||
toolUseId: fp.toolUseId, questionIndex: tab, kind: 'scope', targetIds: [target], independentDecisions: 1,
|
||||
evidence: [{ field: 'question', optionIndex: null, quote: fp.questions[tab - 1]!.question.split('\n')[0]! }],
|
||||
reason: 'This acknowledged question presents the independent decision described by this target.',
|
||||
optionActions: (['include', 'defer', 'cut', 'hold'] as const).map((action, i) => ({ optionIndex: i + 1, action })),
|
||||
});
|
||||
/** Stored shapes: each inner array is one AskUserQuestion call; numbers index PLATFORMS. */
|
||||
function run(groups: number[][]) {
|
||||
const fingerprints = groups.map((group, i) => call(`native-${i}`, group.map(p => question(PLATFORMS[p]!, p + 1))));
|
||||
const input: PlanReviewDecisionInput = { plan: 'Review the five independent integrations in this plan.', kind: 'scope',
|
||||
targets: CEO_SCOPE_CANDIDATES, fingerprints, floor: CEO_SPLIT_CALL_FLOOR, deadlineAt: Date.now() + 60_000 };
|
||||
const questions = groups.flatMap((group, i) => group.map((p, tab) => row(fingerprints[i]!, tab + 1, `E${p + 1}`)));
|
||||
return () => validatePlanReviewDecisionResponse(input, { questions });
|
||||
}
|
||||
|
||||
test('the split floor is the minimum number of calls for five options at four per call', () => {
|
||||
expect(CEO_SPLIT_CALL_FLOOR).toBe(Math.ceil(PLATFORMS.length / 4));
|
||||
});
|
||||
test.each([
|
||||
['five sequential calls', [[0], [1], [2], [3], [4]]],
|
||||
['two batched calls', [[0, 1, 2, 3], [4]]],
|
||||
['three batched calls', [[0, 1], [2, 3], [4]]],
|
||||
])('known-good split passes: %s', (_name, groups) => {
|
||||
expect(run(groups)().coveredTargetIds).toEqual(CEO_SCOPE_CANDIDATES.map(c => c.id));
|
||||
});
|
||||
test.each([
|
||||
['a candidate silently dropped', [[0, 1, 2, 3]], 'missing target decisions'],
|
||||
['one call for a dropped fifth candidate', [[0, 1, 2], [3]], 'missing target decisions'],
|
||||
])('known-bad split fails: %s', (_name, groups, message) => {
|
||||
expect(run(groups)).toThrow(message);
|
||||
});
|
||||
@@ -2,11 +2,19 @@ import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { KIND_NOTE_RE } from './helpers/plan-format-kind-note';
|
||||
import { generateAskUserFormat } from '../scripts/resolvers/preamble/generate-ask-user-format';
|
||||
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
||||
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/plan-format-kind-notes.json'), 'utf8')) as
|
||||
{ known_good: string[]; known_bad: string[] };
|
||||
|
||||
describe('plan-format kind note', () => {
|
||||
test('accepts the kind note the AskUserQuestion format itself prescribes', () => {
|
||||
const format = generateAskUserFormat({ skillName: 'plan-ceo-review', host: 'claude', paths: HOST_PATHS.claude } as TemplateContext);
|
||||
const prescribed = /Note: options differ in kind[^`\n]*/.exec(format)?.[0];
|
||||
expect(prescribed).toBeDefined();
|
||||
expect(prescribed!).toMatch(KIND_NOTE_RE);
|
||||
});
|
||||
test.each(stored.known_good)('accepts %s', line => expect(line).toMatch(KIND_NOTE_RE));
|
||||
test.each(stored.known_bad)('rejects %s', line => expect(line).not.toMatch(KIND_NOTE_RE));
|
||||
});
|
||||
@@ -1,8 +1,10 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { assertNoPlanFileDecisions, seededPlanTargeted } from './helpers/plan-mode-evidence';
|
||||
import { planFileHasDecisionsSection, type PlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const SECRET = 'fixture-private-diagnostic-value';
|
||||
@@ -112,3 +114,40 @@ await import(${JSON.stringify(path.join(ROOT, PAID_FILE))});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Seeded-plan targeting and plan-file decision substitutes (stored observations).
|
||||
{
|
||||
const TOKENS = ['ShardManager', 'ResultMerger'];
|
||||
const base = { outcome: 'asked', summary: '', evidence: '', elapsedMs: 1 } as unknown as PlanSkillObservation;
|
||||
const targetingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-mode-targeting-'));
|
||||
afterAll(() => fs.rmSync(targetingDir, { recursive: true, force: true }));
|
||||
const plan = (name: string, body: string) => { const file = path.join(targetingDir, name); fs.writeFileSync(file, body); return file; };
|
||||
|
||||
describe('seeded plan targeting', () => {
|
||||
test.each([
|
||||
['exact announcement observed', { scopeGateAutoSelectObserved: true }],
|
||||
['paraphrased announcement with seed content in the review', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: true, ResultMerger: false } }],
|
||||
])('known-good: %s', (_name, fields) => {
|
||||
expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(true);
|
||||
});
|
||||
test.each([
|
||||
['no announcement and no seed content', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: false, ResultMerger: false } }],
|
||||
['an untracked token cannot stand in for the seed', { scopeGateAutoSelectObserved: false, tokensObserved: { 'draft-plan': true } }],
|
||||
])('known-bad: %s', (_name, fields) => {
|
||||
expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('plan-file decision substitute', () => {
|
||||
test('a plan_ready plan file with ## Decisions fails', () => {
|
||||
const planFile = plan('decisions.md', '# Plan\n\n## Decisions to confirm\n- A or B?\n\n## GSTACK REVIEW REPORT\n');
|
||||
expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection))
|
||||
.toThrow('not written to the plan as a substitute');
|
||||
});
|
||||
test('a plan_ready plan file without ## Decisions passes, and asked runs are not checked', () => {
|
||||
const planFile = plan('clean.md', '# Plan\n\n## Approach\nShip it.\n\n## GSTACK REVIEW REPORT\n');
|
||||
expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow();
|
||||
expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'asked' } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow();
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { afterAll, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { assertNoPlanFileDecisions, seededPlanTargeted } from './helpers/plan-mode-evidence';
|
||||
import { planFileHasDecisionsSection } from './helpers/claude-pty-runner';
|
||||
import type { PlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
|
||||
const TOKENS = ['ShardManager', 'ResultMerger'];
|
||||
const base = { outcome: 'asked', summary: '', evidence: '', elapsedMs: 1 } as unknown as PlanSkillObservation;
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan-mode-targeting-'));
|
||||
afterAll(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
const plan = (name: string, body: string) => { const file = path.join(dir, name); fs.writeFileSync(file, body); return file; };
|
||||
|
||||
describe('seeded plan targeting', () => {
|
||||
test.each([
|
||||
['exact announcement observed', { scopeGateAutoSelectObserved: true }],
|
||||
['paraphrased announcement with seed content in the review', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: true, ResultMerger: false } }],
|
||||
])('known-good: %s', (_name, fields) => {
|
||||
expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(true);
|
||||
});
|
||||
test.each([
|
||||
['no announcement and no seed content', { scopeGateAutoSelectObserved: false, tokensObserved: { ShardManager: false, ResultMerger: false } }],
|
||||
['an untracked token cannot stand in for the seed', { scopeGateAutoSelectObserved: false, tokensObserved: { 'draft-plan': true } }],
|
||||
])('known-bad: %s', (_name, fields) => {
|
||||
expect(seededPlanTargeted({ ...base, ...fields } as PlanSkillObservation, TOKENS)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('plan-file decision substitute', () => {
|
||||
test('a plan_ready plan file with ## Decisions fails', () => {
|
||||
const planFile = plan('decisions.md', '# Plan\n\n## Decisions to confirm\n- A or B?\n\n## GSTACK REVIEW REPORT\n');
|
||||
expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection))
|
||||
.toThrow('not written to the plan as a substitute');
|
||||
});
|
||||
test('a plan_ready plan file without ## Decisions passes, and asked runs are not checked', () => {
|
||||
const planFile = plan('clean.md', '# Plan\n\n## Approach\nShip it.\n\n## GSTACK REVIEW REPORT\n');
|
||||
expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'plan_ready', planFile } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow();
|
||||
expect(() => assertNoPlanFileDecisions({ ...base, outcome: 'asked' } as PlanSkillObservation, planFileHasDecisionsSection)).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -1852,3 +1852,55 @@ describe('DX checkpoint optional TODO actor', () => {
|
||||
expect(() => pickDevexCheckpointQuestion(ambiguous)).toThrow('multiple recommended');
|
||||
});
|
||||
});
|
||||
|
||||
// CEO split-overflow call floor: stored call shapes for the scope validator.
|
||||
{
|
||||
const { CEO_SCOPE_CANDIDATES } = await import('./helpers/plan-review-cases');
|
||||
const { CEO_SPLIT_CALL_FLOOR } = await import('./helpers/ceo-split-question-policy');
|
||||
const { validatePlanReviewDecisionResponse } = await import('./helpers/plan-review-decisions');
|
||||
type PlanReviewDecision = import('./helpers/plan-review-decisions').PlanReviewDecision;
|
||||
type PlanReviewDecisionInput = import('./helpers/plan-review-decisions').PlanReviewDecisionInput;
|
||||
type AskUserQuestionFingerprint = import('./helpers/claude-pty-runner').AskUserQuestionFingerprint;
|
||||
const PLATFORMS = ['Slack', 'Discord', 'Microsoft Teams', 'Telegram', 'Mattermost'];
|
||||
const question = (platform: string, n: number): NativeQuestion => ({
|
||||
header: platform, multiSelect: false,
|
||||
question: `D3.${n} — ${platform} integration\nELI10: Decide this integration independently. Recommendation: Include.`,
|
||||
options: ['Include', 'Defer', 'Cut', 'Hold'].map(label => ({ label, description: `Choose ${label} for ${platform}.` })),
|
||||
});
|
||||
type NativeCall = AskUserQuestionFingerprint & { toolUseId: string; questions: NativeQuestion[] };
|
||||
const call = (id: string, questions: NativeQuestion[]) => ({
|
||||
toolUseId: id, questions, selectedOptions: questions.map(() => 1), signature: id,
|
||||
promptSnippet: 'diagnostic', options: [], observedAtMs: 1, preReview: true,
|
||||
}) as unknown as NativeCall;
|
||||
const row = (fp: NativeCall, tab: number, target: string): PlanReviewDecision => ({
|
||||
toolUseId: fp.toolUseId, questionIndex: tab, kind: 'scope', targetIds: [target], independentDecisions: 1,
|
||||
evidence: [{ field: 'question', optionIndex: null, quote: fp.questions[tab - 1]!.question.split('\n')[0]! }],
|
||||
reason: 'This acknowledged question presents the independent decision described by this target.',
|
||||
optionActions: (['include', 'defer', 'cut', 'hold'] as const).map((action, i) => ({ optionIndex: i + 1, action })),
|
||||
});
|
||||
/** Stored shapes: each inner array is one AskUserQuestion call; numbers index PLATFORMS. */
|
||||
function run(groups: number[][]) {
|
||||
const fingerprints = groups.map((group, i) => call(`native-${i}`, group.map(p => question(PLATFORMS[p]!, p + 1))));
|
||||
const input: PlanReviewDecisionInput = { plan: 'Review the five independent integrations in this plan.', kind: 'scope',
|
||||
targets: CEO_SCOPE_CANDIDATES, fingerprints, floor: CEO_SPLIT_CALL_FLOOR, deadlineAt: Date.now() + 60_000 };
|
||||
const questions = groups.flatMap((group, i) => group.map((p, tab) => row(fingerprints[i]!, tab + 1, `E${p + 1}`)));
|
||||
return () => validatePlanReviewDecisionResponse(input, { questions });
|
||||
}
|
||||
|
||||
test('the split floor is the minimum number of calls for five options at four per call', () => {
|
||||
expect(CEO_SPLIT_CALL_FLOOR).toBe(Math.ceil(PLATFORMS.length / 4));
|
||||
});
|
||||
test.each([
|
||||
['five sequential calls', [[0], [1], [2], [3], [4]]],
|
||||
['two batched calls', [[0, 1, 2, 3], [4]]],
|
||||
['three batched calls', [[0, 1], [2, 3], [4]]],
|
||||
])('known-good split passes: %s', (_name, groups) => {
|
||||
expect(run(groups)().coveredTargetIds).toEqual(CEO_SCOPE_CANDIDATES.map(c => c.id));
|
||||
});
|
||||
test.each([
|
||||
['a candidate silently dropped', [[0, 1, 2, 3]], 'missing target decisions'],
|
||||
['one call for a dropped fifth candidate', [[0, 1, 2], [3]], 'missing target decisions'],
|
||||
])('known-bad split fails: %s', (_name, groups, message) => {
|
||||
expect(run(groups)).toThrow(message);
|
||||
});
|
||||
}
|
||||
@@ -1755,3 +1755,23 @@ describe('skip actor: deferred-reuse wording in a Skip option (PR lane run 36641
|
||||
await expect(answer('Records an explicit Skip so a future review can reuse the helper once snapshot coverage holds.')).rejects.toThrow('No unambiguous no-change option');
|
||||
});
|
||||
});
|
||||
|
||||
// Stored gh/curl endpoint lists for the PR coverage request budget.
|
||||
{
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs/pr-coverage-endpoints.json'), 'utf8')) as
|
||||
{ known_good: Record<string, string[]>; known_bad: Record<string, string[]> };
|
||||
|
||||
describe('shared-libs PR coverage request budget', () => {
|
||||
test.each(Object.entries(stored.known_good))('passes %s', (_name, endpoints) => {
|
||||
expect(prCoverageRequestViolations(endpoints)).toEqual([]);
|
||||
});
|
||||
test.each(Object.entries(stored.known_bad))('fails %s', (_name, endpoints) => {
|
||||
expect(prCoverageRequestViolations(endpoints).length).toBeGreaterThan(0);
|
||||
});
|
||||
test('file-list pages beyond the stated budget fail', () => {
|
||||
const pages = Array.from({ length: 52 }, (_, i) => `/repos/fixture/shared-libs/pulls/${i < 2 ? 42 : 100 + i}/files?per_page=100&page=${i < 2 ? i + 1 : 1}`);
|
||||
expect(prCoverageRequestViolations(pages)).toEqual(['52 file-list pages exceed the budget of 50 plus PR 7']);
|
||||
expect(prCoverageRequestViolations(pages.slice(0, 51))).toEqual([]);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { prCoverageRequestViolations } from './helpers/shared-libs-eval-fixture';
|
||||
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/shared-libs/pr-coverage-endpoints.json'), 'utf8')) as
|
||||
{ known_good: Record<string, string[]>; known_bad: Record<string, string[]> };
|
||||
|
||||
describe('shared-libs PR coverage request budget', () => {
|
||||
test.each(Object.entries(stored.known_good))('passes %s', (_name, endpoints) => {
|
||||
expect(prCoverageRequestViolations(endpoints)).toEqual([]);
|
||||
});
|
||||
test.each(Object.entries(stored.known_bad))('fails %s', (_name, endpoints) => {
|
||||
expect(prCoverageRequestViolations(endpoints).length).toBeGreaterThan(0);
|
||||
});
|
||||
test('file-list pages beyond the stated budget fail', () => {
|
||||
const pages = Array.from({ length: 52 }, (_, i) => `/repos/fixture/shared-libs/pulls/${i < 2 ? 42 : 100 + i}/files?per_page=100&page=${i < 2 ? i + 1 : 1}`);
|
||||
expect(prCoverageRequestViolations(pages)).toEqual(['52 file-list pages exceed the budget of 50 plus PR 7']);
|
||||
expect(prCoverageRequestViolations(pages.slice(0, 51))).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -11,6 +11,13 @@ const correct = (output: string) => {
|
||||
};
|
||||
|
||||
describe('ship triage label grader', () => {
|
||||
test('grades the two labels the ship triage step itself uses', () => {
|
||||
const triage = fs.readFileSync(path.join(import.meta.dir, '..', 'ship', 'sections', 'tests.md.tmpl'), 'utf8');
|
||||
expect(triage).toMatch(/\bin-branch\b/i);
|
||||
expect(triage).toMatch(/\bpre-existing\b/i);
|
||||
expect(triageLabels('- truncate (string.test.js): in-branch\n- divide (math.test.js): pre-existing'))
|
||||
.toEqual({ string: 'in-branch', math: 'pre-existing' });
|
||||
});
|
||||
test.each(['known-good-json.txt', 'known-good-lines.txt'])('accepts %s', (name) => {
|
||||
expect(correct(read(name))).toBe(true);
|
||||
});
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { hasNarrationLeak } from './helpers/skill-body-narration';
|
||||
|
||||
const DIR = path.join(import.meta.dir, 'fixtures', 'skillify-narration');
|
||||
const read = (name: string) => fs.readFileSync(path.join(DIR, name), 'utf8');
|
||||
const KNOWN_BAD = fs.readdirSync(DIR).filter(name => name.startsWith('known-bad-')).sort();
|
||||
|
||||
describe('skillify narration grader', () => {
|
||||
test('accepts a clean body that says "let me know" in a trigger', () => {
|
||||
expect(hasNarrationLeak(read('known-good.md'))).toBe(false);
|
||||
});
|
||||
test('stored known-bad fixtures exist', () => {
|
||||
expect(KNOWN_BAD.length).toBeGreaterThanOrEqual(4);
|
||||
});
|
||||
test.each(KNOWN_BAD)('still rejects %s', (name) => {
|
||||
expect(hasNarrationLeak(read(name))).toBe(true);
|
||||
});
|
||||
test('the previous grader rejected every known-bad fixture and also the known-good one', () => {
|
||||
const previous = (body: string) => /^I /m.test(body) || /Let me /i.test(body) || /^I'll /m.test(body);
|
||||
for (const name of KNOWN_BAD) expect(previous(read(name))).toBe(true);
|
||||
expect(previous(read('known-good.md'))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
} from './helpers/skill-fixture';
|
||||
import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
import { readShippedSkillRouting } from './helpers/shipped-skill-routing';
|
||||
import { hasNarrationLeak } from './helpers/skill-body-narration';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
@@ -482,3 +483,27 @@ test('routing fixtures read the ## Skill routing section gstack ships', () => {
|
||||
expect(() => readShippedSkillRouting(root)).toThrow('routing-injection heredoc not found');
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// Stored synthesized SKILL.md bodies for the skillify narration grader.
|
||||
{
|
||||
const DIR = path.join(import.meta.dir, 'fixtures', 'skillify-narration');
|
||||
const read = (name: string) => fs.readFileSync(path.join(DIR, name), 'utf8');
|
||||
const KNOWN_BAD = fs.readdirSync(DIR).filter(name => name.startsWith('known-bad-')).sort();
|
||||
|
||||
describe('skillify narration grader', () => {
|
||||
test('accepts a clean body that says "let me know" in a trigger', () => {
|
||||
expect(hasNarrationLeak(read('known-good.md'))).toBe(false);
|
||||
});
|
||||
test('stored known-bad fixtures exist', () => {
|
||||
expect(KNOWN_BAD.length).toBeGreaterThanOrEqual(4);
|
||||
});
|
||||
test.each(KNOWN_BAD)('still rejects %s', (name) => {
|
||||
expect(hasNarrationLeak(read(name))).toBe(true);
|
||||
});
|
||||
test('the previous grader rejected every known-bad fixture and also the known-good one', () => {
|
||||
const previous = (body: string) => /^I /m.test(body) || /Let me /i.test(body) || /^I'll /m.test(body);
|
||||
for (const name of KNOWN_BAD) expect(previous(read(name))).toBe(true);
|
||||
expect(previous(read('known-good.md'))).toBe(true);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
RETENTION_ONE_LINER, RETIREMENT_FIELDS, REVIEW_EVIDENCE_FIELDS, SWEEP_POINTER, TEST_VALUE_BAR_MAX_BYTES, TEST_VALUE_BAR_MODES,
|
||||
WEAK_REASONS, clampCardField, generateTestValueBar, renderValueCard, type TestValueBarMode,
|
||||
} from '../scripts/resolvers/test-value';
|
||||
import { recordsNonTestCallerSearch } from './helpers/test-value-fixture';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const FIX = 'Fix: bun run gen:skill-docs && bun test test/test-value-bar.test.ts';
|
||||
@@ -239,3 +240,18 @@ describe('docs and registration', () => {
|
||||
expect(read('README.md').split('/deslop-shared-libs, /test-audit, /ship').length - 1).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// Stored test-only-export evidence for the review test-value caller-search grader.
|
||||
{
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/test-value-seam-evidence.json'), 'utf8')) as
|
||||
{ known_good: Record<string, unknown>; known_bad: Record<string, unknown> };
|
||||
|
||||
describe('test-only export caller-search evidence', () => {
|
||||
test.each(Object.entries(stored.known_good))('accepts %s', (_name, evidence) => {
|
||||
expect(recordsNonTestCallerSearch({ evidence })).toBe(true);
|
||||
});
|
||||
test.each(Object.entries(stored.known_bad))('rejects %s', (_name, evidence) => {
|
||||
expect(recordsNonTestCallerSearch({ evidence })).toBe(false);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { recordsNonTestCallerSearch } from './helpers/test-value-fixture';
|
||||
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/test-value-seam-evidence.json'), 'utf8')) as
|
||||
{ known_good: Record<string, unknown>; known_bad: Record<string, unknown> };
|
||||
|
||||
describe('test-only export caller-search evidence', () => {
|
||||
test.each(Object.entries(stored.known_good))('accepts %s', (_name, evidence) => {
|
||||
expect(recordsNonTestCallerSearch({ evidence })).toBe(true);
|
||||
});
|
||||
test.each(Object.entries(stored.known_bad))('rejects %s', (_name, evidence) => {
|
||||
expect(recordsNonTestCallerSearch({ evidence })).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -8,7 +8,7 @@ import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { prepareWorkflowJudgeCache, validWorkflowJudgePanel, validWorkflowJudgeScore, workflowJudgeDependencies, type WorkflowCacheOptions } from './helpers/workflow-judge-cache';
|
||||
import { browseJudgeFloorsMet, prepareWorkflowJudgeCache, validWorkflowJudgePanel, validWorkflowJudgeScore, workflowJudgeDependencies, type WorkflowCacheOptions } from './helpers/workflow-judge-cache';
|
||||
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from './helpers/workflow-judge-input';
|
||||
|
||||
const roots: string[] = [];
|
||||
@@ -577,3 +577,15 @@ describe('judge panel', () => {
|
||||
expect(source).not.toMatch(/\bscores\.reasoning\b|\bresult\.reasoning\b/);
|
||||
});
|
||||
});
|
||||
|
||||
// Stored browse reference panel means against the judge floors.
|
||||
{
|
||||
type Scores = { clarity: number; completeness: number; actionability: number };
|
||||
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/browse-judge/panel-means.json'), 'utf8')) as
|
||||
{ known_good: Record<string, Scores>; known_bad: Record<string, Scores> };
|
||||
|
||||
describe('browse judge floors', () => {
|
||||
test.each(Object.entries(stored.known_good))('passes %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(true));
|
||||
test.each(Object.entries(stored.known_bad))('fails %s', (_name, scores) => expect(browseJudgeFloorsMet(scores)).toBe(false));
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user