CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-10-04 02:16:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

test: retire the finding-count cluster and trim its helpers (C)

Browse Source 
- C0/C1: the five never-green evals (skill-e2e-autoplan-chain and
  skill-e2e-plan-{ceo,eng,design,devex}-finding-count) failed on harness and
  budget, never on skill behavior; delete them, their touchfile/tier ids,
  AUTOPLAN_CHAIN_BUDGET and the dedicated eighth periodic slice (--slices 7).
- C2: delete the helper groups whose only paid consumers were those files
  (11 modules), trim claude-pty-runner and eng-seeded-coverage to the paid
  closure, and delete the free replay tests whose assertions exercised only
  that dead code (89 files, 135 orphaned fixtures). Blocks that used dead code
  only as input for a live subject keep their assertions: the multiSelect
  default moved to plan-review-decisions, runner PTY tests use inline caller
  policies, and the timer-safe budget checks moved to eng-finding-retry-budget.
- The eight production-touching files stay except ceo-current-decision-record
  (its template read only feeds the retired counter).
- CARVE_GUARDS.autoplan is behavioral 'none'; TODOS records the lost chain
  and per-finding cadence coverage with their re-entry tests.
This commit is contained in:
garrytan committed 2026-09-29 06:08:49 +00:00
1 parent 53e7f3212f
commit 6415690a18
317 files changed
+315 -54111
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/eng-seeded-coverage.test.ts
+2 -91
View File
@@ -1,90 +1,7 @@
import fb10Public from './fixtures/eng-fb10-count-public.json';
import { describe, expect, test } from 'bun:test';
import type { NativePlanQuestionCall, PlanCountTranscript } from './helpers/plan-count-transcript';
import { ENG_DECISION_SEEDS, buildEngSeedDecisionInput, isEngBatchingIssueAUQ } from './helpers/eng-seeded-coverage';
import { buildPlanReviewDecisionPrompt, validatePlanReviewDecisionResponse } from './helpers/plan-review-decisions';
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
import { isEngBatchingIssueAUQ } from './helpers/eng-seeded-coverage';
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
import { E2E_TOUCHFILES, matchGlob } from './helpers/touchfiles';
describe('Eng semantic native evidence boundary', () => {
const start = Date.parse(fb10Public.windowStart), end = Date.parse(fb10Public.windowEnd);
const native = (): PlanCountTranscript => ({ status: 'ready', calls: structuredClone(fb10Public.calls) as NativePlanQuestionCall[], assistantMessages: [] });
const input = (t = native()) => buildEngSeedDecisionInput({ plan: fb10Public.plan, transcript: t,
startedAt: start, finishedAt: end, deadlineAt: Date.now() + 60_000 });
// This deliberately supplied response proves only local protocol checks.
// No model has classified this capture, and no paid result is inferred.
const response = (data = input()) => ({ questions: data.fingerprints.flatMap((fp, i) => fp.questions!.map((q, index) => ({
toolUseId: fp.toolUseId!, questionIndex: index + 1, kind: 'finding',
targetIds: ({ 0: ['sequential-idp'], 2: ['complexity'], 3: ['shared-cache'], 5: ['swallowed-errors'] } as Record<number, string[]>)[i] ?? [],
independentDecisions: 1, evidence: [{ field: 'question', optionIndex: null, quote: q.question.split('\n')[0]! }],
reason: 'Synthetic response for structural validation, not a semantic verdict.', optionActions: [],
}))) });
test('uses complete native fields and actual answers with all four independent targets', () => {
const t = native(), data = input(t);
expect(data.targets.map(target => target.id)).toEqual([...ENG_DECISION_SEEDS]);
expect(data.floor).toBe(4);
expect(data.ceiling).toBeUndefined();
expect(data.fingerprints).toHaveLength(t.calls.length);
data.fingerprints.forEach((fp, index) => {
const call = t.calls[index]!;
expect(fp.questions).toEqual(call.questions);
expect(fp.nativeCall).toEqual(call);
expect(fp.toolUseId).toBe(`${call.sessionId}:${call.toolUseId}`);
expect(fp.selectedOptions).toEqual(call.questions.map(q => q.options.findIndex(o => o.label === call.answers![q.question]) + 1));
});
expect(data.fingerprints[2]!.selectedOptions).toEqual([1]); // The actor kept all five classes, not recommended B.
expect(validatePlanReviewDecisionResponse(data, response(data)).coveredTargetIds).toEqual([...ENG_DECISION_SEEDS]);
const prompt = buildPlanReviewDecisionPrompt(data);
for (const fp of data.fingerprints) for (const q of fp.questions!) {
expect(prompt).toContain(JSON.stringify(q.question));
for (const o of q.options) expect(prompt).toContain(JSON.stringify(o.description));
}
});
test('takes an immutable snapshot before asynchronous classification', () => {
const t = native(), data = input(t), before = structuredClone(data);
t.calls[2]!.questions[0]!.question += '\nAltered after binding';
t.calls[2]!.answers = {};
expect(data).toEqual(before);
});
for (const [name, change] of [
['foreign session', (t: PlanCountTranscript) => { t.calls[2]!.sessionId = 'foreign'; }],
['duplicate native identity', (t: PlanCountTranscript) => { t.calls.push(structuredClone(t.calls[2]!)); }],
['unanswered call', (t: PlanCountTranscript) => { t.calls[2]!.answered = false; }],
['failed call', (t: PlanCountTranscript) => { t.calls[2]!.failed = true; }],
['unanswered tab', (t: PlanCountTranscript) => { t.calls[2]!.unansweredQuestionIndices = [0]; }],
['unoffered actual answer', (t: PlanCountTranscript) => { const c = t.calls[2]!; c.answers![c.questions[0]!.question] = 'Use a different option'; }],
['duplicate offered labels', (t: PlanCountTranscript) => { const q = t.calls[2]!.questions[0]!; q.options[1]!.label = q.options[0]!.label; }],
['out-of-window answer', (t: PlanCountTranscript) => { t.calls[2]!.answeredAt = new Date(start - 1).toISOString(); }],
['foreign extra answer', (t: PlanCountTranscript) => { t.calls[2]!.answers!['Foreign question'] = 'A'; }],
] as const) test(`rejects ${name} before a judge can run`, () => {
const t = native(); change(t); expect(() => input(t)).toThrow('complete owned');
});
for (const [name, change, error] of [
['missing row', (r: ReturnType<typeof response>) => { r.questions.pop(); }, 'missing native question rows'],
['duplicate row', (r: ReturnType<typeof response>) => { r.questions.push(structuredClone(r.questions[0]!)); }, 'duplicate native question'],
['foreign identity', (r: ReturnType<typeof response>) => { r.questions[2]!.toolUseId = 'foreign'; }, 'phantom'],
['wrong native quote', (r: ReturnType<typeof response>) => { r.questions[2]!.evidence[0]!.quote = r.questions[5]!.evidence[0]!.quote; }, 'exact native field'],
['missing seed', (r: ReturnType<typeof response>) => { r.questions[2]!.targetIds = []; }, 'missing target decisions'],
['bundled remedies', (r: ReturnType<typeof response>) => { r.questions[2]!.independentDecisions = 2; }, 'bundled independent decisions'],
['two seeds in one choice', (r: ReturnType<typeof response>) => { r.questions[2]!.targetIds.push('swallowed-errors'); r.questions[5]!.targetIds = []; }, 'bundled independent decisions'],
['uncertainty', (r: ReturnType<typeof response>) => { Object.assign(r.questions[2]!, { kind: 'uncertain', targetIds: [], independentDecisions: 0 }); }, 'uncertain classification'],
] as const) test(`rejects judge ${name}`, () => {
const data = input(), raw = response(data); change(raw);
expect(() => validatePlanReviewDecisionResponse(data, raw)).toThrow(error);
});
test('keeps the absolute deadline and does not grant a new judge window', () => {
const deadlineAt = Date.now() - 1;
const data = buildEngSeedDecisionInput({ plan: fb10Public.plan, transcript: native(), startedAt: start, finishedAt: end, deadlineAt });
expect(data.deadlineAt).toBe(deadlineAt);
expect(() => buildPlanReviewDecisionPrompt(data)).toThrow('absolute case deadline exhausted');
expect(() => buildEngSeedDecisionInput({ plan: fb10Public.plan, transcript: native(), startedAt: start, finishedAt: end, deadlineAt: end - 1 })).toThrow('original deadline');
});
});
function question(call: NativePlanQuestionCall, text: string) {
const answer = call.answers![call.questions[0]!.question]!;
@@ -92,12 +9,6 @@ function question(call: NativePlanQuestionCall, text: string) {
}
describe('Eng seeded coverage from completed native decisions', () => {
test('local evidence dependencies select both Eng consumers', () => {
for (const file of ['test/helpers/eng-seeded-coverage.ts', 'test/eng-seeded-coverage.test.ts']) {
expect(Object.entries(E2E_TOUCHFILES).filter(([, patterns]) => patterns.some(p => matchGlob(file, p))).map(([key]) => key))
.toEqual(['plan-eng-finding-count', 'plan-eng-multi-finding-batching']);
}
});
});