fix(review): resolve the judged revalidation, setup-authority, plan-gate and findings-record ambiguities

The census review workflow judge scored clarity/actionability 3 on both
attempts: smoke-clock limits appeared to forbid post-repair revalidation,
the caller deadline was undefined, 'ask for setup' conflicted with the
report-only browser rule, fallback-sourced HIGH discrepancies had no gate
decision, and the Step 5.8 record omitted adversarial findings.
This commit is contained in:
garrytan committed 2026-09-29 16:19:39 +00:00
1 parent 05ffcaf9e2
commit 175b12933d
15 files changed
+76 -43

No files matched your search

+2 -2
View File
@@ -1996,8 +1996,8 @@ Run the shared preflight; start its smoke guard once. Guard every smoke probe. F
- Required: plan commands/assertions, listed separately. Other ideas are optional, untested.
**3. Run smoke and plan checks.**
Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit.
Then run required plan checks, even after smoke expires, using the same procedure but no smoke guard; never reset the clock.
Follow the shared Probe loop for smoke checks and replays until the smoke limit.
Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock.
Use finite command timeouts, capped at the caller's remaining time if it has a deadline.
Await clock/guard results before acting. When the caller's deadline expires, mark unfinished checks not-run.
+2 -2
View File
@@ -2255,8 +2255,8 @@ Run the shared preflight; start its smoke guard once. Guard every smoke probe. F
- Required: plan commands/assertions, listed separately. Other ideas are optional, untested.
**3. Run smoke and plan checks.**
Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit.
Then run required plan checks, even after smoke expires, using the same procedure but no smoke guard; never reset the clock.
Follow the shared Probe loop for smoke checks and replays until the smoke limit.
Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock.
Use finite command timeouts, capped at the caller's remaining time if it has a deadline.
Await clock/guard results before acting. When the caller's deadline expires, mark unfinished checks not-run.
+1 -1
View File
@@ -149,7 +149,7 @@ describe('compact QA browser recipes retain native operations', () => {
]) expect(loop).toContain(contract);
for (const field of ['observationCommand', 'observed', 'hypothesis', 'nextCommand']) expect(loop).toContain(`${field}:`);
expect(loop).toContain('Functional Full, Quick and Regression have no default total timer');
if (skillName !== 'qa-only') expect(loop).toContain('Explicit plan checks remain required beyond this smoke budget');
if (skillName !== 'qa-only') expect(loop).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
}
});
+1 -1
View File
@@ -70,7 +70,7 @@ describe('review and ship completion freshness contracts', () => {
expect(gate).toContain('Reporting reserves cannot stop required revalidation within the caller\'s deadline');
expect(body).toContain('Await clock/guard results before acting');
expect(body).toContain('Smoke: 5 minutes/12 probes');
expect(body).toContain('Then run required plan checks, even after smoke expires');
expect(body).toContain('Then run required plan checks and revalidation, even after smoke expires');
expect(body).toContain('no smoke guard; never reset the clock');
expect(body).toContain('Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline');
});
+3 -3
View File
@@ -608,8 +608,8 @@ describe('generated actual parent paths', () => {
expect(load).toContain('Templates cannot replace them');
const flat = parent.replace(/\s+/g, ' ');
expect(flat).toContain('Only the parent runs report-only discovery');
expect(flat).toContain('Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit');
expect(flat).toContain('Then run required plan checks, even after smoke expires');
expect(flat).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit');
expect(flat).toContain('Then run required plan checks and revalidation, even after smoke expires');
expect(flat).toContain('using the same procedure but no smoke guard; never reset the clock');
expect(flat).toContain("Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline");
expect(flat).toContain('When the caller\'s deadline expires, mark unfinished checks not-run');
@@ -640,7 +640,7 @@ describe('generated actual parent paths', () => {
expect(body).toContain('Stop after 5 minutes or 12 probes, whichever comes first');
expect(body).toContain('G enforces the deadline');
expect(body).toContain('Never reset D/bypass G');
expect(body).toContain('Explicit plan checks remain required beyond this smoke budget');
expect(body).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
expect(body).toContain('leaves /review incomplete');
expect(body).toContain('/ship blocked unless the user explicitly accepts that named risk');
});
+6 -4
View File
@@ -40,8 +40,8 @@ function assertBoundsAndLayout(text: string) {
function assertPlanExecution(text: string, shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude })) {
const step = text.slice(text.indexOf('**3. Run smoke and plan checks.**'), text.indexOf('**4. Check freshness before reporting.**')).replace(/\s+/g, ' ');
for (const contract of [
'Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit',
'Then run required plan checks, even after smoke expires',
'Follow the shared Probe loop for smoke checks and replays until the smoke limit',
'Then run required plan checks and revalidation, even after smoke expires',
'using the same procedure but no smoke guard; never reset the clock',
"Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline",
'When the caller\'s deadline expires, mark unfinished checks not-run',
@@ -261,17 +261,19 @@ describe('QA probe entry and checkpoint gates', () => {
const text = generateQAReview({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude }).replace(/\s+/g, ' ');
assertPlanExecution(text);
for (const [before, after] of [
['Then run required plan checks, even after smoke expires', 'Skip plan checks when smoke expired'],
['Then run required plan checks and revalidation, even after smoke expires', 'Skip plan checks when smoke expired'],
['no smoke guard; never reset the clock', 'restart and use the smoke guard'],
['same procedure', 'Start a new checkpoint sequence'],
['at the caller\'s remaining time', 'with no caller cap'],
['When the caller\'s deadline expires, mark unfinished checks not-run', 'If that deadline expired, mark the check passed'],
['Await clock/guard results before acting', 'Ignore clock results'],
['plan checks and revalidation, even', 'plan checks, even'],
['smoke checks and replays until', 'smoke checks, replays and revalidation until'],
]) {
expect(text).toContain(before);
expect(() => assertPlanExecution(text.replace(before, after))).toThrow();
}
const smoke = 'Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit.';
const smoke = 'Follow the shared Probe loop for smoke checks and replays until the smoke limit.';
expect(() => assertPlanExecution(text.replace(smoke, '').replace('**4. Check', smoke + '\n**4. Check'))).toThrow();
const shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude });
for (const contract of ['First demonstrate success: output AND durable effects',
+25 -3
View File
@@ -32,7 +32,7 @@ test('review audits deliverables before deferring behavioral plan checks to the
expect(audit.indexOf('Inspect the validator and its hooks')).toBeLessThan(audit.indexOf('If found and verified safe above, invoke it'));
expect(audit).not.toContain('For each extracted plan item, run the verification dispatch');
const qa = generateQAReview(ctx);
expect(qa).toContain('Then run required plan checks, even after smoke expires');
expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires');
expect(qa).toContain('Report clean/completed only when all required checks pass on current inputs');
});
@@ -249,7 +249,7 @@ test('caller QA defines execution, evidence ownership and report adaptation befo
for (const contract of [
'Only the parent runs report-only discovery',
'Never overwrite another run',
'Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit',
'Follow the shared Probe loop for smoke checks and replays until the smoke limit',
'using the same procedure but no smoke guard; never reset the clock',
'Read agent/user updates and await results without batching them with reporting/logging',
'Compare each probe\'s recorded source, tests, contracts, commands and fixtures (or input fingerprint) with current inputs, even without updates',
@@ -378,7 +378,7 @@ test('review finalization ownership: the plan audit retains its high-impact gate
expect(plan).not.toContain('never blocks the review');
expect(plan).toContain('{{PLAN_COMPLETION_AUDIT_REVIEW}}');
const audit = readFileSync(join(root, 'review/sections/plan-completion.md'), 'utf8');
const gate = audit.indexOf('**HIGH-impact discrepancies** trigger AskUserQuestion');
const gate = audit.indexOf('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion');
expect(gate).toBeGreaterThan(-1);
expect(gate).toBeLessThan(audit.indexOf('When continuing after the audit (no HIGH-impact gate, or option B/C)'));
expect(audit).toContain('then it gates via AskUserQuestion');
@@ -564,3 +564,25 @@ test('ship review clarity: parent settlement gate precedes classification and ca
expect(gate).toContain('After terminal failure, independent evidence may support fixes');
expect(gate).toContain('missing dispatched output still blocks continuation, even with a QA exception');
});
test('review resolves the judged smoke-clock, setup-authority, plan-gate and findings-source ambiguities', () => {
const ctx = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude } as TemplateContext;
const qa = generateQAReview(ctx).replace(/\s+/g, ' ');
expect(qa).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit');
expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard');
expect(qa).toContain('/review sets none; only an invoker-supplied EARLIER_UTC counts');
expect(qa).toContain('Report-only /review never runs setup, installs or cookie import, even after approval');
expect(qa).toContain('After a grant, recheck readiness and run affected checks; otherwise they stay blocked');
expect(qa).not.toContain('Ask for setup/permission');
expect(generateQAReview({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).not.toContain('/review sets none');
const shared = generateQAExploratory({ ...ctx, skillName: 'qa' }).replace(/\s+/g, ' ');
expect(shared).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
const audit = generatePlanCompletionAuditReview(ctx).replace(/\s+/g, ' ');
expect(audit).toContain('No plan file found → say "No plan file detected." and use the Fallback Intent Sources below');
expect(audit).not.toContain('skip with "No plan file detected — skipping."');
expect(audit).toContain('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion');
expect(audit).toContain('Discrepancies derived only from fallback sources (commit messages, TODOS.md, PR description) never trigger this question');
expect(generatePlanCompletionAuditShip({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).toContain('skip with "No plan file detected — skipping."');
const persist = skill.slice(skill.indexOf('### 2. Fill the record')).replace(/\s+/g, ' ');
expect(persist).toContain('findings Step 5 combined (core, specialist, Step 4.8 adversarial, VALID & ACTIONABLE Greptile and verified exploratory QA findings)');
});
+1 -1
View File
@@ -14,7 +14,7 @@ test('Ship initializes and applies its smoke guard independently of required pla
expect(body).toContain('Run the shared preflight; start its smoke guard once. Guard every smoke probe.');
expect(body.indexOf('start its smoke guard once')).toBeLessThan(body.indexOf('**3. Run smoke and plan checks.**'));
expect(body).toContain('Required even for small diffs or missing plans/servers');
expect(body).toContain('Then run required plan checks, even after smoke expires');
expect(body).toContain('Then run required plan checks and revalidation, even after smoke expires');
expect(body).toContain('using the same procedure but no smoke guard; never reset the clock');
}
});