mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
fix(review): resolve the judged revalidation, setup-authority, plan-gate and findings-record ambiguities
The census review workflow judge scored clarity/actionability 3 on both attempts: smoke-clock limits appeared to forbid post-repair revalidation, the caller deadline was undefined, 'ask for setup' conflicted with the report-only browser rule, fallback-sourced HIGH discrepancies had no gate decision, and the Step 5.8 record omitted adversarial findings.
This commit is contained in:
1 parent
05ffcaf9e2
commit
175b12933d
15 files changed
+76
-43
No files matched your search
+2
-2
@@ -1996,8 +1996,8 @@ Run the shared preflight; start its smoke guard once. Guard every smoke probe. F
|
||||
- Required: plan commands/assertions, listed separately. Other ideas are optional, untested.
|
||||
|
||||
**3. Run smoke and plan checks.**
|
||||
Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit.
|
||||
Then run required plan checks, even after smoke expires, using the same procedure but no smoke guard; never reset the clock.
|
||||
Follow the shared Probe loop for smoke checks and replays until the smoke limit.
|
||||
Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock.
|
||||
Use finite command timeouts, capped at the caller's remaining time if it has a deadline.
|
||||
Await clock/guard results before acting. When the caller's deadline expires, mark unfinished checks not-run.
|
||||
|
||||
|
||||
+2
-2
@@ -2255,8 +2255,8 @@ Run the shared preflight; start its smoke guard once. Guard every smoke probe. F
|
||||
- Required: plan commands/assertions, listed separately. Other ideas are optional, untested.
|
||||
|
||||
**3. Run smoke and plan checks.**
|
||||
Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit.
|
||||
Then run required plan checks, even after smoke expires, using the same procedure but no smoke guard; never reset the clock.
|
||||
Follow the shared Probe loop for smoke checks and replays until the smoke limit.
|
||||
Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard; never reset the clock.
|
||||
Use finite command timeouts, capped at the caller's remaining time if it has a deadline.
|
||||
Await clock/guard results before acting. When the caller's deadline expires, mark unfinished checks not-run.
|
||||
|
||||
|
||||
@@ -149,7 +149,7 @@ describe('compact QA browser recipes retain native operations', () => {
|
||||
]) expect(loop).toContain(contract);
|
||||
for (const field of ['observationCommand', 'observed', 'hypothesis', 'nextCommand']) expect(loop).toContain(`${field}:`);
|
||||
expect(loop).toContain('Functional Full, Quick and Regression have no default total timer');
|
||||
if (skillName !== 'qa-only') expect(loop).toContain('Explicit plan checks remain required beyond this smoke budget');
|
||||
if (skillName !== 'qa-only') expect(loop).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -70,7 +70,7 @@ describe('review and ship completion freshness contracts', () => {
|
||||
expect(gate).toContain('Reporting reserves cannot stop required revalidation within the caller\'s deadline');
|
||||
expect(body).toContain('Await clock/guard results before acting');
|
||||
expect(body).toContain('Smoke: 5 minutes/12 probes');
|
||||
expect(body).toContain('Then run required plan checks, even after smoke expires');
|
||||
expect(body).toContain('Then run required plan checks and revalidation, even after smoke expires');
|
||||
expect(body).toContain('no smoke guard; never reset the clock');
|
||||
expect(body).toContain('Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline');
|
||||
});
|
||||
|
||||
@@ -608,8 +608,8 @@ describe('generated actual parent paths', () => {
|
||||
expect(load).toContain('Templates cannot replace them');
|
||||
const flat = parent.replace(/\s+/g, ' ');
|
||||
expect(flat).toContain('Only the parent runs report-only discovery');
|
||||
expect(flat).toContain('Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit');
|
||||
expect(flat).toContain('Then run required plan checks, even after smoke expires');
|
||||
expect(flat).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit');
|
||||
expect(flat).toContain('Then run required plan checks and revalidation, even after smoke expires');
|
||||
expect(flat).toContain('using the same procedure but no smoke guard; never reset the clock');
|
||||
expect(flat).toContain("Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline");
|
||||
expect(flat).toContain('When the caller\'s deadline expires, mark unfinished checks not-run');
|
||||
@@ -640,7 +640,7 @@ describe('generated actual parent paths', () => {
|
||||
expect(body).toContain('Stop after 5 minutes or 12 probes, whichever comes first');
|
||||
expect(body).toContain('G enforces the deadline');
|
||||
expect(body).toContain('Never reset D/bypass G');
|
||||
expect(body).toContain('Explicit plan checks remain required beyond this smoke budget');
|
||||
expect(body).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
|
||||
expect(body).toContain('leaves /review incomplete');
|
||||
expect(body).toContain('/ship blocked unless the user explicitly accepts that named risk');
|
||||
});
|
||||
|
||||
@@ -40,8 +40,8 @@ function assertBoundsAndLayout(text: string) {
|
||||
function assertPlanExecution(text: string, shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude })) {
|
||||
const step = text.slice(text.indexOf('**3. Run smoke and plan checks.**'), text.indexOf('**4. Check freshness before reporting.**')).replace(/\s+/g, ' ');
|
||||
for (const contract of [
|
||||
'Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit',
|
||||
'Then run required plan checks, even after smoke expires',
|
||||
'Follow the shared Probe loop for smoke checks and replays until the smoke limit',
|
||||
'Then run required plan checks and revalidation, even after smoke expires',
|
||||
'using the same procedure but no smoke guard; never reset the clock',
|
||||
"Use finite command timeouts, capped at the caller\'s remaining time if it has a deadline",
|
||||
'When the caller\'s deadline expires, mark unfinished checks not-run',
|
||||
@@ -261,17 +261,19 @@ describe('QA probe entry and checkpoint gates', () => {
|
||||
const text = generateQAReview({ host: 'claude', skillName, tmplPath: '', paths: HOST_PATHS.claude }).replace(/\s+/g, ' ');
|
||||
assertPlanExecution(text);
|
||||
for (const [before, after] of [
|
||||
['Then run required plan checks, even after smoke expires', 'Skip plan checks when smoke expired'],
|
||||
['Then run required plan checks and revalidation, even after smoke expires', 'Skip plan checks when smoke expired'],
|
||||
['no smoke guard; never reset the clock', 'restart and use the smoke guard'],
|
||||
['same procedure', 'Start a new checkpoint sequence'],
|
||||
['at the caller\'s remaining time', 'with no caller cap'],
|
||||
['When the caller\'s deadline expires, mark unfinished checks not-run', 'If that deadline expired, mark the check passed'],
|
||||
['Await clock/guard results before acting', 'Ignore clock results'],
|
||||
['plan checks and revalidation, even', 'plan checks, even'],
|
||||
['smoke checks and replays until', 'smoke checks, replays and revalidation until'],
|
||||
]) {
|
||||
expect(text).toContain(before);
|
||||
expect(() => assertPlanExecution(text.replace(before, after))).toThrow();
|
||||
}
|
||||
const smoke = 'Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit.';
|
||||
const smoke = 'Follow the shared Probe loop for smoke checks and replays until the smoke limit.';
|
||||
expect(() => assertPlanExecution(text.replace(smoke, '').replace('**4. Check', smoke + '\n**4. Check'))).toThrow();
|
||||
const shared = generateQAExploratory({ host: 'claude', skillName: 'qa', tmplPath: '', paths: HOST_PATHS.claude });
|
||||
for (const contract of ['First demonstrate success: output AND durable effects',
|
||||
|
||||
@@ -32,7 +32,7 @@ test('review audits deliverables before deferring behavioral plan checks to the
|
||||
expect(audit.indexOf('Inspect the validator and its hooks')).toBeLessThan(audit.indexOf('If found and verified safe above, invoke it'));
|
||||
expect(audit).not.toContain('For each extracted plan item, run the verification dispatch');
|
||||
const qa = generateQAReview(ctx);
|
||||
expect(qa).toContain('Then run required plan checks, even after smoke expires');
|
||||
expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires');
|
||||
expect(qa).toContain('Report clean/completed only when all required checks pass on current inputs');
|
||||
});
|
||||
|
||||
@@ -249,7 +249,7 @@ test('caller QA defines execution, evidence ownership and report adaptation befo
|
||||
for (const contract of [
|
||||
'Only the parent runs report-only discovery',
|
||||
'Never overwrite another run',
|
||||
'Follow the shared Probe loop for smoke checks, replays and revalidation until the smoke limit',
|
||||
'Follow the shared Probe loop for smoke checks and replays until the smoke limit',
|
||||
'using the same procedure but no smoke guard; never reset the clock',
|
||||
'Read agent/user updates and await results without batching them with reporting/logging',
|
||||
'Compare each probe\'s recorded source, tests, contracts, commands and fixtures (or input fingerprint) with current inputs, even without updates',
|
||||
@@ -378,7 +378,7 @@ test('review finalization ownership: the plan audit retains its high-impact gate
|
||||
expect(plan).not.toContain('never blocks the review');
|
||||
expect(plan).toContain('{{PLAN_COMPLETION_AUDIT_REVIEW}}');
|
||||
const audit = readFileSync(join(root, 'review/sections/plan-completion.md'), 'utf8');
|
||||
const gate = audit.indexOf('**HIGH-impact discrepancies** trigger AskUserQuestion');
|
||||
const gate = audit.indexOf('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion');
|
||||
expect(gate).toBeGreaterThan(-1);
|
||||
expect(gate).toBeLessThan(audit.indexOf('When continuing after the audit (no HIGH-impact gate, or option B/C)'));
|
||||
expect(audit).toContain('then it gates via AskUserQuestion');
|
||||
@@ -564,3 +564,25 @@ test('ship review clarity: parent settlement gate precedes classification and ca
|
||||
expect(gate).toContain('After terminal failure, independent evidence may support fixes');
|
||||
expect(gate).toContain('missing dispatched output still blocks continuation, even with a QA exception');
|
||||
});
|
||||
|
||||
test('review resolves the judged smoke-clock, setup-authority, plan-gate and findings-source ambiguities', () => {
|
||||
const ctx = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'claude', paths: HOST_PATHS.claude } as TemplateContext;
|
||||
const qa = generateQAReview(ctx).replace(/\s+/g, ' ');
|
||||
expect(qa).toContain('Follow the shared Probe loop for smoke checks and replays until the smoke limit');
|
||||
expect(qa).toContain('Then run required plan checks and revalidation, even after smoke expires, using the same procedure but no smoke guard');
|
||||
expect(qa).toContain('/review sets none; only an invoker-supplied EARLIER_UTC counts');
|
||||
expect(qa).toContain('Report-only /review never runs setup, installs or cookie import, even after approval');
|
||||
expect(qa).toContain('After a grant, recheck readiness and run affected checks; otherwise they stay blocked');
|
||||
expect(qa).not.toContain('Ask for setup/permission');
|
||||
expect(generateQAReview({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).not.toContain('/review sets none');
|
||||
const shared = generateQAExploratory({ ...ctx, skillName: 'qa' }).replace(/\s+/g, ' ');
|
||||
expect(shared).toContain('Explicit plan checks and revalidation remain required beyond this smoke budget');
|
||||
const audit = generatePlanCompletionAuditReview(ctx).replace(/\s+/g, ' ');
|
||||
expect(audit).toContain('No plan file found → say "No plan file detected." and use the Fallback Intent Sources below');
|
||||
expect(audit).not.toContain('skip with "No plan file detected — skipping."');
|
||||
expect(audit).toContain('**HIGH-impact plan-file discrepancies** trigger AskUserQuestion');
|
||||
expect(audit).toContain('Discrepancies derived only from fallback sources (commit messages, TODOS.md, PR description) never trigger this question');
|
||||
expect(generatePlanCompletionAuditShip({ ...ctx, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl' })).toContain('skip with "No plan file detected — skipping."');
|
||||
const persist = skill.slice(skill.indexOf('### 2. Fill the record')).replace(/\s+/g, ' ');
|
||||
expect(persist).toContain('findings Step 5 combined (core, specialist, Step 4.8 adversarial, VALID & ACTIONABLE Greptile and verified exploratory QA findings)');
|
||||
});
|
||||
@@ -14,7 +14,7 @@ test('Ship initializes and applies its smoke guard independently of required pla
|
||||
expect(body).toContain('Run the shared preflight; start its smoke guard once. Guard every smoke probe.');
|
||||
expect(body.indexOf('start its smoke guard once')).toBeLessThan(body.indexOf('**3. Run smoke and plan checks.**'));
|
||||
expect(body).toContain('Required even for small diffs or missing plans/servers');
|
||||
expect(body).toContain('Then run required plan checks, even after smoke expires');
|
||||
expect(body).toContain('Then run required plan checks and revalidation, even after smoke expires');
|
||||
expect(body).toContain('using the same procedure but no smoke guard; never reset the clock');
|
||||
}
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user