From 8965b915aec61c686900890b0144e21575267476 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 2 Nov 2026 15:05:47 -0800 Subject: [PATCH] v1.93.23.0 GSTA-253 flaky pass: plan-design focus header, shared-libs revalidation sources, .capy leak-scan skip, fixture state root (#3878) * fix(flaky): plan-design focus header, shared-libs revalidation sources, .capy leak-scan skip, fixture state root (GSTA-253) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: register the GSTA-253 focus capture in FREE_FIXTURES Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 13 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- package.json | 2 +- .../design-ui-focus-capture-41414111284.json | 115 ++++++++++++++++++ test/git-fixture-template.test.ts | 18 +-- test/helpers/free-fixtures-data.ts | 1 + test/helpers/git-fixture-template.ts | 9 +- test/helpers/pregate-fixture.ts | 2 +- test/helpers/restamp-fixture.ts | 2 +- test/helpers/shared-libs-checker-evidence.ts | 2 +- test/plan-design-with-ui-fixture.test.ts | 32 ++++- test/plan-reality-receipts.test.ts | 15 ++- test/shared-libs-cancellation.test.ts | 2 +- ...hecker-interface-evidence-declared.test.ts | 4 +- ...ed-libs-checker-interface-evidence.test.ts | 41 ++++++- test/skill-e2e-plan-design-with-ui.test.ts | 7 +- test/skill-e2e-shared-libs.test.ts | 2 + 18 files changed, 240 insertions(+), 31 deletions(-) create mode 100644 test/fixtures/design-ui-focus-capture-41414111284.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 7618fa598..63d0b33e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.93.23.0] - 2026-11-02 + +**Four tests from the 2026-11-02 flaky pass stop failing correct runs, and one more names its cause when it fails.** + +### Fixed +- **`test/skill-e2e-plan-design-with-ui.test.ts`**: the Step 0D focus detector accepts the native question header "Focus" with the design rating and an all-7 option, instead of needing one more label spelling. Train gate 41414111284 offered "Full review, all 7 (recommended)", which the old label rule missed. All ten captured focus questions carry that header, and no other captured question does. The capture is saved as a fixture, and eight negative controls cover a wrong header, a missing rating, a 4/7 rating, issue-numbered options, a single option and no all-7 option. +- **`test/skill-e2e-shared-libs.test.ts`**: the `shared-libs-review-revalidation` callback passes the advisory's source files to the checker, so a batched read that exits nonzero but prints the whole file counts, as the checker already allows elsewhere. Train screen 40737741454 failed this way: its one read of `src/retry-worker.ts` was `cat …; echo ===ROUTE; …`, which exited 1 after printing the file. A truncated output and a single-command errored read still fail. +- **`test/helpers/git-fixture-template.ts`**: the template leak scan skips every `.capy` directory, not only `.capy/git-shim.log`. Capy's git shim creates `$HOME/.capy` in whatever home it runs under, so a fixture home that ran git holds the machine's files, not gstack's. A file named `.capy` and look-alike directories (`.capy-old`, `capy`) are still scanned. +- **`test/helpers/restamp-fixture.ts`, `test/helpers/pregate-fixture.ts`**: fixture repositories pin `GSTACK_STATE_ROOT` to their own home, so a value exported by the gstack launch file no longer redirects their state when bun runs without the test preload (`--config=/dev/null`). `test/review-targeted.test.ts`'s end-to-end case failed every time under that setup and passes with the fix. + +### Changed +- `test/plan-reality-receipts.test.ts` checks each git step's exit status and prints its stderr, so a red like Free Tests 41316085384 and 41375226438 (an empty commit sha) names the git step that failed. + ## [1.93.22.0] - 2026-11-02 ### Fixed diff --git a/VERSION b/VERSION index 3e56411d5..f6f8c5632 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.93.22.0 +1.93.23.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index dbef6f88c..068816677 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.93.22.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.93.23.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/package.json b/package.json index 720d53155..13d24a6c0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.93.22", + "version": "1.93.23", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/fixtures/design-ui-focus-capture-41414111284.json b/test/fixtures/design-ui-focus-capture-41414111284.json new file mode 100644 index 000000000..f75db6a85 --- /dev/null +++ b/test/fixtures/design-ui-focus-capture-41414111284.json @@ -0,0 +1,115 @@ +{ + "calls": [ + { + "preReview": true, + "observedAtMs": 76003, + "call": { + "sessionId": "4767bbfe-f301-4d0c-9cfb-eeff94eb0850", + "toolUseId": "toolu_01S2K6zpLEy8wZXVMiG7teSA", + "questions": [ + { + "question": "D1 — Review focus for the dashboard plan\nProject/branch/task: main, design review of the User Dashboard Page plan (review-input.md).\nELI10: I've rated this plan 4/10 on design completeness. The data and safety rules are solid, but it never says what the user sees first, how the three panels sit on phone vs desktop, or what the empty/error/toast/modal moments actually say and do. I'll generate visual mockups next, then review all 7 design dimensions. You can narrow that to specific areas instead.\nStakes if we pick wrong: Narrowing too early leaves gaps like toast accessibility or mobile layout for the implementer to guess at, and they ship as generic defaults.\nRecommendation: A because the gaps span hierarchy, states, responsive and a11y, so a partial pass would miss real issues.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) Full review, all 7 passes (recommended)\n ✅ Covers hierarchy, states, journey, slop risk, design system, responsive and a11y\n ✅ Every gap gets its own decision so nothing is silently assumed\n ❌ More questions to answer, one per real gap found across the passes\nB) Focus on specific areas only\n ✅ Fewer questions if you only care about, say, states and mobile layout\n ✅ Faster path to an amended plan for the areas you name\n ❌ Unreviewed dimensions stay at their current low rating and ship as guesses\nNet: Full coverage with more decisions, versus speed with known blind spots.", + "header": "Focus", + "multiSelect": false, + "options": [ + { + "label": "Full review, all 7 (recommended)", + "description": "Mockups, then all 7 passes. ✅ Covers every dimension. ✅ Each gap decided individually. ❌ More questions overall." + }, + { + "label": "Focus on specific areas", + "description": "Name the areas in a note. ✅ Fewer questions. ✅ Faster. ❌ Other dimensions remain unreviewed guesses." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D1 — Review focus for the dashboard plan\nProject/branch/task: main, design review of the User Dashboard Page plan (review-input.md).\nELI10: I've rated this plan 4/10 on design completeness. The data and safety rules are solid, but it never says what the user sees first, how the three panels sit on phone vs desktop, or what the empty/error/toast/modal moments actually say and do. I'll generate visual mockups next, then review all 7 design dimensions. You can narrow that to specific areas instead.\nStakes if we pick wrong: Narrowing too early leaves gaps like toast accessibility or mobile layout for the implementer to guess at, and they ship as generic defaults.\nRecommendation: A because the gaps span hierarchy, states, responsive and a11y, so a partial pass would miss real issues.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) Full review, all 7 passes (recommended)\n ✅ Covers hierarchy, states, journey, slop risk, design system, responsive and a11y\n ✅ Every gap gets its own decision so nothing is silently assumed\n ❌ More questions to answer, one per real gap found across the passes\nB) Focus on specific areas only\n ✅ Fewer questions if you only care about, say, states and mobile layout\n ✅ Faster path to an amended plan for the areas you name\n ❌ Unreviewed dimensions stay at their current low rating and ship as guesses\nNet: Full coverage with more decisions, versus speed with known blind spots.": "Full review, all 7 (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-11-02T08:39:03.879Z" + } + }, + { + "preReview": true, + "observedAtMs": 106367, + "call": { + "sessionId": "4767bbfe-f301-4d0c-9cfb-eeff94eb0850", + "toolUseId": "toolu_01AqE7XDKvovVJAvKF31kq8z", + "questions": [ + { + "question": "Want outside design voices before the detailed review?\nD2 — Outside design voices\nProject/branch/task: main, design review of the User Dashboard Page plan.\nELI10: Optionally, two other reviewers read the plan before my 7 passes: Codex checks hard design rules and a litmus scorecard, and an independent Claude subagent checks completeness. Their disagreements get raised in the relevant pass.\nStakes if we pick wrong: Skipping means one reviewer's eye; running costs a few minutes and may only get one voice if Codex is missing.\nRecommendation: No, proceed without — this is a taste call, no strong preference either way\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons: see option descriptions.\nNet: A cross-model check versus the shortest path to a fixed plan.", + "header": "Voices", + "multiSelect": false, + "options": [ + { + "label": "Yes, run outside design voices", + "description": "✅ Codex hard-rule + litmus scorecard plus an independent Claude completeness read. ✅ Disagreements surface with both perspectives in the relevant pass. ❌ Adds a few minutes; Codex may be missing here, in which case only the Claude voice runs and it's tagged single-model." + }, + { + "label": "No, proceed without (recommended)", + "description": "✅ Straight into the 7 passes with the approved mockup as anchor. ✅ Fewest steps to a fixed plan. ❌ Findings rest on one reviewer; no cross-model check on slop risk or hierarchy." + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "Want outside design voices before the detailed review?\nD2 — Outside design voices\nProject/branch/task: main, design review of the User Dashboard Page plan.\nELI10: Optionally, two other reviewers read the plan before my 7 passes: Codex checks hard design rules and a litmus scorecard, and an independent Claude subagent checks completeness. Their disagreements get raised in the relevant pass.\nStakes if we pick wrong: Skipping means one reviewer's eye; running costs a few minutes and may only get one voice if Codex is missing.\nRecommendation: No, proceed without — this is a taste call, no strong preference either way\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons: see option descriptions.\nNet: A cross-model check versus the shortest path to a fixed plan.": "No, proceed without (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-11-02T08:39:34.676Z" + } + }, + { + "preReview": false, + "observedAtMs": 132955, + "call": { + "sessionId": "4767bbfe-f301-4d0c-9cfb-eeff94eb0850", + "toolUseId": "toolu_01PDy4qdecNenP7T1QhxRkFR", + "questions": [ + { + "question": "D3 — 1: What does a just-logged-in user see first, second, third?\nProject/branch/task: main, User Dashboard plan, Pass 1 Information Architecture.\nELI10: The plan has three panels but no order. If we don't pick, the engineer puts them in a grid of three equal boxes and nobody knows where to look. Only notifications ask the user to do something, so they're the natural anchor.\nStakes if we pick wrong: Users skim past the one thing needing attention, or the page feels like a generic widget grid.\nRecommendation: 1A because unread notifications are the only actionable content, and Krug says each screen needs one obvious next step.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Action row, then Notifications primary, Activity secondary (recommended)\n ✅ The one actionable thing (unread items) gets the wide main column and first scan\n ✅ Quick actions sit in the page header as a compact row, one primary button\n ❌ Activity feed is visually demoted; heavy activity users scroll or look right\nB) Activity primary, Notifications in a side rail\n ✅ Suits products where 'what changed' matters more than inbox triage\n ✅ Notifications rail stays visible while scanning activity history\n ❌ Unread items compete in a narrow rail and are easier to ignore\nNet: Anchor on what needs action (notifications) or on what changed (activity).", + "header": "Hierarchy", + "multiSelect": false, + "options": [ + { + "label": "1A Notifications primary (recommended)", + "description": "Header + quick-action row first, Notifications wide main column second, Activity secondary column third.", + "preview": "lg (≥1024px)\n+------------------------------------------------+\n| Dashboard [Open workspace] [Review notifications (3)] |\n+-------------------------------+----------------+\n| Notifications 3 unread | Recent activity|\n| [Mark all as read] | • edited ... |\n| ● New comment on Q4 plan 2m | • shared ... |\n| ● ... | • ... |\n| ... | |\n+-------------------------------+----------------+\n 1st: page title + primary action\n 2nd: unread notifications\n 3rd: recent activity" + }, + { + "label": "1B Activity primary", + "description": "Header + quick-action row first, Activity wide main column second, Notifications side rail third.", + "preview": "lg (≥1024px)\n+------------------------------------------------+\n| Dashboard [Open workspace] [Review notifications (3)] |\n+-------------------------------+----------------+\n| Recent activity | Notifications |\n| • You edited Q4 roadmap 2m | 3 unread |\n| • ... | [Mark all read]|\n| • ... | ● ... |\n+-------------------------------+----------------+\n 1st: page title + primary action\n 2nd: recent activity\n 3rd: notifications rail" + } + ] + } + ], + "answered": true, + "failed": false, + "answers": { + "D3 — 1: What does a just-logged-in user see first, second, third?\nProject/branch/task: main, User Dashboard plan, Pass 1 Information Architecture.\nELI10: The plan has three panels but no order. If we don't pick, the engineer puts them in a grid of three equal boxes and nobody knows where to look. Only notifications ask the user to do something, so they're the natural anchor.\nStakes if we pick wrong: Users skim past the one thing needing attention, or the page feels like a generic widget grid.\nRecommendation: 1A because unread notifications are the only actionable content, and Krug says each screen needs one obvious next step.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Action row, then Notifications primary, Activity secondary (recommended)\n ✅ The one actionable thing (unread items) gets the wide main column and first scan\n ✅ Quick actions sit in the page header as a compact row, one primary button\n ❌ Activity feed is visually demoted; heavy activity users scroll or look right\nB) Activity primary, Notifications in a side rail\n ✅ Suits products where 'what changed' matters more than inbox triage\n ✅ Notifications rail stays visible while scanning activity history\n ❌ Unread items compete in a narrow rail and are easier to ignore\nNet: Anchor on what needs action (notifications) or on what changed (activity).": "1A Notifications primary (recommended)" + }, + "unansweredQuestionIndices": [], + "answeredAt": "2026-11-02T08:40:01.258Z" + } + } + ], + "provenance": { + "kind": "public native AskUserQuestion calls and answers only (observation.json fingerprints[].nativeCall, byte-exact)", + "runId": 41414111284, + "artifact": "paid-slice-6-a1", + "prNumber": null, + "headSha": "4a280637792b46576474f3349a1ba2e977c5f5c9", + "testedSha": "4a280637792b46576474f3349a1ba2e977c5f5c9", + "recordedOutcome": "ceiling_reached", + "recordedReviewCount": 1, + "recordedStep0Count": 2, + "recordedPassed": false, + "note": "Step 0D titled \"Review focus\" with header Focus, the 4/10 design-completeness rating in its ELI10 body, and an all-7 option that does not lead with the count (\"Full review, all 7 (recommended)\")" + } +} diff --git a/test/git-fixture-template.test.ts b/test/git-fixture-template.test.ts index f63a6c002..29b31e1ae 100644 --- a/test/git-fixture-template.test.ts +++ b/test/git-fixture-template.test.ts @@ -71,21 +71,23 @@ describe('git fixture template', () => { expect(() => leaky.copy()).toThrow(/still references its template [^\n]*\n.*config/); }); - test("Capy's git shim log in a fixture home (home/.capy/git-shim.log) is not a leak; any other file naming the template still is", () => { + test("Capy's directory in a fixture home (home/.capy, the git shim's log and anything else under it) is not a leak; any other file naming the template still is", () => { const plant = (rel: string) => gitFixtureTemplate({ prefix: 'gstack-fixture-template-shim-', build: (root) => { const meta = build(root); fs.mkdirSync(path.dirname(path.join(root, rel)), { recursive: true }); fs.writeFileSync(path.join(root, rel), '# capy git shim v2: ts cwd ppid parent verb\n2026-11-02T02:15:30Z\t' + path.join(root, 'work') + '\t1\tbun\tcat-file\n'); return meta; }, rebind: (copy, tpl) => rebindGitDir(path.join(copy, 'work', '.git'), tpl, copy) }); - const shim = plant(path.join('home', '.capy', 'git-shim.log')); - const other = plant(path.join('home', '.capy', 'state.json')); + const skipped = [path.join('home', '.capy', 'git-shim.log'), path.join('home', '.capy', 'git-shim.log.1'), path.join('home', '.capy', 'state', 'run.json')].map(plant); + const leaks = [path.join('home', 'state.json'), path.join('home', '.capy-old', 'state.json'), path.join('home', '.config', '.capy'), path.join('home', 'capy', 'git-shim.log')].map(rel => ({ rel, t: plant(rel) })); try { - const c = shim.copy(); - roots.push(c.root); - expect(git(path.join(c.root, 'work'), 'remote', 'get-url', 'origin')).toBe(path.join(c.root, 'origin.git')); - expect(() => other.copy()).toThrow(/still references its template [^\n]*\n.*state\.json/); - } finally { shim.dispose(); other.dispose(); } + for (const s of skipped) { + const c = s.copy(); + roots.push(c.root); + expect(git(path.join(c.root, 'work'), 'remote', 'get-url', 'origin')).toBe(path.join(c.root, 'origin.git')); + } + for (const { rel, t } of leaks) expect(() => t.copy()).toThrow(new RegExp('still references its template [^\\n]*\\n.*' + rel.replace(/[.\\/]/g, m => '\\' + m) + '$')); + } finally { for (const s of skipped) s.dispose(); for (const { t } of leaks) t.dispose(); } }); test('native, forward-slash, backslash-escaped and Git Bash spellings are each rewritten, and each one left behind is found', () => { diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index a78b3ffca..6baccf554 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -92,6 +92,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/design-ui-focus-capture-40664336141.json': ['test/plan-design-with-ui-fixture.test.ts'], 'test/fixtures/design-ui-focus-capture-41178131219.json': ['test/plan-design-with-ui-fixture.test.ts'], 'test/fixtures/design-ui-focus-capture-41186376825.json': ['test/plan-design-with-ui-fixture.test.ts'], + 'test/fixtures/design-ui-focus-capture-41414111284.json': ['test/plan-design-with-ui-fixture.test.ts'], 'test/fixtures/design-ui-focus-capture.json': ['test/plan-design-with-ui-fixture.test.ts'], 'test/fixtures/detector-corpora/auto-decide-preserved/**': ['test/detector-corpus-auto-decide-preserved.test.ts'], 'test/fixtures/detector-corpora/plan-ceo-section-loading/**': ['test/detector-corpus-plan-ceo-section-loading.test.ts'], diff --git a/test/helpers/git-fixture-template.ts b/test/helpers/git-fixture-template.ts index 5fb9f72eb..f296ea648 100644 --- a/test/helpers/git-fixture-template.ts +++ b/test/helpers/git-fixture-template.ts @@ -54,8 +54,9 @@ export function rebindGitDir(gitDir: string, from: string, to: string): void { /** * Files and symlinks under dir (and the template's meta value) that still name templateRoot. Skipped: zlib-compressed object - * storage, and the log Capy's /usr/bin/git shim appends to in whatever HOME it runs under (.capy/git-shim.log), which names - * the template whenever a fixture home ran git during build; nothing reads it back. + * storage, and every .capy directory: Capy's /usr/bin/git shim creates $HOME/.capy in whatever HOME it runs under and logs + * there, so a fixture home that ran git during build holds the machine's writes, not gstack's (install-fixture's tree() skips + * it for the same reason); nothing reads them back. */ export function templateReferences(dir: string, templateRoot: string, meta?: unknown): string[] { const needles = pathSpellings(templateRoot).map(s => Buffer.from(s)); @@ -65,8 +66,8 @@ export function templateReferences(dir: string, templateRoot: string, meta?: unk for (const entry of fs.readdirSync(d, { withFileTypes: true })) { const p = path.join(d, entry.name); if (entry.isSymbolicLink()) { if (names(Buffer.from(fs.readlinkSync(p)))) hits.push(p); } - else if (entry.isDirectory()) { if (!(path.basename(d) === 'objects' && /^(pack|[0-9a-f]{2})$/.test(entry.name))) walk(p); } - else if (entry.isFile() && !(entry.name === 'git-shim.log' && path.basename(d) === '.capy') && names(fs.readFileSync(p))) hits.push(p); + else if (entry.isDirectory()) { if (entry.name !== '.capy' && !(path.basename(d) === 'objects' && /^(pack|[0-9a-f]{2})$/.test(entry.name))) walk(p); } + else if (entry.isFile() && names(fs.readFileSync(p))) hits.push(p); } }; walk(dir); diff --git a/test/helpers/pregate-fixture.ts b/test/helpers/pregate-fixture.ts index baef6855e..e7f0462fc 100644 --- a/test/helpers/pregate-fixture.ts +++ b/test/helpers/pregate-fixture.ts @@ -107,7 +107,7 @@ function fixtureRepoAt(root: string): FixtureRepo { const origin = path.join(root, 'origin.git'); const work = path.join(root, 'work'); const home = path.join(root, 'home'); - const env = { GSTACK_HOME: home, GSTACK_STATE_DIR: home, HOME: home, GIT_CONFIG_GLOBAL: path.join(root, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' }; + const env = { GSTACK_STATE_ROOT: home, GSTACK_HOME: home, GSTACK_STATE_DIR: home, HOME: home, GIT_CONFIG_GLOBAL: path.join(root, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' }; const git = (...args: string[]): string => { const r = gitArgvIn(work, args, 30_000, { ...process.env, ...env }); if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed: status=${r.status} signal=${r.signal} error=${r.error?.message ?? ''} stderr=${r.stderr?.toString() ?? ''}`); diff --git a/test/helpers/restamp-fixture.ts b/test/helpers/restamp-fixture.ts index 6ef5b188f..a2ed17620 100644 --- a/test/helpers/restamp-fixture.ts +++ b/test/helpers/restamp-fixture.ts @@ -46,7 +46,7 @@ export function fixtureRepoAt(root: string): FixtureRepo { const origin = path.join(root, 'origin.git'); const work = path.join(root, 'work'); const home = path.join(root, 'home'); - const env = { GSTACK_HOME: home, GSTACK_STATE_DIR: home, HOME: home, GIT_CONFIG_GLOBAL: path.join(root, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' }; + const env = { GSTACK_STATE_ROOT: home, GSTACK_HOME: home, GSTACK_STATE_DIR: home, HOME: home, GIT_CONFIG_GLOBAL: path.join(root, 'gitconfig'), GIT_CONFIG_NOSYSTEM: '1' }; const git = (...args: string[]): string => { const r = gitArgvIn(work, args, 30_000, { ...process.env, ...env }); if (r.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${r.stderr?.toString()}`); diff --git a/test/helpers/shared-libs-checker-evidence.ts b/test/helpers/shared-libs-checker-evidence.ts index b20dab453..ee56a0c8e 100644 --- a/test/helpers/shared-libs-checker-evidence.ts +++ b/test/helpers/shared-libs-checker-evidence.ts @@ -99,7 +99,7 @@ export async function capture(change: string, prepared?: PathEligibilityFixture, branch: fixtureGit(f, 'symbolic-ref', '--quiet', '--short', 'HEAD'), wtree: fixtureWorkingTree(f), startedAt: receipt.review_start.started_at, finding: current, reusable: change === 'unchanged', coveredPaths: receipt.snapshot.covered_paths, - ...(prepared ? { sources: Object.fromEntries(current!.evidence_paths.map((file: string) => [file, fs.readFileSync(path.join(f.repo, file), 'utf8')])) } : {}) }; + sources: Object.fromEntries(current!.evidence_paths.map((file: string) => [file, fs.readFileSync(path.join(f.repo, file), 'utf8')])) }; return { f, current, token, receipt, events, checkAt, finishAt, expected, prepared, resumed, prerequisites, ...(declared ? { questions } : {}) }; } diff --git a/test/plan-design-with-ui-fixture.test.ts b/test/plan-design-with-ui-fixture.test.ts index 5cba9eac2..e54ca47d0 100644 --- a/test/plan-design-with-ui-fixture.test.ts +++ b/test/plan-design-with-ui-fixture.test.ts @@ -103,6 +103,10 @@ const captured40664336141 = JSON.parse(fs.readFileSync(${JSON.stringify(path.joi // rated in its ELI10 body, and lettered its options ("A) All 7 passes", "A) All 7 dimensions"). const [captured41178131219, captured41186376825] = ['design-ui-focus-capture-41178131219.json', 'design-ui-focus-capture-41186376825.json'].map(name => JSON.parse(fs.readFileSync(${JSON.stringify(ROOT)} + '/test/fixtures/' + name, 'utf8')) .calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview))); +// Run 41414111284 slice 6 (train 4a280637): header "Focus", rated in its ELI10 body, and the all-7 option +// put the count second ("Full review, all 7 (recommended)"). +const captured41414111284 = JSON.parse(fs.readFileSync(${JSON.stringify(path.join(ROOT, 'test/fixtures/design-ui-focus-capture-41414111284.json'))}, 'utf8')) + .calls.map(({ preReview, observedAtMs, call }) => nativePlanCallFingerprint(call, observedAtMs, preReview)); const unnumberedFinding = fp('unnumbered-finding', "Which region is the dashboard's primary anchor, and how is the page composed?", false); unnumberedFinding.nativeCall.questions[0].options = nativeFinding.nativeCall.questions[0].options; const finding = fp('finding', 'Which loading feedback should Save show?', false); @@ -241,6 +245,28 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts' expect([0, 2].some(index => opts.isLastStep0AUQ(capture[index]) || opts.isReviewAUQ(capture[index]))).toBe(false); expect(opts.isReviewAUQ(capture[3])).toBe(true); } + expect(opts.isLastStep0AUQ(captured41414111284[0])).toBe(true); + expect(opts.isReviewAUQ(captured41414111284[0])).toBe(false); + expect(opts.isLastStep0AUQ(captured41414111284[1]) || opts.isReviewAUQ(captured41414111284[1])).toBe(false); + expect(opts.isReviewAUQ(captured41414111284[2])).toBe(true); + const headed = (id, header, question, labels) => { + const call = labeled(id, question, labels); + call.nativeCall.questions[0].header = header; + return call; + }; + const focusBody = 'D1 — Review focus\\nProject/branch/task: main.\\nELI10: I rated this plan 4/10 on design completeness.'; + const fullFirst = ['Full review, all 7 (recommended)', 'Focus on specific areas']; + expect(opts.isLastStep0AUQ(headed('focus-header', 'Focus', focusBody, fullFirst))).toBe(true); + for (const [label, header, question, labels] of [ + ['focus-header-unrated', 'Focus', 'D1 — Review focus\\nELI10: I will review every dimension unless you narrow it.', fullFirst], + ['focus-header-out-of-seven', 'Focus', focusBody.replace('4/10', '4/7'), fullFirst], + ['focus-header-issue-labeled', 'Focus', focusBody, ['1A Full review, all 7', '1B Focus on hierarchy']], + ['focus-header-single-option', 'Focus', focusBody, fullFirst.slice(0, 1)], + ['focus-header-no-all-seven', 'Focus', focusBody, ['Every dimension (recommended)', 'States + hierarchy only']], + ['focus-header-all-four', 'Focus', focusBody, ['Full review, all 4 (recommended)', 'Focus on specific areas']], + ['other-header', 'Hierarchy', focusBody, fullFirst], + ['header-names-more', 'Focus order', focusBody, fullFirst], + ]) expect(opts.isLastStep0AUQ(headed(label, header, question, labels)), label).toBe(false); const letteredFocus = 'D2 — Review focus\\nProject/branch/task: main.\\nELI10: I rated this plan 4/10 on design completeness.'; for (const labels of [['A) All 7 dimensions (recommended)', 'B) Focus on specific areas'], ['A. All seven passes (recommended)', 'B. Hierarchy+states only'], ['A: All 7 areas (recommended)', 'B: Only specific areas']]) { @@ -344,6 +370,8 @@ mock.module(${JSON.stringify(path.join(ROOT, 'test/helpers/claude-pty-runner.ts' : mode === 'captured-41178131219-no-focus' ? captured41178131219.filter((_, index) => index !== 1) : mode === 'captured-41186376825' ? captured41186376825 : mode === 'captured-41186376825-no-focus' ? captured41186376825.filter((_, index) => index !== 1) + : mode === 'captured-41414111284' ? captured41414111284 + : mode === 'captured-41414111284-no-focus' ? captured41414111284.filter((_, index) => index !== 0) : mode === 'captured-focus-after-finding' ? [...captured.filter((_, index) => index !== 1), captured[1]] : mode.startsWith('target-menu') ? [target, fp('other', 'Which artifact should I inspect?', false)] : mode === 'early-exit' ? [] : mode === 'focus-only' ? [chosenFocus] @@ -388,12 +416,12 @@ await import(${JSON.stringify(path.join(ROOT, 'test/skill-e2e-plan-design-with-u } finally { fs.rmSync(directory, { recursive: true, force: true }); } } -test.each(['source', 'paraphrase', 'native-sequence', 'native-unnumbered', 'native-passes', 'captured', 'captured-rated', 'captured-39298269902', 'captured-39363705692', 'captured-39852003387', 'captured-40664336141', 'captured-41178131219', 'captured-41186376825'])('UI gate seeds the exact target and accepts Design progress with an unselected no-UI alternative (%s)', mode => { +test.each(['source', 'paraphrase', 'native-sequence', 'native-unnumbered', 'native-passes', 'captured', 'captured-rated', 'captured-39298269902', 'captured-39363705692', 'captured-39852003387', 'captured-40664336141', 'captured-41178131219', 'captured-41186376825', 'captured-41414111284'])('UI gate seeds the exact target and accepts Design progress with an unselected no-UI alternative (%s)', mode => { const result = exercise(mode); expect(result.code, result.output).toBe(0); }, 20_000); -test.each(['target-menu', 'target-menu-design-system', 'focus-only', 'captured-no-focus', 'captured-focus-after-finding', 'captured-rated-no-focus', 'captured-39298269902-no-focus', 'captured-39298269902-focus-only', 'captured-39363705692-no-focus', 'captured-39852003387-no-focus', 'captured-40664336141-no-focus', 'captured-41178131219-no-focus', 'captured-41186376825-no-focus', 'native-setup-only', 'native-voices-only', 'early-exit', 'timeout', 'exited', 'unanswered-finding', 'failed-finding', 'no-native-finding'])('UI gate rejects %s and removes its fixture', mode => { +test.each(['target-menu', 'target-menu-design-system', 'focus-only', 'captured-no-focus', 'captured-focus-after-finding', 'captured-rated-no-focus', 'captured-39298269902-no-focus', 'captured-39298269902-focus-only', 'captured-39363705692-no-focus', 'captured-39852003387-no-focus', 'captured-40664336141-no-focus', 'captured-41178131219-no-focus', 'captured-41186376825-no-focus', 'captured-41414111284-no-focus', 'native-setup-only', 'native-voices-only', 'early-exit', 'timeout', 'exited', 'unanswered-finding', 'failed-finding', 'no-native-finding'])('UI gate rejects %s and removes its fixture', mode => { const result = exercise(mode); expect(result.code, result.output).toBe(1); expect(result.output).toContain('plan-design-review with UI scope FAILED'); diff --git a/test/plan-reality-receipts.test.ts b/test/plan-reality-receipts.test.ts index 35acb9980..0d1c2239c 100644 --- a/test/plan-reality-receipts.test.ts +++ b/test/plan-reality-receipts.test.ts @@ -16,11 +16,17 @@ import { receiptTokens, resolveReceipts } from '../lib/plan-reality-receipts'; const ROOT = path.resolve(import.meta.dir, '..'); const BIN = path.join(ROOT, 'bin', 'gstack-plan-reality'); +function gitOk(cwd: string, ...args: string[]) { + const r = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 10_000 }); + if (r.status !== 0) throw new Error(`git ${args.join(' ')} exited ${r.status} (signal ${r.signal}, error ${r.error?.message ?? 'none'}): ${r.stderr}`); + return r; +} + function fixtureRepo(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'reality-receipts-')); fs.mkdirSync(path.join(dir, 'lib')); fs.writeFileSync(path.join(dir, 'lib', 'a.ts'), 'one\ntwo\nthree\n'); - spawnSync('git', ['init', '-q'], { cwd: dir, timeout: 10_000 }); + gitOk(dir, 'init', '-q'); return dir; } @@ -99,10 +105,9 @@ describe('resolveReceipts', () => { test('a commit sha in the repository resolves; an absent one is unverifiable', () => { const dir = fixtureRepo(); - const git = (...a: string[]) => spawnSync('git', a, { cwd: dir, encoding: 'utf8', timeout: 10_000 }); - git('add', '.'); - git('-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'seed'); - const sha = git('rev-parse', 'HEAD').stdout.trim(); + gitOk(dir, 'add', '.'); + gitOk(dir, '-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-qm', 'seed'); + const sha = gitOk(dir, 'rev-parse', 'HEAD').stdout.trim(); const out = resolveReceipts(parseRealityLines(`REALITY: numbers pass ok ${sha.slice(0, 10)} abcdef1234`), { roots: [dir] }); expect(out.map(r => r.outcome)).toEqual(['resolved', 'unverifiable']); }); diff --git a/test/shared-libs-cancellation.test.ts b/test/shared-libs-cancellation.test.ts index 0cecc1868..916540a15 100644 --- a/test/shared-libs-cancellation.test.ts +++ b/test/shared-libs-cancellation.test.ts @@ -93,7 +93,7 @@ function harness(options: { drainOnClose?: boolean; refuse?: boolean } = {}) { expect(name).toBe('shared-libs-review-revalidation'); expect(timeout).toBe(CAPTURE_LONG_MS); registered = callback; }, createSharedLibsFixture: (label: string) => ({ root: `/virtual/${label}`, repo: '/virtual', env: {} }), - seedReviewSources() {}, fixtureWrite() {}, installNormalizingFilter() {}, seedSkippedAdvisory: async () => ({}), + seedReviewSources() {}, fixtureWrite() {}, installNormalizingFilter() {}, seedSkippedAdvisory: async () => ({ evidence_paths: [] }), fixtureWorkingTree: () => 'same', fixtureGit() {}, reviewLifecycleInstructions: () => '', seedPathReviewPrerequisites: () => ({ input: '' }), checkPathReviewPrerequisites: () => ({ settled: true }), reviewRevalidationPrompt: (f: any) => f.root, diff --git a/test/shared-libs-checker-interface-evidence-declared.test.ts b/test/shared-libs-checker-interface-evidence-declared.test.ts index 2682870cb..60739c33c 100644 --- a/test/shared-libs-checker-interface-evidence-declared.test.ts +++ b/test/shared-libs-checker-interface-evidence-declared.test.ts @@ -98,7 +98,7 @@ describe('native executable shared-code checker evidence', () => { const start = scenario.indexOf(marker) + marker.length; const body = scenario.slice(start, scenario.indexOf('\n });', start)); const verify = new Function('deps', 'result', new Bun.Transpiler({ loader: 'ts' }).transformSync(`const { - change, questions, expect, toolCommandTrace, reviewRecords, createHash, path, f, current, + change, questions, expect, toolCommandTrace, reviewRecords, createHash, path, f, current, sources, fixtureGit, fixtureWorkingTree, hasTrustedReviewStartRead, hasTrustedSharedLibsCheck, SHARED_LIBS_ROOT, resumed, prerequisites, checkPathReviewPrerequisites, hasPathReviewPrerequisiteReceipt } = deps; ${body}`)); @@ -106,7 +106,7 @@ describe('native executable shared-code checker evidence', () => { .map((event: any) => ({ tool: event.message.content[0].name, input: event.message.content[0].input })) }; let checks = 0; verify({ change, questions: run.questions, expect, toolCommandTrace, reviewRecords, createHash, path, - f: run.f, current: run.current, fixtureGit, fixtureWorkingTree, hasTrustedReviewStartRead, SHARED_LIBS_ROOT, + f: run.f, current: run.current, sources: run.expected.sources, fixtureGit, fixtureWorkingTree, hasTrustedReviewStartRead, SHARED_LIBS_ROOT, resumed: run.resumed, prerequisites: run.prerequisites, checkPathReviewPrerequisites, hasPathReviewPrerequisiteReceipt, hasTrustedSharedLibsCheck: (events: unknown[], expected: any) => { checks++; diff --git a/test/shared-libs-checker-interface-evidence.test.ts b/test/shared-libs-checker-interface-evidence.test.ts index 4159746c2..3fe58e75b 100644 --- a/test/shared-libs-checker-interface-evidence.test.ts +++ b/test/shared-libs-checker-interface-evidence.test.ts @@ -144,7 +144,7 @@ describe('native executable shared-code checker evidence', () => { const start = scenario.indexOf(marker) + marker.length; const body = scenario.slice(start, scenario.indexOf('\n });', start)); const verify = new Function('deps', 'result', new Bun.Transpiler({ loader: 'ts' }).transformSync(`const { - change, questions, expect, toolCommandTrace, reviewRecords, createHash, path, f, current, + change, questions, expect, toolCommandTrace, reviewRecords, createHash, path, f, current, sources, fixtureGit, fixtureWorkingTree, hasTrustedReviewStartRead, hasTrustedSharedLibsCheck, SHARED_LIBS_ROOT, resumed, prerequisites, checkPathReviewPrerequisites, hasPathReviewPrerequisiteReceipt } = deps; ${body}`)); @@ -153,7 +153,7 @@ describe('native executable shared-code checker evidence', () => { .map((event: any) => ({ tool: event.message.content[0].name, input: event.message.content[0].input })) }; let checks = 0; const deps = { change, questions: change === 'unchanged' ? [] : [{}], expect, toolCommandTrace, - reviewRecords, createHash, path, f: run.f, current: run.current, fixtureGit, fixtureWorkingTree, + reviewRecords, createHash, path, f: run.f, current: run.current, sources: run.expected.sources, fixtureGit, fixtureWorkingTree, hasTrustedReviewStartRead, SHARED_LIBS_ROOT, resumed: run.resumed, prerequisites: run.prerequisites, checkPathReviewPrerequisites, hasPathReviewPrerequisiteReceipt, hasTrustedSharedLibsCheck: (events: unknown[], expected: any) => { checks++; @@ -182,6 +182,43 @@ describe('native executable shared-code checker evidence', () => { expect(() => verify({ ...deps, reviewRecords: () => [] }, native)).toThrow(); }); + test('an errored batched caller read counts only when its output holds the whole source, and the registered filtered callback passes the sources (red 40737741454)', () => { + const run = replay('filtered'); + const [file, next] = run.current.evidence_paths; + expect(call(run, 2).input.command).toBe(`cat ${shellQuote(file)}`); + const whole = result(run, 2).content; + call(run, 2).input.command = `cat ${file}; echo ===ROUTE; cat ${next}; cat missing.ts`; + Object.assign(result(run, 2), { is_error: true, content: `Exit code 1\n${whole}\n===ROUTE\ncat: missing.ts: No such file or directory` }); + expect(hasTrustedSharedLibsCheck(run.events, run.expected)).toBe(true); + expect(hasTrustedSharedLibsCheck(run.events, { ...run.expected, sources: undefined })).toBe(false); + const truncated = structuredClone(run); + result(truncated, 2).content = `Exit code 1\n${whole.slice(0, Math.floor(whole.length / 2))}`; + expect(hasTrustedSharedLibsCheck(truncated.events, truncated.expected)).toBe(false); + const single = structuredClone(run); + call(single, 2).input.command = `cat ${file}`; + expect(hasTrustedSharedLibsCheck(single.events, single.expected)).toBe(false); + const scenario = source.slice(source.indexOf("test('shared-libs-review-revalidation'")); + const marker = '}, result => {'; + const start = scenario.indexOf(marker) + marker.length; + const body = scenario.slice(start, scenario.indexOf('\n });', start)); + const verify = new Function('deps', 'result', new Bun.Transpiler({ loader: 'ts' }).transformSync(`const { + change, questions, expect, toolCommandTrace, reviewRecords, createHash, path, f, current, sources, + fixtureGit, fixtureWorkingTree, hasTrustedReviewStartRead, hasTrustedSharedLibsCheck, SHARED_LIBS_ROOT, + resumed, prerequisites, checkPathReviewPrerequisites, hasPathReviewPrerequisiteReceipt + } = deps; ${body}`)); + const native = { events: run.events, toolCalls: run.events.filter((event: any) => event.type === 'assistant') + .map((event: any) => ({ tool: event.message.content[0].name, input: event.message.content[0].input })) }; + let passed: unknown[] = []; + verify({ change: 'filtered', questions: [{}], expect, toolCommandTrace, reviewRecords, createHash, path, f: run.f, + current: run.current, sources: run.expected.sources, fixtureGit, fixtureWorkingTree, hasTrustedReviewStartRead, + SHARED_LIBS_ROOT, resumed: run.resumed, prerequisites: run.prerequisites, checkPathReviewPrerequisites, + hasPathReviewPrerequisiteReceipt, hasTrustedSharedLibsCheck: (events: unknown[], expected: any) => { + passed = Object.keys(expected.sources ?? {}); + return hasTrustedSharedLibsCheck(events, expected); + } }, native); + expect(passed.sort()).toEqual([...run.current.evidence_paths].sort()); + }); + test.each(pathKinds)('the actual %s callback consumes false proof and enforces final excluded coverage', async kind => { const run = replay(`path-${kind}`); run.kind = kind; diff --git a/test/skill-e2e-plan-design-with-ui.test.ts b/test/skill-e2e-plan-design-with-ui.test.ts index c579fac99..e0a064819 100644 --- a/test/skill-e2e-plan-design-with-ui.test.ts +++ b/test/skill-e2e-plan-design-with-ui.test.ts @@ -24,7 +24,7 @@ const ROOT = path.resolve(import.meta.dir, '..'); const FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md'); const designFocusBoundary = (fp: AskUserQuestionFingerprint): boolean => - fp.nativeCall?.answered === true && !fp.nativeCall.failed && fp.nativeCall.questions.some(({ question, options }) => { + fp.nativeCall?.answered === true && !fp.nativeCall.failed && fp.nativeCall.questions.some(({ header, question, options }) => { const text = question.trim().replace(/^D\d+(?:\.\d+)?\s*[—–:-]\s*/i, ''); // A target menu can mention a design system without reviewing this plan. // The source Step 0D question, or a native title (first line) that offers all @@ -46,7 +46,12 @@ const designFocusBoundary = (fp: AskUserQuestionFingerprint): boolean => // or focus on a few?") and lettered its options ("A) Review all 7 areas"). Either place // carries the offer; issue-numbered options are a finding, never Step 0D. Runs 41178131219 // and 41186376825 lettered the leading option itself ("A) All 7 passes (recommended)"). + // Every captured Step 0D (ten calls, runs 39199692993 to 41414111284) set the native header chip + // to "Focus", and no other captured question did; run 41414111284's label put the count second + // ("Full review, all 7 (recommended)"). With that header, the rating and an all-7 option suffice. const lead = text.split(/\r?\n|\s+(?=Project\/branch\/task:)/, 1)[0]!; + if (/^\s*focus\s*$/i.test(header ?? '') && rated(text) && options.length >= 2 && !issueLabeled(options) + && options.some(({ label }) => /\ball (?:7|seven)\b/i.test(label))) return true; return /^I(?:['’]ve| have) rated this plan (?:10(?:\.0+)?|[0-9](?:\.\d+)?)\/10 on design completeness\.[\s\S]*\bWant me to focus on specific areas instead(?: of all (?:7|seven))?\?/i.test(text) || (/\ball (?:7|seven) (?:design )?(?:dimensions|passes)\b/i.test(title) && /\bfocus\b/i.test(title) && (/\bdesign\b/i.test(title) || rated(text.slice(title.length))) && /\?\s*$/.test(title)) diff --git a/test/skill-e2e-shared-libs.test.ts b/test/skill-e2e-shared-libs.test.ts index ebe15eba4..d2c75ff44 100644 --- a/test/skill-e2e-shared-libs.test.ts +++ b/test/skill-e2e-shared-libs.test.ts @@ -215,6 +215,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => { const instructions = reviewLifecycleInstructions(f); const input = path.join(f.root, 'current-advisory.jsonl'); const { action: _priorAction, ...current } = prior; + const sources = Object.fromEntries(current.evidence_paths.map((file: string) => [file, fs.readFileSync(path.join(f.repo, file), 'utf8')])); fs.writeFileSync(input, JSON.stringify({ ...current, specialist: 'maintainability' }) + '\n'); const resumed = seedPathReviewPrerequisites(f); const prerequisites = checkPathReviewPrerequisites(f, resumed.input); @@ -248,6 +249,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => { coveredPaths: change === 'unchanged' ? current.evidence_paths : last.findings.find((finding: any) => finding.advisory && finding.action === 'skipped' && finding.fingerprint === current.fingerprint)?.snapshot_covered_paths, + sources, })).toBe(true); } else if (change === 'unchanged' || change === 'filtered') { // A true hash comparison cannot substitute for the trusted capture and path checks.