mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-09 10:18:32 +01:00
fix(flaky): coverage-audit wc -c and bounded-display credit, tpa-broken Aside absence, register docsync fixture (GSTA-240 follow-up)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
091b5113e0
commit
c3baa46815
9 files changed
+196
-11
No files matched your search
@@ -6,5 +6,8 @@
|
||||
- **`.gstack/pregate.json`**: the local pregate lane runs `bun test --timeout=30000 {files}`, the 30 s per-test timeout CI and the remote pregate already use, so a case that takes 5 s on a busy machine no longer fails only locally.
|
||||
- **`test/gstack-ship-receipt.test.ts`**: the full `write` case, about 62 git calls, gets a 15 s timeout instead of bun's 5 s default.
|
||||
|
||||
- **`test/helpers/coverage-audit-evidence.ts`**: the plan-eng coverage audit detector credits complete reads proved by a literal `wc -c` byte count, and by a bounded display (`grep | head`) when exactly one copy of the reads follows; an unbounded display, a second copy or a tampered read still gets no credit (train gates 41209768188, 41238366045).
|
||||
- **`test/helpers/third-party-actions.ts`**: a statement that Aside is not available ("Recommended, since Aside isn't available") no longer counts as offering an Aside drive in the tpa-broken check; any other Aside mention in the option still does (gate 41246728073).
|
||||
|
||||
### Changed
|
||||
- `test/helpers/docsync-contract.ts` names the missing `documentation_section` when a document-release completion fails the contract, so the next paid red says which field was absent.
|
||||
@@ -20,6 +20,8 @@ import captured_cd_chain from './fixtures/coverage-audit-cd-chain.json';
|
||||
import captured_wc_anchor from './fixtures/coverage-audit-wc-anchor.json';
|
||||
import captured_separator from './fixtures/coverage-audit-separator-collision.json';
|
||||
import captured_two_file_grep from './fixtures/coverage-audit-two-file-grep.json';
|
||||
import captured_plan_eng_wc_bytes from './fixtures/coverage-audit-plan-eng-wc-bytes.json';
|
||||
import captured_plan_eng_grep_head from './fixtures/coverage-audit-plan-eng-grep-head-before-reads.json';
|
||||
|
||||
const clone = <T>(v:T):T => structuredClone(v);
|
||||
const diagram = '```text\nsrc/billing.ts\n├── processPayment: happy path [TESTED]\n└── refundPayment [UNTESTED]\n```';
|
||||
@@ -1289,3 +1291,67 @@ describe('coverage audit native evidence: an escaped grep over two literal files
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
const capturedCall = (capture: typeof captured_plan_eng_wc_bytes) => {
|
||||
const both = { sourceRead: true, testsRead: true }, neither = { sourceRead: false, testsRead: false };
|
||||
const files = { cwd: capture.cwd, source: { path: capture.cwd + '/src/billing.ts', content: capture.files.source },
|
||||
tests: { path: capture.cwd + '/test/billing.test.ts', content: capture.files.tests } };
|
||||
const transcript = (command: string, output: string) => [
|
||||
{ type: 'system', subtype: 'init', session_id: capture.sessionId, cwd: capture.cwd },
|
||||
{ type: 'assistant', session_id: capture.sessionId, parent_tool_use_id: null, message: { role: 'assistant', content: [{ type: 'tool_use', id: capture.call.toolUseId, name: 'Bash', input: { command } }] } },
|
||||
{ type: 'user', session_id: capture.sessionId, parent_tool_use_id: null, message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: capture.call.toolUseId, is_error: capture.call.isError, content: output }] } },
|
||||
];
|
||||
const reads = (command = capture.call.command, output = capture.call.output) => coverageAuditReadEvidence(transcript(command, output), files);
|
||||
const passes = () => coverageAuditVerdict({ exitReason: 'success', browseErrors: [], output: capture.finalOutput,
|
||||
transcript: transcript(capture.call.command, capture.call.output) } as any, files);
|
||||
const nonce = capture.files.source.match(/coverage-read-evidence: (\S+)/)![1]!;
|
||||
return { both, neither, reads, passes, nonce };
|
||||
};
|
||||
|
||||
describe('coverage audit native evidence: a wc -c byte count after both reads (CI 41209768188 slice 5, plan-eng)', () => {
|
||||
const capture = captured_plan_eng_wc_bytes, { both, neither, reads, passes, nonce } = capturedCall(capture);
|
||||
const count = 'wc -c plan-eng-review/sections/review-sections.md';
|
||||
|
||||
test('both complete reads before a literal wc -c count are credited and the recorded run passes', () => {
|
||||
expect(capture.provenance.recordedPassed).toBe(false);
|
||||
expect(capture.call.command).toContain('; cat -n src/billing.ts; echo ----; cat -n test/billing.test.ts; echo ---; ' + count + '; grep -n');
|
||||
expect(reads()).toEqual(both);
|
||||
expect(passes()).toEqual({ ...both, diagram: true, passed: true, failures: [] });
|
||||
});
|
||||
|
||||
test('a wc -c operand that is not a literal in-cwd file, or a tampered read, still gets no credit', () => {
|
||||
for (const operands of ['../outside.md', '/etc/passwd', '"$(touch x)"', '-', 'plan-eng-review/SKILL.md > out.txt', '--files0-from=list']) {
|
||||
expect(reads(capture.call.command.replace(count, 'wc -c ' + operands)), operands).toEqual(neither);
|
||||
}
|
||||
expect(reads(capture.call.command.replace(count, 'wc -w plan-eng-review/sections/review-sections.md'))).toEqual(neither);
|
||||
expect(reads(capture.call.command, capture.call.output.replace(nonce, nonce.slice(1) + '0'))).toMatchObject({ sourceRead: false });
|
||||
expect(reads(capture.call.command.replace('cat -n test/billing.test.ts', 'cat -n test/other.test.ts'))).toMatchObject({ testsRead: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe('coverage audit native evidence: owned reads after a bounded grep | head display (CI 41238366045 slice 5, plan-eng)', () => {
|
||||
const capture = captured_plan_eng_grep_head, { both, neither, reads, passes, nonce } = capturedCall(capture);
|
||||
const display = 'grep -n -i "test review\\|coverage diagram" plan-eng-review/SKILL.md | head;';
|
||||
const run = capture.call.output.slice(capture.call.output.indexOf('\n') + 1);
|
||||
|
||||
test('both complete reads after a bounded display and an exact count are credited and the recorded run passes', () => {
|
||||
expect(capture.provenance.recordedPassed).toBe(false);
|
||||
expect(capture.call.command).toContain('; ' + display + ' wc -l plan-eng-review/sections/review-sections.md; cat -n src/billing.ts; echo ----; cat -n test/billing.test.ts');
|
||||
expect(reads()).toEqual(both);
|
||||
expect(passes()).toEqual({ ...both, diagram: true, passed: true, failures: [] });
|
||||
const lines = (n: number) => Array.from({ length: n }, (_, i) => 'SKILL.md:' + (i + 1) + ': test review').join('\n') + '\n';
|
||||
expect(reads(capture.call.command, lines(10) + capture.call.output)).toEqual(both);
|
||||
});
|
||||
|
||||
test('display output beyond its bound, a second copy of the reads, an unbounded display or a tampered read gets no credit', () => {
|
||||
const lines = (n: number) => Array.from({ length: n }, (_, i) => 'SKILL.md:' + (i + 1) + ': test review').join('\n') + '\n';
|
||||
expect(reads(capture.call.command, lines(11) + capture.call.output)).toEqual(neither);
|
||||
expect(reads(capture.call.command.replace('| head;', '| head -60;'), run + '\n' + capture.call.output)).toEqual(neither);
|
||||
expect(reads(capture.call.command, capture.call.output + '\n' + run)).toEqual(neither);
|
||||
for (const tail of ['| head -c 400;', ';', '| cat;']) {
|
||||
expect(reads(capture.call.command.replace('| head;', tail)), tail).toEqual(neither);
|
||||
}
|
||||
expect(reads(capture.call.command, capture.call.output.replace(nonce, nonce.slice(1) + '0'))).toMatchObject({ sourceRead: false });
|
||||
expect(reads(capture.call.command.replace('cat -n test/billing.test.ts', 'cat -n test/other.test.ts'))).toMatchObject({ testsRead: false });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"provenance": {
|
||||
"source": "https://github.com/garrytan/gstack/actions/runs/41238366045 eval slice 5 (train gate, train 0926fe2e), plan-eng-coverage-audit (claude-opus-5-5); the second parent Bash call and its result, byte-exact",
|
||||
"recordedPassed": false,
|
||||
"recordedFailures": [
|
||||
"missing successful source-file read",
|
||||
"missing successful test-file read"
|
||||
],
|
||||
"note": "The first call printed 109 KB and the native result kept only a preview, so it delivered neither file. The second call ran a grep | head display (0 to 10 lines) and a wc -l count before reading both files completely with no label in between; an owned run after unknown output of inexact length was refused outright, so the whole chain got no credit."
|
||||
},
|
||||
"cwd": "/home/runner/.cache/gstack-paid-shard-mTTODF/tmp/skill-e2e-plan-eng-review-coverage-uevVNY",
|
||||
"sessionId": "8186288c-6f03-42de-87f6-64bf80fc74cf",
|
||||
"call": {
|
||||
"toolUseId": "toolu_01X9wLDqv1PS1FuGjcidiZoF",
|
||||
"command": "cd /home/runner/.cache/gstack-paid-shard-mTTODF/tmp/skill-e2e-plan-eng-review-coverage-uevVNY; grep -n -i \"test review\\|coverage diagram\" plan-eng-review/SKILL.md | head; wc -l plan-eng-review/sections/review-sections.md; cat -n src/billing.ts; echo ----; cat -n test/billing.test.ts",
|
||||
"isError": false,
|
||||
"output": "1585 plan-eng-review/sections/review-sections.md\n 1\t\n 2\texport function processPayment(amount: number, currency: string) {\n 3\t if (amount <= 0) throw new Error('Invalid amount');\n 4\t if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n 5\t return { status: 'success', amount, currency };\n 6\t}\n 7\t\n 8\texport function refundPayment(paymentId: string, reason: string) {\n 9\t if (!paymentId) throw new Error('Payment ID required');\n 10\t if (!reason) throw new Error('Reason required');\n 11\t return { status: 'refunded', paymentId, reason };\n 12\t}\n 13\t\n 14\t// coverage-read-evidence: 3320eaa9-e797-42f5-80d3-4faa22c320e0\n----\n 1\t\n 2\timport { describe, test, expect } from 'vitest';\n 3\timport { processPayment } from '../src/billing';\n 4\t\n 5\tdescribe('processPayment', () => {\n 6\t test('processes valid payment', () => {\n 7\t const result = processPayment(100, 'USD');\n 8\t expect(result.status).toBe('success');\n 9\t });\n 10\t // GAP: no test for invalid amount\n 11\t // GAP: no test for unsupported currency\n 12\t // GAP: refundPayment not tested at all\n 13\t});\n 14\t\n 15\t// coverage-read-evidence: 93992020-084f-411f-8800-10e1e9885342"
|
||||
},
|
||||
"files": {
|
||||
"source": "\nexport function processPayment(amount: number, currency: string) {\n if (amount <= 0) throw new Error('Invalid amount');\n if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n return { status: 'success', amount, currency };\n}\n\nexport function refundPayment(paymentId: string, reason: string) {\n if (!paymentId) throw new Error('Payment ID required');\n if (!reason) throw new Error('Reason required');\n return { status: 'refunded', paymentId, reason };\n}\n\n// coverage-read-evidence: 3320eaa9-e797-42f5-80d3-4faa22c320e0\n",
|
||||
"tests": "\nimport { describe, test, expect } from 'vitest';\nimport { processPayment } from '../src/billing';\n\ndescribe('processPayment', () => {\n test('processes valid payment', () => {\n const result = processPayment(100, 'USD');\n expect(result.status).toBe('success');\n });\n // GAP: no test for invalid amount\n // GAP: no test for unsupported currency\n // GAP: refundPayment not tested at all\n});\n\n// coverage-read-evidence: 93992020-084f-411f-8800-10e1e9885342\n"
|
||||
},
|
||||
"finalOutput": "**Section 3. Test review: targeted audit of `src/billing.ts` against `test/billing.test.ts`**\n\nThe tests use Vitest (`import ... from 'vitest'` in `test/billing.test.ts:2`). I read both files only and did not run the tests.\n\n```\nCODE PATHS USER FLOWS\n[+] src/billing.ts [+] Take a payment\n ├── processPayment(amount, currency) :2-6 ├── [GAP] [→E2E] Pay → success response\n │ ├── [★★ TESTED] Valid USD → {status:'success'} └── [GAP] [→E2E] Pay with bad input → error shown\n │ │ — billing.test.ts:6 (checks .status \n │ │ only, not amount/currency) [+] Refund a payment\n │ ├── [GAP] Valid EUR → success (other side ├── [GAP] [→E2E] Refund → refunded response\n │ │ of the currency guard) :4 └── [GAP] Refund without reason → error\n │ ├── [GAP] amount <= 0 → throw \n │ │ 'Invalid amount' (0, -1) :3 [+] Error states\n │ └── [GAP] currency ∉ {USD,EUR} → throw ├── [GAP] 'Invalid amount' surfaced to caller\n │ 'Unsupported currency' :4 └── [GAP] 'Unsupported currency' surfaced\n │ (e.g. 'GBP', 'usd', '') \n └── refundPayment(paymentId, reason) :8-12 \n ├── [GAP] !paymentId ('') → throw \n │ 'Payment ID required' :9 \n ├── [GAP] !reason ('') → throw \n │ 'Reason required' :10 \n └── [GAP] Valid → {status:'refunded', \n paymentId, reason} :11 \n (never imported or called by tests)\n\nCOVERAGE: 1/13 paths tested (8%) | Code paths: 1/7 (14%) | User flows: 0/6 (0%)\nQUALITY: ★★★:0 ★★:1 ★:0 | GAPS: 12 (3 marked [→E2E])\n```\nLegend: ★★★ behavior + edge cases + error paths | ★★ happy path only | ★ smoke check | [→E2E] needs an integration test\n\n**Coverage summary**\n- **`processPayment`** is partly covered (1 of 4 code paths). There is one ★★ test for the valid-USD case, and it only checks `status`. Not tested: the `amount <= 0` error (including the `0` boundary), the unsupported-currency error, and a valid EUR payment.\n- **`refundPayment`** has no coverage (0 of 3 code paths). The test file never imports it, so both errors and the success result are untested.\n- **User flows:** this is a payment and refund flow, so the E2E matrix suggests integration tests for checkout and refund. None exist yet.\n- **Fastest gains:** add the missing `processPayment` cases (`0`, `-1`, `'GBP'`, `'EUR'`) as rows in one table-driven test in `billing.test.ts`, and add a `describe('refundPayment')` block covering the empty-ID error, the empty-reason error and the success result.\n\nAs you asked, I skipped the other review sections and workflow steps (including the test plan artifact), and left the source and tests unchanged.\n\n{\"tested\": [\"processPayment\"], \"untested\": [\"refundPayment\"]}"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"provenance": {
|
||||
"source": "https://github.com/garrytan/gstack/actions/runs/41209768188 eval slice 5 (train gate, train d7441d9d), plan-eng-coverage-audit (claude-opus-5-5); the second parent Bash call and its result, byte-exact",
|
||||
"recordedPassed": false,
|
||||
"recordedFailures": [
|
||||
"missing successful source-file read",
|
||||
"missing successful test-file read"
|
||||
],
|
||||
"note": "The first call printed 109 KB and the native result kept only a preview, so it delivered neither file. The second call read both files completely in a ; chain, then printed a byte count with wc -c over a literal in-cwd file and a grep display; wc -c was not a closed display stage (only wc -l was), so the whole chain got no credit."
|
||||
},
|
||||
"cwd": "/home/runner/.cache/gstack-paid-shard-yPuCXj/tmp/skill-e2e-plan-eng-review-coverage-9aEaFX",
|
||||
"sessionId": "9e250e03-5ab5-4d6f-a605-646ba6d44306",
|
||||
"call": {
|
||||
"toolUseId": "toolu_01RsKm1skedEXrons396Cg2N",
|
||||
"command": "cd /home/runner/.cache/gstack-paid-shard-yPuCXj/tmp/skill-e2e-plan-eng-review-coverage-9aEaFX; cat -n src/billing.ts; echo ----; cat -n test/billing.test.ts; echo ---; wc -c plan-eng-review/sections/review-sections.md; grep -n \"Test review\" -A80 plan-eng-review/sections/review-sections.md | head -200",
|
||||
"isError": false,
|
||||
"output": " 1\t\n 2\texport function processPayment(amount: number, currency: string) {\n 3\t if (amount <= 0) throw new Error('Invalid amount');\n 4\t if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n 5\t return { status: 'success', amount, currency };\n 6\t}\n 7\t\n 8\texport function refundPayment(paymentId: string, reason: string) {\n 9\t if (!paymentId) throw new Error('Payment ID required');\n 10\t if (!reason) throw new Error('Reason required');\n 11\t return { status: 'refunded', paymentId, reason };\n 12\t}\n 13\t\n 14\t// coverage-read-evidence: 1e8364a6-52aa-42a0-8c25-33c0a2dcb7ea\n----\n 1\t\n 2\timport { describe, test, expect } from 'vitest';\n 3\timport { processPayment } from '../src/billing';\n 4\t\n 5\tdescribe('processPayment', () => {\n 6\t test('processes valid payment', () => {\n 7\t const result = processPayment(100, 'USD');\n 8\t expect(result.status).toBe('success');\n 9\t });\n 10\t // GAP: no test for invalid amount\n 11\t // GAP: no test for unsupported currency\n 12\t // GAP: refundPayment not tested at all\n 13\t});\n 14\t\n 15\t// coverage-read-evidence: 248d88f4-56af-41a0-bed2-18d23cf0027a\n---\n100809 plan-eng-review/sections/review-sections.md\n23:When Test review or Outside Voice refers to the plan, use the current working\n24-plan and this target evidence. Trace current behavior and proposed changes\n25-separately. Include actual decisions in Outside Voice's bounded input.\n26-\n27-Choose the **report file** before any ledger write:\n28-1. Use the output/report path explicitly requested by the user.\n29-2. Otherwise use the selected plan file, if there is one.\n30-3. Otherwise use `$GSTACK_STATE_ROOT/projects/$SLUG/$BRANCH-eng-review-{YYYYMMDD-HHMMSS}.md`, adding a suffix on collision. Obtain assignments from `~/.claude/skills/gstack/bin/gstack-paths` and `~/.claude/skills/gstack/bin/gstack-slug`; failed commands or missing values make this path unavailable.\n31-\n32-Never substitute an unrelated active plan or silently replace a requested destination.\n33-\n34-**Check each artifact and parent directory's permission before writing.** Honor\n35-user and host limits, including active-plan-only restrictions. Permission for one\n36-path authorizes no other; implementation edits require explicit authority.\n37-\n38-| Artifact | Destination | If writing is forbidden |\n39-|---|---|---|\n40-| Working plan, ledger and complete review report | Selected report file | Ask for a permitted destination if the user can supply one; wait without completion telemetry. If none is permitted, use **Read-only review** below, then **Report-back outcome**. |\n41-| QA Test Plan and task JSONL | Discovery paths below | Present each completely as **not persisted** and continue. |\n42-| TODOS.md | The project's TODO file | Present accepted TODO content as **not persisted** and continue. |\n43-| Required Review Log | The helper's state location | Present its fields as **not persisted**; at Review Log, use **Report-back outcome** instead of publishing a saved review. The final gate cannot pass without this log. |\n44-| Best-effort metadata/learning logs | Helper-defined locations | Skip forbidden writes; otherwise keep their best-effort behavior. |\n45-\n46-QA Test Plan/task JSONL keep discovery paths `$GSTACK_STATE_ROOT/projects/{slug}/`:\n47-`{user}-{branch}-eng-review-test-plan-{datetime}.md` and\n48-`tasks-eng-review-{datetime}.jsonl`. Keep their formats; do not relocate.\n49-\n50-**First report save:** Name the fixed target in the report header. Read an\n51-existing destination and preserve its content. For a new file, create permitted\n52-parent directories, then write that header, an unchanged copy of the original\n53-plan (for plan targets), and the scope record or ledger being saved. Recheck option 3's collision before\n54-creation; use a suffix rather than overwrite. Findings and fixes first enter\n55-through Scope Challenge C's ledger saves, never earlier. Put records before an existing `## GSTACK REVIEW REPORT`, or\n56-at EOF if absent; create that terminal report only at Plan File Review Report.\n57-\n58-**Write routes.** A forbidden write follows its row above.\n59-- **Read-only review:** At each scope/decision record save, present the complete\n60- record, grid and authorized amendments as **not persisted** instead. At both\n61- pre-question and post-answer verification gates, perform the same comparisons\n62- on that presentation instead of a saved Read. This supports chat review, never\n63- the saved-report gate.\n64-- **Failed permitted save** (error or failed Read-back): use **Recovery routing →\n65- Repairable write/read failure**, never the routes above. Do not ask from an\n66- unsaved record; an unrecovered failed write blocks the review.\n67-\n68-## Prior Learnings\n69-\n70-Search for relevant learnings fromLine truncated
|
||||
},
|
||||
"files": {
|
||||
"source": "\nexport function processPayment(amount: number, currency: string) {\n if (amount <= 0) throw new Error('Invalid amount');\n if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n return { status: 'success', amount, currency };\n}\n\nexport function refundPayment(paymentId: string, reason: string) {\n if (!paymentId) throw new Error('Payment ID required');\n if (!reason) throw new Error('Reason required');\n return { status: 'refunded', paymentId, reason };\n}\n\n// coverage-read-evidence: 1e8364a6-52aa-42a0-8c25-33c0a2dcb7ea\n",
|
||||
"tests": "\nimport { describe, test, expect } from 'vitest';\nimport { processPayment } from '../src/billing';\n\ndescribe('processPayment', () => {\n test('processes valid payment', () => {\n const result = processPayment(100, 'USD');\n expect(result.status).toBe('success');\n });\n // GAP: no test for invalid amount\n // GAP: no test for unsupported currency\n // GAP: refundPayment not tested at all\n});\n\n// coverage-read-evidence: 248d88f4-56af-41a0-bed2-18d23cf0027a\n"
|
||||
},
|
||||
"finalOutput": "## Test review — targeted audit of `src/billing.ts` (feature/billing)\n\nTest framework: **vitest** (found `vitest.config.ts` and `vitest` in devDependencies). The `package.json` `test` script is only a stub: `echo \"no tests yet\"`. Coverage below comes from reading the code and tests. No tests were run.\n\n```\nCODE PATHS USER FLOWS\n[+] src/billing.ts [+] Charge a customer\n ├── processPayment(amount, currency) :2-6 ├── [★★ TESTED] Valid USD charge — billing.test.ts:6\n │ ├── [GAP] amount <= 0 → throw ├── [GAP] [→E2E] Rejected amount → visible error\n │ │ 'Invalid amount' :3 └── [GAP] [→E2E] Unsupported currency → visible error\n │ │ └── [GAP] boundary: amount = 0, -1, NaN\n │ ├── [GAP] currency ∉ {USD,EUR} → throw [+] Refund a payment\n │ │ 'Unsupported currency' :4 ├── [GAP] [→E2E] Successful refund\n │ ├── [★★ TESTED] USD → {status:'success',...} └── [GAP] [→E2E] Refund without ID/reason → error\n │ │ billing.test.ts:6-9\n │ │ (asserts status only; amount/\n │ │ currency echo not asserted)\n │ └── [GAP] EUR → success branch :4-5\n │\n └── refundPayment(paymentId, reason) :8-12 ◄── NO TESTS (not even imported)\n ├── [GAP] !paymentId ('' / undefined) → throw 'Payment ID required' :9\n ├── [GAP] !reason ('' / undefined) → throw 'Reason required' :10\n └── [GAP] valid → {status:'refunded', paymentId, reason} :11\n\nCOVERAGE: 2/12 paths tested (17%) | Code paths: 1/7 (14%) | User flows: 1/5 (20%)\nQUALITY: ★★★:0 ★★:2 ★:0 | GAPS: 10 (4 E2E)\n```\nLegend: ★★★ behavior + edge + error | ★★ happy path | ★ smoke check | [→E2E] = needs integration test\n\nThe two ★★ ticks are the same test at `billing.test.ts:6`. It counts once as a code path and once as a user flow.\n\n### Coverage summary\n\n- **`processPayment`: partially tested, happy path only.** One test checks a valid USD payment and only looks at `result.status`. Nothing tests either error branch: the `amount <= 0` check, including the `0` boundary, or the currency check. The EUR success branch is also untested.\n- **`refundPayment`: not tested at all.** The test file doesn't import it. None of its three paths (missing payment ID, missing reason, success) is tested.\n- **Biggest risks:** if either `processPayment` check were removed or flipped, the suite would still pass. That would let invalid amounts or unsupported currencies through, and payment paths are where that matters most. Any change to `refundPayment` would go completely undetected.\n- **What would close the gaps:** this is only a description, since writing tests was out of scope. Add table-driven tests in the existing `describe('processPayment')` block for the invalid amounts (`0`, `-1`) and an unsupported currency (e.g. `'GBP'`). Add EUR as another success row and check the whole returned object. Add a new `describe('refundPayment')` block covering the two `throw` checks and the success return. Plain unit tests are enough because both are pure functions. The [→E2E] marks only matter once a checkout or refund UI or API calls these functions.\n\nSource and test files were not modified.\n\n{\"tested\": [\"processPayment\"], \"untested\": [\"refundPayment\"]}"
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"source": "assistantText of the claude-opus-5-5 tpa-broken gate records in run 41246728073 (PR #3727 @ ff81a147): paid-slice-16-a1 (recorded red: asideDriveOptions returned the B) gstack-drive option) and its in-run rule-16 rerun (green)",
|
||||
"renders": [
|
||||
{
|
||||
"case": "tpa-broken",
|
||||
"run": "41246728073",
|
||||
"attempt": "slice-16-a1",
|
||||
"recordedPassed": false,
|
||||
"assistantText": "The contract says to run the Aside readiness probe before asking anything, so I'll do that first.\nThe Aside readiness check came back as `ASIDE_CLI_ERROR` (exit 1). Under the policy, that means Aside is not detected for this task, so I've left out the option to drive your Aside browser. This machine is Linux, so I won't suggest installing Aside either. I haven't installed or browsed anything.\n\n---\n\n**Question: How should I create the test-mode API token in the Acme vendor dashboard (https://dashboard.acme.test)?**\n\nThe task is to create a test-mode API token on dashboard.acme.test. I would only touch that site and only to create that token. You would handle any sign-in, password, CAPTCHA or identity check yourself. The token would never be shown in chat. It would go into a local file only you can read (0600) at a path you approve. I'd then check it with one read-only API call.\n\n- **B)** I drive it in gstack's own visible browser, and hand it to you whenever sign-in is needed. **(Recommended, since Aside isn't available)**\n- **C)** I give you step-by-step instructions and you do it yourself.\n- **D)** Skip it for now and mark this step as blocked until you do it.\n\nReply with B, C or D. This answer covers this task only.\n\n---\n\nI've stopped here, as you asked."
|
||||
},
|
||||
{
|
||||
"case": "tpa-broken",
|
||||
"run": "41246728073",
|
||||
"attempt": "rerun-1",
|
||||
"recordedPassed": true,
|
||||
"assistantText": "The user asked for the detection, so I'm running the readiness probe the contract specifies.\nI ran the Aside readiness probe and it returned `ASIDE_CLI_ERROR: exit 1`. Under the policy, only `READY` counts as detected, so the Aside browser isn't available for this task and I've left that option out. I have not opened a browser or done anything else.\n\nThe file also contains an unrelated \"Ship\" release workflow below the third-party actions section. It has nothing to do with this task, so I didn't follow it.\n\n**Question: How do you want to handle this?**\n\nSite: https://dashboard.acme.test (Acme vendor dashboard)\nAction: create a test-mode API token\n\n- **B) I drive it in gstack's own visible browser.** I open the dashboard and go to the API token page. If a sign-in, password, CAPTCHA or identity check appears, I hand control to you. You complete it and tell me \"done\", then I continue. I only touch the Acme dashboard and only to create the token. You copy the token yourself with the dashboard's copy button and save it to a local file you approve, with owner-only (0600) permissions. Then I check it with one read-only API call. The token never appears in chat.\n- **C) Manual instructions.** I write out the steps for you to do yourself, and this step stays blocked until you finish.\n- **D) Defer.** We skip this for now and mark the step blocked on you.\n\nReply B, C or D. Your choice applies to this task only."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -140,8 +140,8 @@ function stagePieces(stage: string[], scope: ShellScope, alone: boolean): Piece[
|
||||
}
|
||||
if (name === 'grep' || name === 'ls') return [unknown()];
|
||||
if (text === 'pwd' || text === 'wc -l') return [unknown(1, 1)];
|
||||
if (name === 'wc' && args[0] === '-l' && args.length > 1) {
|
||||
// One count line per literal in-cwd file, plus a total for several.
|
||||
if (name === 'wc' && (args[0] === '-l' || args[0] === '-c') && args.length > 1) {
|
||||
// One line or byte count line per literal in-cwd file, plus a total for several.
|
||||
if (args.slice(1).some(operand => !scopedFile(operand, scope))) return undefined;
|
||||
const lines = args.length - 1 + (args.length > 2 ? 1 : 0);
|
||||
return [unknown(lines, lines)];
|
||||
@@ -254,14 +254,27 @@ function shellReads(command: unknown, output: unknown, scope: ShellScope): Set<s
|
||||
if (anchor.kind !== 'label') {
|
||||
// Output of an exact line count (wc -l over literal in-cwd files, pwd)
|
||||
// places the next run exactly that many lines after the cursor.
|
||||
if (min !== max) return new Set();
|
||||
let start = cursor;
|
||||
for (let line = 0; line < min; line++) {
|
||||
const end = out.indexOf('\n', start);
|
||||
if (end < 0) return new Set();
|
||||
start = end + 1;
|
||||
if (min !== max) {
|
||||
// Bounded output (grep | head) may end on any line in its bounds, so
|
||||
// credit only when exactly one line after the cursor starts a
|
||||
// complete copy of the run; the bounds are checked below.
|
||||
if (!Number.isFinite(max)) return new Set();
|
||||
const pattern = new RegExp(run.map(pieceSource).join(''), 'y'), copies: number[] = [];
|
||||
for (let start = cursor; start <= out.length; start = out.indexOf('\n', start) + 1 || out.length + 1) {
|
||||
pattern.lastIndex = start;
|
||||
if (pattern.test(out)) copies.push(start);
|
||||
}
|
||||
if (copies.length !== 1) return new Set();
|
||||
starts = copies;
|
||||
} else {
|
||||
let start = cursor;
|
||||
for (let line = 0; line < min; line++) {
|
||||
const end = out.indexOf('\n', start);
|
||||
if (end < 0) return new Set();
|
||||
start = end + 1;
|
||||
}
|
||||
starts = [start];
|
||||
}
|
||||
starts = [start];
|
||||
} else {
|
||||
let found = anywhere(anchor.text);
|
||||
if (owns) {
|
||||
|
||||
@@ -121,6 +121,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/docsync-replay/37237194905-late-result-calls.json': ['test/docsync-authority.test.ts'],
|
||||
'test/fixtures/docsync-replay/37239016858-completion-native.json': ['test/docsync-nested-writes.test.ts'],
|
||||
'test/fixtures/docsync-ship-receipt-not-found-public.json': ['test/docsync-lifecycle-interface.test.ts'],
|
||||
'test/fixtures/docsync-spawned-missing-section-public.json': ['test/ship-document-release-dispatch.test.ts'],
|
||||
'test/fixtures/docsync-store-38969924945-bash.json': ['test/docsync-command-grammar.test.ts'],
|
||||
'test/fixtures/docsync-two-attempts-public.json': ['test/ship-document-release-dispatch.test.ts'],
|
||||
'test/fixtures/dx-manual-handoff-ao.json': ['test/plan-count-completion.test.ts'],
|
||||
|
||||
@@ -34,14 +34,22 @@ export function letteredOptions(text: string): string[] {
|
||||
return options;
|
||||
}
|
||||
|
||||
/**
|
||||
* A statement that Aside is absent, the probe's result, names no Aside drive: "(Recommended, since Aside isn't
|
||||
* available)" on the gstack drive option (gate 41246728073). Only Aside as the subject of a negated availability
|
||||
* predicate is discounted, so any other Aside mention in the option still counts.
|
||||
*/
|
||||
const ASIDE_ABSENCE = /\bAside(?: browser| app)?\s+(?:(?:is|was)(?:n['’]t|\s+not)\s+(?:available|installed|detected|running|ready|reachable)|is\s+unavailable)\b/gi;
|
||||
|
||||
/** Inspect lettered consent options, not narration that quotes an unavailable option. */
|
||||
export function asideDriveOptions(text: string): string[] {
|
||||
return letteredOptions(text).filter(option => {
|
||||
const currentOffer = option.replace(
|
||||
const named = option.replace(ASIDE_ABSENCE, '');
|
||||
const currentOffer = named.replace(
|
||||
/\b(?:I|we)(?:['’]ll| will) (?:re[- ]?ask|ask again)(?: (?:you|this question))? with (?:the )?(?:"[^"]+"|“[^”]+”|[\w -]+?) option(?: included)?\b/gi,
|
||||
'',
|
||||
);
|
||||
return /\bAside\b/.test(option) && (/\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(currentOffer)
|
||||
return /\bAside\b/.test(named) && (/\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(currentOffer)
|
||||
|| /\bI(?:['’]ll| will)? (?:do|handle|create|complete) (?:it|this)\b/i.test(currentOffer));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ import manualWording from './fixtures/third-party-actions-manual-wording.json';
|
||||
import opusRenders from './fixtures/third-party-actions-opus-consent-renders.json';
|
||||
import recoveryFixture from './fixtures/third-party-actions-recovery-public.json';
|
||||
import pitchCaptures from './fixtures/third-party-actions-pitch-captures.json';
|
||||
import brokenAbsence from './fixtures/third-party-actions-broken-absence-reason.json';
|
||||
import { expectMentions, expectTokens } from './helpers/prompt-structure';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, "..");
|
||||
@@ -611,3 +612,29 @@ describe('Opus consent renders (gate captures)', () => {
|
||||
'A) You generate it yourself on any device.\nB) You enter it through the masked prompt.']) expect(offersDriveOption(text), text).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// Gate 41246728073 slice 16 (PR #3727): Opus offered only the gstack drive and gave the
|
||||
// recommendation's reason as "since Aside isn't available". A statement that Aside is
|
||||
// absent names no Aside drive; any other Aside mention in the same option still does.
|
||||
describe('consent offer extraction: an Aside-absence reason on the gstack drive option', () => {
|
||||
test('the captured tpa-broken renders offer no Aside drive', () => {
|
||||
const [red, rerun] = brokenAbsence.renders;
|
||||
expect(red.recordedPassed).toBe(false);
|
||||
expect(letteredOptions(red.assistantText)[0]).toContain("(Recommended, since Aside isn't available)");
|
||||
for (const render of brokenAbsence.renders) expect(asideDriveOptions(render.assistantText), render.attempt).toEqual([]);
|
||||
expect(offersDriveOption(rerun.assistantText)).toBe(true);
|
||||
});
|
||||
|
||||
test('absence statements are discounted; an Aside drive beside them is still caught', () => {
|
||||
for (const option of ["B) I drive it in gstack's own visible browser (Recommended, since Aside isn't available)",
|
||||
"B) I drive it in gstack's own visible browser, because the Aside app is not running.",
|
||||
"B) I drive it in gstack's own browser; Aside wasn't detected.", "B) I drive it in gstack's own browser (Aside is unavailable).",
|
||||
"B) I drive it in gstack's own browser since Aside isn’t installed."])
|
||||
expect(asideDriveOptions(option), option).toEqual([]);
|
||||
for (const option of ["A) I drive it in your Aside browser, even though Aside isn't running yet.",
|
||||
"B) I drive it in gstack's own browser, or in Aside once Aside is not busy.", "B) Since Aside isn't available, I drive it in Aside's fallback window.",
|
||||
"A) Aside isn't available yet, so open it; then I drive it in your Aside browser.", "A) I drive it in Aside; gstack isn't available.",
|
||||
"B) I drive it in gstack's own browser and switch to Aside if it's running."])
|
||||
expect(asideDriveOptions(option), option).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user