mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 15:57:33 +01:00
fix(evals): repair the five Opus gate reds from run 39161502626
- coverage audit: owned cd; followed by one && chain, wc -l on in-cwd files - shared-libs: Claude Code 2.1.292 git --version probe, exact argv only - ship hook: accept 'integrate the hook yourself' / 'combine ... by hand' - qa-only: deadline starts only after the charter Write completes - plan floor: tool-only parent session identifies the owned Create grant; floor runner fails fast with PERMISSION_DIALOG_UNANSWERED Each repair replays its captured run as a free regression with negative controls. HARNESS_VERSION stays 6 (re-recorded from main's 5). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
d182b91397
commit
e03bfeb060
19 files changed
+415
-24
No files matched your search
+10
-2
@@ -8,8 +8,16 @@
|
||||
- **Model under test:** `CLAUDE_CAPTURE_EVAL_MODEL` in `lib/eval-model.ts` (renamed from `CLAUDE_FRONTIER_EVAL_MODEL`; Opus 5.5 is the catalog's smart tier, not its frontier) is `claude-opus-5-5`. Every `claude -p`, PTY and Agent SDK eval runner passes it as `--model` unless a test names a model or `EVALS_MODEL` is set.
|
||||
- **The judge stays Fable 5.1.** The new `CLAUDE_JUDGE_EVAL_MODEL` (`claude-fable-5-1`) is the `judge` default, the arm benchmark's judge and the judge calibration configs' model, because the calibration fixtures are keyed to it. The `claude-fable-5-*` → `fable-5` overlay routing now holds while the judge is a Fable 5 release.
|
||||
- **plan-review-report** drops its `claude-opus-4-7` pin and follows the capture default; it was the only PR-gate case on Opus 4.7. The periodic Opus 4.7 files keep their pins (TODOS).
|
||||
- **A new pass-rate series:** `HARNESS_VERSION` 5 → 6, so no Fable trial pools with an Opus one, and shard-reuse receipts already key on `lib/eval-model.ts`. Warmup and distill stay on Haiku.
|
||||
- **Gate integrity:** no threshold, assertion, trial count, case kind or judge changed. Test edits only update asserted model IDs and their prices, plus one assertion naming `scripts/harness-version.json` so pregate maps the manifest to a test lane.
|
||||
- **A new pass-rate series:** `HARNESS_VERSION` 5 → 6 (which also covers the floor runner change below), so no Fable trial pools with an Opus one, and shard-reuse receipts already key on `lib/eval-model.ts`. Warmup and distill stay on Haiku.
|
||||
- **Gate integrity:** no threshold, trial count, case kind or judge changed. The detector and prompt edits are listed under Fixed. Eight floor tests that expected `timeout` for a permission dialog nobody answered now expect the named failure `permission_unanswered`. Other test edits update asserted model IDs and their prices, plus one assertion naming `scripts/harness-version.json` so pregate maps the manifest to a test lane.
|
||||
|
||||
### Fixed
|
||||
The second Opus gate (run 39161502626) went 127/132. Each of the five reds is now replayed from its captured run as a free regression, with negative controls that still fail.
|
||||
- **review-coverage-audit:** the read detector rejected `cd <cwd>; cat -n … && echo ---- && cat -n …` and a trailing `wc -l <files>`. A leading `cd` to the recorded cwd followed by `;` may now precede one uniform `&&` chain, and `wc -l` accepts literal files inside the cwd. Other separator mixes and outside paths still fail.
|
||||
- **shared-libs-unsupported-git:** Claude Code 2.1.292's own `git --version` probe (which starts with `core.askPass=`) is recognized as that exact argv only. Any added argument, a non-Claude parent or a model command that copies it still fails.
|
||||
- **ship-local-hook-preservation:** "integrate the hook yourself" and "combine … by hand" now count as reporting manual integration, beside "manual".
|
||||
- **qa-only-no-fix:** the fixture prompt now says the 30-second guard starts only after the charter Write has completed. The budget and the assertion are unchanged.
|
||||
- **plan-ceo-finding-floor:** Opus saved its D1 decision to the owned working plan before printing any text, as the 0D Save step requires. The permission check identified the parent session only from text or questions, so it never granted that Create and the run timed out after 600 s. A parent with only tool events is now identified by them, still as exactly one session that owns the pending Write. The floor runner also fails within 20 s with `PERMISSION_DIALOG_UNANSWERED` when a permission dialog gets no owned grant.
|
||||
|
||||
### If you…
|
||||
- **…compare pass rates across the switch:** Fable trials carry `harness_version: 5` and Opus trials `6`; they are separate series.
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
"test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":6},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"6faa013d55901f59ae7b7d1ef6f3a9ee6c9a3034","since":6},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"57fa44c31300754b00473276cf7b972d4e9e47d8","since":6},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
|
||||
"test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6},
|
||||
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6},
|
||||
|
||||
@@ -16,6 +16,7 @@ import fixture_coverage_shell_display_aq from './fixtures/coverage-shell-display
|
||||
import billing_coverage_shell_display_aq from './fixtures/coverage-audit-ae.json';
|
||||
import captured_parallel_column from './fixtures/coverage-audit-parallel-column.json';
|
||||
import captured_sed_context from './fixtures/coverage-audit-sed-context.json';
|
||||
import captured_cd_chain from './fixtures/coverage-audit-cd-chain.json';
|
||||
|
||||
const clone = <T>(v:T):T => structuredClone(v);
|
||||
const diagram = '```text\nsrc/billing.ts\n├── processPayment: happy path [TESTED]\n└── refundPayment [UNTESTED]\n```';
|
||||
@@ -537,9 +538,13 @@ test('AF complete literal reads permit a successful chain and one leading owned
|
||||
}
|
||||
});
|
||||
|
||||
test('AF the new conditional-chain grammar conservatively rejects mixed separators', () => {
|
||||
const v=delivered(`cd ${actual(2).cwd}; ${both}`);
|
||||
expect(v.sourceRead).toBe(false); expect(v.testsRead).toBe(false);
|
||||
test('AF only an owned `cd CWD;` may precede a uniform && chain; other separator mixes stay rejected', () => {
|
||||
const cwd=actual(2).cwd, accepted=delivered(`cd ${cwd}; ${both}`,undefined,fixture.files.source+fixture.files.tests);
|
||||
expect(accepted.sourceRead).toBe(true); expect(accepted.testsRead).toBe(true);
|
||||
for (const command of [`cd ${cwd} && cat -n src/billing.ts; cat -n test/billing.test.ts`, `cd ${cwd}; ${both}; pwd`,
|
||||
`cd ${cwd}; cat -n src/billing.ts; cat -n test/billing.test.ts && pwd`, `cd ${cwd} || ${both}`, `cd /foreign; ${both}`]) {
|
||||
const v=delivered(command,undefined,fixture.files.source+fixture.files.tests); expect(v.sourceRead).toBe(false); expect(v.testsRead).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('AF read recognition rejects foreign or midstream cwd changes and nonliteral targets', () => {
|
||||
@@ -1147,3 +1152,42 @@ describe('coverage audit native evidence: pathspec git displays', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('coverage audit native evidence: captured owned-cd chains (CI 39161502626 slice 3)', () => {
|
||||
const capture = captured_cd_chain;
|
||||
const [wcChain, andChain] = capture.calls;
|
||||
const both = { sourceRead: true, testsRead: true }, neither = { sourceRead: false, testsRead: false };
|
||||
const files = { cwd: capture.cwd, source: { path: capture.cwd + '/src/billing.ts', content: capture.files.source },
|
||||
tests: { path: capture.cwd + '/test/billing.test.ts', content: capture.files.tests } };
|
||||
const transcript = (call: typeof wcChain, command = call!.command, output = call!.output) => [
|
||||
{ type: 'system', subtype: 'init', session_id: capture.sessionId, cwd: capture.cwd },
|
||||
{ type: 'assistant', session_id: capture.sessionId, parent_tool_use_id: null, message: { role: 'assistant', content: [{ type: 'tool_use', id: call!.toolUseId, name: 'Bash', input: { command } }] } },
|
||||
{ type: 'user', session_id: capture.sessionId, parent_tool_use_id: null, message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: call!.toolUseId, is_error: call!.isError, content: output }] } },
|
||||
];
|
||||
const reads = (call: typeof wcChain, command?: string, output?: string) => coverageAuditReadEvidence(transcript(call, command, output), files);
|
||||
|
||||
test('both recorded reads deliver the owned files and the run passes', () => {
|
||||
expect(capture.provenance.recordedPassed).toBe(false);
|
||||
expect(reads(wcChain)).toEqual(both);
|
||||
expect(reads(andChain)).toEqual(both);
|
||||
expect(coverageAuditVerdict({ exitReason: 'success', browseErrors: [], output: capture.finalOutput, transcript: transcript(wcChain) } as any, files))
|
||||
.toEqual({ ...both, diagram: true, passed: true, failures: [] });
|
||||
});
|
||||
|
||||
test('a chain that never reads an owned marker file gets no credit for it', () => {
|
||||
expect(reads(wcChain, wcChain!.command.replace('cat src/billing.ts', 'cat src/billing.ts.orig'))).toEqual({ sourceRead: false, testsRead: true });
|
||||
expect(reads(andChain, andChain!.command.replace('cat -n test/billing.test.ts', 'grep -c GAP test/billing.test.ts'))).toEqual({ sourceRead: true, testsRead: false });
|
||||
expect(reads(wcChain, wcChain!.command, capture.files.source + capture.files.tests)).toEqual(neither);
|
||||
});
|
||||
|
||||
test('wc -l counts only literal in-cwd files and stays inside its line bounds', () => {
|
||||
for (const operand of ['../review/SKILL.md', '/etc/passwd', '"$HOME"/x', '-c']) {
|
||||
expect(reads(wcChain, wcChain!.command.replace('wc -l review/SKILL.md', `wc -l ${operand}`)), operand).toEqual(neither);
|
||||
}
|
||||
const counts = ' 856 review/SKILL.md\n 263 review/sections/review-army.md\n 1119 total\n';
|
||||
const command = `cd ${capture.cwd}; wc -l review/SKILL.md review/sections/review-army.md; echo ----; cat src/billing.ts`;
|
||||
expect(reads(wcChain, command, counts + '----\n' + capture.files.source)).toMatchObject({ sourceRead: true });
|
||||
expect(reads(wcChain, command, counts + 'injected\n----\n' + capture.files.source)).toEqual(neither);
|
||||
});
|
||||
});
|
||||
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"provenance": {
|
||||
"source": "CI run 39161502626 attempt 1, eval slice 3, native capture review-coverage-audit (2026-10-18, claude-opus-5-5)",
|
||||
"recordedPassed": false,
|
||||
"recordedFailures": [
|
||||
"missing successful source-file read",
|
||||
"missing successful test-file read"
|
||||
],
|
||||
"schema": "Exact public init cwd/session, the two parent Bash calls that read both owned files (a leading owned cd, then a ; chain ending in wc -l on in-cwd files, and a ; then && chain), their complete successful tool results, and the final diagram text."
|
||||
},
|
||||
"cwd": "/home/runner/.cache/gstack-paid-shard-rFg0zo/tmp/skill-e2e-review-coverage-01WKAl",
|
||||
"sessionId": "ad37bb17-0cba-43cf-ad39-862905e00e20",
|
||||
"calls": [
|
||||
{
|
||||
"toolUseId": "toolu_01WfQLCRZpUHmx6Qy4C6yW8Q",
|
||||
"command": "cd /home/runner/.cache/gstack-paid-shard-rFg0zo/tmp/skill-e2e-review-coverage-01WKAl; cat review/specialists/testing.md; echo ----; cat src/billing.ts; echo ----; cat test/billing.test.ts; echo ----; wc -l review/SKILL.md review/sections/review-army.md",
|
||||
"isError": false,
|
||||
"output": "# Testing Specialist Review Checklist\n\nScope: Always-on (every review)\nOutput: JSON objects, one finding per line. Schema:\n{\"severity\":\"CRITICAL|INFORMATIONAL\",\"confidence\":N,\"path\":\"file\",\"line\":N,\"category\":\"testing\",\"summary\":\"...\",\"fix\":\"...\",\"fingerprint\":\"path:line:testing\",\"specialist\":\"testing\"}\nOptional: line, fix, fingerprint, evidence, test_stub.\nIf no findings: output `NO FINDINGS` and nothing else.\n\nIf the caller explicitly asks for an ASCII coverage diagram, first read the\nnamed source and test files directly in a dedicated tool call. Keep that tool\ncall limited to those two file reads, using either native Read calls or a simple\nshell display such as `cat -n src/file && echo ---- && cat -n test/file`. Read\ndiffs, package files, configs, or other context in separate tool calls. Then\noutput the diagram before any JSON findings. Use one function root per public or\nchanged function, put the `[OK]` or `[GAP]` marker on the same branch row as the\ntested or missing path, and keep the legend inside the same diagram block:\n\n```text\nsrc/billing.ts\nprocessPayment(amount, currency)\n├── valid USD happy path returns success [OK]\n└── invalid amount / unsupported currency branches [GAP]\nrefundPayment(paymentId, reason)\n└── refund success and guard branches not imported or untested [GAP]\nLegend: [OK] tested [GAP] no test\n```\n\nDo not rely on a summary table, prose paragraph, or distant nested marker as the\nonly coverage evidence. Covered happy-path rows must name the successful, valid,\nor concrete tested input path; gap rows must stay under the function that owns\nthe missing path.\n\n---\n\n## Categories\n\n### Exploratory hypotheses\n\nThe parent runs one shared exploratory QA pass before Fix-First, including small diffs\nthat skip specialists. Supply high-risk changed contracts, adverse scenarios and native\ntest candidates to that pass; do not launch another explorer, edit product/tests or\ncommit. A test proposal uses `test_stub` and keeps the caller's approval requirements.\nCheck real request/queue/storage effects, retries, duplicates and interrupted recovery\nwhere applicable. A diagram or test count is not executed proof. For discoveries promoted\nto regressions, require failure for the reproduced bug before repair and green plus\noriginal/adjacent probes afterward; never accept buggy-output goldens or discarded valid\nred tests. Unit tests suit logic; real integration/E2E tests protect boundaries mocks hide.\n\n### Missing Negative-Path Tests\n- New code paths that handle errors, rejections, or invalid input with NO corresponding test\n- Guard clauses and early returns that are untested\n- Error branches in try/catch, rescue, or error boundaries with no failure-path test\n- Permission/auth checks that are asserted in code but never tested for the \"denied\" case\n\n### Missing Edge-Case Coverage\n- Boundary values: zero, negative, max-int, empty string, empty array, nil/null/undefined\n- Single-element collections (off-by-one on loops)\n- Unicode and special characters in user-facing inputs\n- Concurrent access patterns with no race-condition test\n\n### Negative Assertions\n- A new assertion that something is absent (`not.toContain`, `not in`, `assertNotIn`,\n `pytest.raises` on a call that used to succeed) records a behavior removal, not coverage.\n Cite the spec line, plan item or PR text that authorizes the removal; with none, report it\n as a product decision shipped as a test.\n- A rename or removal in this diff can empty a standing negative assertion: a test asserting\n the old token is absent now passes forever and guards nothing. For each identifier, flag,\n heading or string the diff renames or deletes, grep the test suite for negative assertions\n naming the old value, including inside tests that assert something else, and report each\n as a coverage gap whose fix names the new value.\n\n### Test Isolation Violations\n- Tests sharing mutable state (class variables, global singletons, DB records not cleaned up)\n- Order-dependent tests (pass in sequence, fail when randomized)\n- Tests that depend on system clock, timezone, or locale\n- Tests that make real network calls instead of using stubs/mocks\n\n### Flaky Test Patterns\n- Timing-dependent assertions (sleep, setTimeout, waitFor with tight timeouts)\n- Assertions on ordering of unordered results (hash keys, Set iteration, async resolution order)\n- Tests that depend on external services (APIs, databases) without fallback\n- Randomized test data without seed control\n\n### Security Enforcement Tests Missing\n- Auth/authz checks in controllers with no test for the \"unauthorized\" case\n- Rate limiting logic with no test proving it actually blocks\n- Input sanitization with no test for malicious input\n- CSRF/CORS configuration with no integration test\n\n### Coverage Gaps\n- New public methods/functions with zero test coverage\n- ChangeLine truncated
|
||||
},
|
||||
{
|
||||
"toolUseId": "toolu_01QZ27zPhc3JKwRc26DFyBnr",
|
||||
"command": "cd /home/runner/.cache/gstack-paid-shard-rFg0zo/tmp/skill-e2e-review-coverage-01WKAl; cat -n src/billing.ts && echo ---- && cat -n test/billing.test.ts",
|
||||
"isError": false,
|
||||
"output": " 1\t\n 2\texport function processPayment(amount: number, currency: string) {\n 3\t if (amount <= 0) throw new Error('Invalid amount');\n 4\t if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n 5\t return { status: 'success', amount, currency };\n 6\t}\n 7\t\n 8\texport function refundPayment(paymentId: string, reason: string) {\n 9\t if (!paymentId) throw new Error('Payment ID required');\n 10\t if (!reason) throw new Error('Reason required');\n 11\t return { status: 'refunded', paymentId, reason };\n 12\t}\n 13\t\n 14\t// coverage-read-evidence: b69cb3dd-00bd-47ea-975d-064667b48434\n----\n 1\t\n 2\timport { describe, test, expect } from 'vitest';\n 3\timport { processPayment } from '../src/billing';\n 4\t\n 5\tdescribe('processPayment', () => {\n 6\t test('processes valid payment', () => {\n 7\t const result = processPayment(100, 'USD');\n 8\t expect(result.status).toBe('success');\n 9\t });\n 10\t // GAP: no test for invalid amount\n 11\t // GAP: no test for unsupported currency\n 12\t // GAP: refundPayment not tested at all\n 13\t});\n 14\t\n 15\t// coverage-read-evidence: 897be9fc-e2b4-4338-9bff-ba46ab756fbc"
|
||||
}
|
||||
],
|
||||
"finalOutput": "I ran the testing checklist by hand against `src/billing.ts` and `test/billing.test.ts`. I didn't dispatch other specialists, run the test suite or change any files.\n\n```text\nsrc/billing.ts\nprocessPayment(amount, currency) [src/billing.ts:2]\n├── valid USD happy path (100, 'USD') returns status 'success' [OK] test/billing.test.ts:6\n├── valid EUR path ('EUR' accepted) [GAP]\n├── guard: amount <= 0 throws 'Invalid amount' (0, negative) [GAP] src/billing.ts:3\n└── guard: unsupported currency throws 'Unsupported currency' [GAP] src/billing.ts:4\nrefundPayment(paymentId, reason) [src/billing.ts:8]\n├── refund success path returns status 'refunded' [GAP] (not imported in the test file)\n├── guard: empty paymentId throws 'Payment ID required' [GAP] src/billing.ts:9\n└── guard: empty reason throws 'Reason required' [GAP] src/billing.ts:10\nLegend: [OK] tested [GAP] no test\n```\n\n**Coverage summary**\n- **`processPayment`:** 1 of 4 paths tested (25%). Only the valid USD happy path has a test, and it checks only `status`. It never checks that `amount` or `currency` come back unchanged.\n- **`refundPayment`:** 0 of 3 paths tested (0%). The test file never imports it.\n- **Overall:** 1 of 7 paths tested (about 14%). Neither function's error guards have a test, and boundary values (0, negative amounts, empty strings) aren't tested either.\n- This diagram comes from reading the code, not from running the tests.\n\n**Findings**\n```\n{\"severity\":\"CRITICAL\",\"confidence\":10,\"path\":\"src/billing.ts\",\"line\":8,\"category\":\"testing\",\"summary\":\"refundPayment is an exported function with no tests: the success path and both guard clauses are untested, and the test file doesn't import it\",\"fix\":\"Import refundPayment in test/billing.test.ts and test the success return plus the empty paymentId and empty reason errors\",\"fingerprint\":\"src/billing.ts:8:testing\",\"specialist\":\"testing\",\"test_stub\":\"test('refunds payment', () => { expect(re",
|
||||
"files": {
|
||||
"source": "\nexport function processPayment(amount: number, currency: string) {\n if (amount <= 0) throw new Error('Invalid amount');\n if (currency !== 'USD' && currency !== 'EUR') throw new Error('Unsupported currency');\n return { status: 'success', amount, currency };\n}\n\nexport function refundPayment(paymentId: string, reason: string) {\n if (!paymentId) throw new Error('Payment ID required');\n if (!reason) throw new Error('Reason required');\n return { status: 'refunded', paymentId, reason };\n}\n\n// coverage-read-evidence: b69cb3dd-00bd-47ea-975d-064667b48434\n",
|
||||
"tests": "\nimport { describe, test, expect } from 'vitest';\nimport { processPayment } from '../src/billing';\n\ndescribe('processPayment', () => {\n test('processes valid payment', () => {\n const result = processPayment(100, 'USD');\n expect(result.status).toBe('success');\n });\n // GAP: no test for invalid amount\n // GAP: no test for unsupported currency\n // GAP: refundPayment not tested at all\n});\n\n// coverage-read-evidence: 897be9fc-e2b4-4338-9bff-ba46ab756fbc\n"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"source": "https://github.com/garrytan/gstack/actions/runs/39161502626 eval slice 6, plan-ceo-finding-floor (2026-10-18, claude-opus-5-5): observation.json pendingWriteInputs/publicTools and terminal.screen.log",
|
||||
"recorded": {
|
||||
"outcome": "timeout",
|
||||
"summary": "no qualifying finding question within 600000ms",
|
||||
"transcriptCalls": 0,
|
||||
"transcriptAssistantMessages": 0
|
||||
},
|
||||
"sessionId": "22cab15a-8e98-4511-91b1-957ccb8095ec",
|
||||
"pendingId": "22cab15a-8e98-4511-91b1-957ccb8095ec:toolu_01FNW2M9D8ijgaLedRExx5Uf",
|
||||
"basename": "gstack-test-plan-ceo-floor.md",
|
||||
"publicTools": [
|
||||
{
|
||||
"kind": "use",
|
||||
"name": "Bash",
|
||||
"toolUseId": "toolu_017xtQiLa6ZkZHTRpoxZ2V6t"
|
||||
},
|
||||
{
|
||||
"kind": "result",
|
||||
"name": null,
|
||||
"toolUseId": "toolu_017xtQiLa6ZkZHTRpoxZ2V6t"
|
||||
},
|
||||
{
|
||||
"kind": "use",
|
||||
"name": "Read",
|
||||
"toolUseId": "toolu_01FwLoAsY4oGve2DGNKVxew5"
|
||||
},
|
||||
{
|
||||
"kind": "result",
|
||||
"name": null,
|
||||
"toolUseId": "toolu_01FwLoAsY4oGve2DGNKVxew5"
|
||||
},
|
||||
{
|
||||
"kind": "use",
|
||||
"name": "Bash",
|
||||
"toolUseId": "toolu_01U5evFBgNKjK1rJ5awm7HG9"
|
||||
},
|
||||
{
|
||||
"kind": "result",
|
||||
"name": null,
|
||||
"toolUseId": "toolu_01U5evFBgNKjK1rJ5awm7HG9"
|
||||
},
|
||||
{
|
||||
"kind": "use",
|
||||
"name": "Bash",
|
||||
"toolUseId": "toolu_016UM5dndjTc3ovmGkTAZ59h"
|
||||
},
|
||||
{
|
||||
"kind": "result",
|
||||
"name": null,
|
||||
"toolUseId": "toolu_016UM5dndjTc3ovmGkTAZ59h"
|
||||
}
|
||||
],
|
||||
"writeContent": "# CEO Review Working Plan: \"Developer-friendly\" pricing tier\n\nSource: `PLAN.md:11-21` (reviewed plan). Skill: /plan-ceo-review. Branch: main. Date: 2026-10-18.\n\n## Plan under review (verbatim scope, preserved)\n- Goal: Increase developer adoption. (`PLAN.md:13-14`)\n- Success metric: More signups. (`PLAN.md:16-17`)\n- Premise: No developer has been asked whether current pricing is a barrier; team \"feels like\" it should be cheaper. (`PLAN.md:19-21`)\n\n## Settled inputs (from user instructions, `CLAUDE.md` / `PLAN.md:1-7`)\n- /office-hours prerequisite: skipped (user instruction).\n- Routing setup, cross-project recall, outside reviewers: declined (user instruction).\n- Review mode: HOLD SCOPE (user instruction). Product scope preserved.\n- Review depth: strategy-only. The plan has no code, files or interfaces; the user asked for a plan review of a pricing/strategy change. Implementation detail stays as \"implementation owner must prove ___\".\n- Landscape search: not run. Proceeding with in-distribution knowledge only.\n\n## Pre-review system audit\n- Repo contains only `CLAUDE.md` and `PLAN.md`; one commit (`e3ac2de Seed review plan`). No stashes, no TODOS.md, no TODO/FIXME markers, no design doc, no prior decisions, no learnings.\n- No pricing code, billing system, analytics or funnel data exists in this repo to verify any claim. Every factual claim below is therefore unverified.\n\n## Step 0A: Premise challenge\n- **Stated problem:** developer adoption is lower than wanted.\n- **Plan's remedy:** a cheaper tier.\n- **Gap:** the plan admits (`PLAN.md:20-21`) nobody checked that price is the barrier. Developers commonly fail to adopt for other reasons: signup friction, time-to-first-success, docs quality, missing SDKs/integrations, no free trial/sandbox, procurement, or simply not knowing the product exists. A cheaper tier only fixes the price cause.\n- **Proxy risk:** \"More signups\" (`PLAN.md:17`) is a proxy for adoption. A cheap tier can raise signups while activation, retention and revenue fall (cannibalization of existing paid plans).\n- **Do-nothing cost:** unknown. No baseline adoption number is cited.\n- **One-way-door note:** pricing cuts are hard to reverse. Raising a price later means grandfathering, churn and trust cost.\n\n### Premise table (unverified=4)\n| Premise | Claim | Receipt | Verdict |\n|---|---|---|---|\n| P1 | Developer adoption is too low | none (`PLAN.md:14` states goal, no baseline) | unverified |\n| P2 | Current pricing is a barrier for developers | `PLAN.md:20-21` says untested | unverified (plan admits it) |\n| P3 | A cheaper tier will raise adoption | none | unverified |\n| P4 | More signups = more adoption | `PLAN.md:17`, no definition | unverified |\n\n## Step 0B: Existing leverage\nNo code in repo. Reuse candidates are organizational: existing signup funnel analytics, pricing-page analytics, churn/cancel-reason surveys, sales/support tickets mentioning price. Implementation owner must prove which of these exist.\n\n### Reality rows\n- REALITY: premise-table finding 4 premises, unverified=4 PLAN.md:19\n- REALITY: already-done pass no origin remote and single seed commit; nothing shipped PLAN.md:11\n- REALITY: surface-check n/a-equivalent pass plan names no CLI commands, flags or routes PLAN.md:11\n- REALITY: binding-decisions pass decision-search returned none; plan has no Binding decisions section PLAN.md:11\n- REALITY: numbers finding plan cites no numbers; no baseline signups or target [re-measure before gate] PLAN.md:17\n- REALITY: deferred-asks pass no linked issue PLAN.md:11\n- REALITY: pr-count pass no .gstack/ship-policy.json; default one PR per wave, and plan is non-code PLAN.md:11\n\n## Step 0C: Dream state\n```\n CURRENT STATE THIS PLAN 12-MONTH IDEAL\n Adoption \"too low\" (no ---> Cheaper tier launched on ---> Developers self-serve to first\n baseline); cause unknown; gut feel; success = raw success fast; pricing matches\n no developer interviews signup count measured willingness to pay;\n adoption tracked as activation\n + retention, not signups\n```\nThe plan moves toward the ideal only if price is in fact the barrier. If it is not, it spends a one-way pricing move and leaves the real cause untouched.\n\n## Decision ledger\n| ID and owner | Contract and evidence | Current | Proposed | Status | Exact approval and scope |\n|---|---|---|---|---|---|\n| MODE (user) | CLAUDE.md instruction | HOLD SCOPE | n/a | approved | User instruction: \"choose HOLD SCOPE (CEO)\" |\n| PREMISE-1 (plan owner) | `PLAN.md:19-21`: price-barrier premise untested | Launch tier on team intuition | A) launch as written; B) validation gate before launch; C) launch as a time-boxed experiment | unresolved | pending D1 |\n\n## currentDecisLine truncated
|
||||
"screen": " 73 Reversibility of price move | pending | low | low | low, but decided o\n n data | high during test window\n 74 Time to launch | pending | now | now | +~2 weeks \n | now, decision in ~4-6 wks\n 75 Success metric definition | separate row (later) | \"more signups\" | unchanged here | unchanged here \n | unchanged here\n 76 ```\n 77\n 78 Question: D1 — PREMISE-1: Should the tier launch on the untested \"pricing is the barrier\" premise? Project: PLAN\n .md on main proposes a cheaper developer tier. ELI10: The team wants to make a cheaper plan for developers, but \n nobody has asked developers if price is what stops them. If the real blocker is something else (hard signup, bad\n docs), a cheaper plan won't help and price cuts are hard to undo. Stakes: we cut price, lose revenue from peopl\n e who would have paid more, and adoption stays flat. Recommendation: B because a two-week check is cheap compare\n d to a price cut you can't easily take back. Note: options differ in kind, not coverage; no completeness score. \n Net: speed of launch versus confidence that the launch fixes the real problem.\n 79 Header: Premise\n 80 A) Launch as written\n 81 Ship the tier now on team intuition. Effort S (zero extra work), risk high, no verification. ✅ Fastest path; th\n e tier exists within the current timeline and plan text stays unchanged. ✅ Zero added discovery work or researc\n h cost before the team can start building. ❌ If price is not the barrier, revenue drops and adoption stays flat\n , and the cut is hard to reverse.\n 82 B) Validate first (recommended)\n 83 Gate the launch on 10-15 developer interviews plus pricing-page and signup funnel drop-off analysis; launch only\n if price shows up as a top barrier. Effort M (human ~2 weeks / CC ~1 day to draft scripts and queries), risk lo\n w. ✅ Tier stays in scope; it just ships on evidence instead of a feeling. ✅ If the real barrier is elsewhere, \n the team learns it before spending a one-way pricing move. ❌ Delays launch about two weeks and needs someone to\n own the interviews.\n 84 C) Launch as a time-boxed experiment\n 85 Offer the tier to a random cohort of new visitors for 4-6 weeks with a pre-declared success bar, then keep or ki\n ll. Effort L, risk medium. ✅ Measures real behavior, not stated preferences, so it catches what interviews miss\n . ✅ Reversible during the window because only a cohort sees the price. ❌ Needs billing support for cohort pric\n ing and enough traffic for a readable result.\n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n Do you want to create gstack-test-plan-ceo-floor.md?\n ❯ 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel · Tab to amend\n"
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"source": "https://github.com/garrytan/gstack/actions/runs/39161502626/job/117778103794 (Claude Code 2.1.292, claude-opus-5-5, shared-libs-unsupported-git)",
|
||||
"source_sha256": "fb5ea656ef691f674ce6ac9eaf61070abd16f6fce2f51e73d4702d8acba65f8e",
|
||||
"public_request": {
|
||||
"tool": "git",
|
||||
"args": [
|
||||
"-c",
|
||||
"core.askPass=",
|
||||
"-c",
|
||||
"protocol.ext.allow=never",
|
||||
"-c",
|
||||
"submodule.recurse=false",
|
||||
"-c",
|
||||
"log.showSignature=false",
|
||||
"-c",
|
||||
"format.pretty=medium",
|
||||
"-c",
|
||||
"gc.auto=0",
|
||||
"-c",
|
||||
"maintenance.auto=false",
|
||||
"-c",
|
||||
"hook.applypatch-msg.enabled=false",
|
||||
"-c",
|
||||
"hook.pre-applypatch.enabled=false",
|
||||
"-c",
|
||||
"hook.post-applypatch.enabled=false",
|
||||
"-c",
|
||||
"hook.pre-commit.enabled=false",
|
||||
"-c",
|
||||
"hook.pre-merge-commit.enabled=false",
|
||||
"-c",
|
||||
"hook.prepare-commit-msg.enabled=false",
|
||||
"-c",
|
||||
"hook.commit-msg.enabled=false",
|
||||
"-c",
|
||||
"hook.post-commit.enabled=false",
|
||||
"-c",
|
||||
"hook.pre-rebase.enabled=false",
|
||||
"-c",
|
||||
"hook.post-checkout.enabled=false",
|
||||
"-c",
|
||||
"hook.post-merge.enabled=false",
|
||||
"-c",
|
||||
"hook.pre-push.enabled=false",
|
||||
"-c",
|
||||
"hook.pre-receive.enabled=false",
|
||||
"-c",
|
||||
"hook.update.enabled=false",
|
||||
"-c",
|
||||
"hook.proc-receive.enabled=false",
|
||||
"-c",
|
||||
"hook.post-receive.enabled=false",
|
||||
"-c",
|
||||
"hook.post-update.enabled=false",
|
||||
"-c",
|
||||
"hook.reference-transaction.enabled=false",
|
||||
"-c",
|
||||
"hook.push-to-checkout.enabled=false",
|
||||
"-c",
|
||||
"hook.pre-auto-gc.enabled=false",
|
||||
"-c",
|
||||
"hook.post-rewrite.enabled=false",
|
||||
"-c",
|
||||
"hook.sendemail-validate.enabled=false",
|
||||
"-c",
|
||||
"hook.fsmonitor-watchman.enabled=false",
|
||||
"-c",
|
||||
"hook.p4-changelist.enabled=false",
|
||||
"-c",
|
||||
"hook.p4-prepare-changelist.enabled=false",
|
||||
"-c",
|
||||
"hook.p4-post-changelist.enabled=false",
|
||||
"-c",
|
||||
"hook.p4-pre-submit.enabled=false",
|
||||
"-c",
|
||||
"hook.post-index-change.enabled=false",
|
||||
"-c",
|
||||
"core.hooksPath=/dev/null",
|
||||
"-c",
|
||||
"core.fsmonitor=",
|
||||
"--version"
|
||||
],
|
||||
"env": {
|
||||
"GIT_NO_LAZY_FETCH": "1",
|
||||
"GIT_TERMINAL_PROMPT": "0"
|
||||
},
|
||||
"cwd": "/home/runner/.cache/gstack-paid-shard-3oSFtA/tmp/gstack-shared-unsupported-2cfAqa/repo",
|
||||
"pid": 3872,
|
||||
"ppid": 3779,
|
||||
"parentExecutable": "/usr/local/lib/node_modules/@anthropic-ai/claude-code/bin/claude.exe",
|
||||
"parentCommand": "claude -p --model claude-opus-5-5 --output-format stream-json --verbose --dangerously-skip-permissions --max-turns 24 --allowed-tools Bash Read Write Edit Glob Grep --tools Bash,Read,Write,Edit,Glob,Grep --include-partial-messages --strict-mcp-config "
|
||||
},
|
||||
"matching_model_commands": []
|
||||
}
|
||||
+7
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"source": "https://github.com/garrytan/gstack/actions/runs/39161502626 eval slice 14, ship-local-hook-preservation (2026-10-18, claude-opus-5-5)",
|
||||
"recordedFailures": [
|
||||
"manual integration was not reported"
|
||||
],
|
||||
"assistant": "Unmanaged hook present. Inspecting per the rules.\nI ran the guard, and it did **not** install or change any hook. Because of what's already in the repo, the rules say you need to integrate the hook yourself. I stopped before pushing.\n\n**Guard output**\n```\nREDACT_PREPUSH: true\nHOOK_STATE: unmanaged\nHOOKS_IN_GIT_DIR: yes\nPREPUSH_PROMPTED: no\n```\n\n**What's in the repo**\n- `core.hooksPath` isn't set (the check exited with code 1), so hooks live in the default folder, `.git/hooks`.\n- `.git/hooks/pre-push` is a hook gstack doesn't manage. Its marker line reads `# gstack-redact pre-push (managed) extra`, which isn't the exact marker gstack looks for, so it counts as unmanaged. The script just runs `exit 42`.\n- `.git/hooks/pre-push.local` also exists. It saves everything passed to it into `$HOME/received`, then runs `exit 37`.\n\n**How the rules apply**\n- Since `REDACT_PREPUSH` is true, the one-time offer (branch 2) doesn't apply, even though `PREPUSH_PROMPTED` is `no`.\n- The installer didn't run because the existing hook is unmanaged.\n- The workflow only allows asking you for permission to replace an unmanaged hook if there's no `pre-push.local`. There is one, so I didn't ask. Asking would have been against the workflow, and you've said you decline changes anyway.\n- I didn't overwrite either hook.\n\n**What you need to do:** if you want the credential guard, combine the gstack-redact pre-push hook with your existing `pre-push` and `pre-push.local` by hand.\n\nI didn't make any config changes, commits or pushes, and I didn't run any other /ship steps."
|
||||
}
|
||||
@@ -138,6 +138,12 @@ function stagePieces(stage: string[], scope: ShellScope, alone: boolean): Piece[
|
||||
}
|
||||
if (name === 'grep' || name === 'ls') return [unknown()];
|
||||
if (text === 'pwd' || text === 'wc -l') return [unknown(1, 1)];
|
||||
if (name === 'wc' && args[0] === '-l' && args.length > 1) {
|
||||
// One count line per literal in-cwd file, plus a total for several.
|
||||
if (args.slice(1).some(operand => !scopedFile(operand, scope))) return undefined;
|
||||
const lines = args.length - 1 + (args.length > 2 ? 1 : 0);
|
||||
return [unknown(lines, lines)];
|
||||
}
|
||||
if (/^\[ -f [A-Za-z0-9_.\/-]+ \]$/.test(text)) return [unknown(0, 0)];
|
||||
if (displayAwk) return [unknown()];
|
||||
return undefined;
|
||||
@@ -154,12 +160,15 @@ function commandPieces(command: unknown, scope: ShellScope): Piece[] | undefined
|
||||
else lists.at(-1)!.at(-1)!.push(word);
|
||||
}
|
||||
if (lists.some(pipeline => pipeline.some(stage => !stage.length))) return undefined;
|
||||
// One leading assertion of the recorded cwd. Mixed separators after it could
|
||||
// run later relative reads elsewhere, so they stay conservatively unsupported.
|
||||
// One leading assertion of the recorded cwd, followed by one uniform list.
|
||||
// `cd CWD;` runs the list whether or not cd succeeds, exactly like an all-;
|
||||
// list, so only it may precede an && chain. Other mixes (`cd CWD && A; B`)
|
||||
// could run later relative reads elsewhere and stay unsupported.
|
||||
if (lists[0]!.length === 1 && lists[0]![0]![0] === 'cd') {
|
||||
const [, target, ...rest] = lists[0]![0]!;
|
||||
if (rest.length || target === undefined || literal(target) !== scope.cwd || lists.length < 2 ||
|
||||
new Set(separators.filter(separator => separator !== '||')).size !== 1 || separators[0] === '||') return undefined;
|
||||
const [assertion, ...chain] = separators, kinds = new Set(chain.filter(separator => separator !== '||'));
|
||||
if (rest.length || target === undefined || literal(target) !== scope.cwd || lists.length < 2 || assertion === '||' ||
|
||||
kinds.size > 1 || (assertion !== ';' && kinds.size && !kinds.has(assertion!))) return undefined;
|
||||
lists.shift(); separators.shift();
|
||||
}
|
||||
const pieces: Piece[] = [];
|
||||
|
||||
@@ -59,6 +59,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/codex-sandbox/**': ['test/codex-eval-recording.test.ts', 'test/codex-model-probe.test.ts', 'test/outside-review-result.test.ts', 'test/outside-voice-invocation.test.ts'],
|
||||
'test/fixtures/context-bill/**': ['test/context-bill.test.ts', 'test/context-budget-ratchet.test.ts'],
|
||||
'test/fixtures/context-budget.json': ['test/helpers/capture-context-budget.ts', 'test/test-value-bar.test.ts'],
|
||||
'test/fixtures/coverage-audit-cd-chain.json': ['test/coverage-audit-evidence.test.ts'],
|
||||
'test/fixtures/coverage-audit-parallel-column.json': ['test/coverage-audit-evidence.test.ts'],
|
||||
'test/fixtures/coverage-audit-summary.json': ['test/coverage-audit-evidence.test.ts'],
|
||||
'test/fixtures/cso-eval/**': ['scripts/cso-eval.ts', 'test/cso-eval.test.ts', 'test/cso-stack-cold-integration.test.ts', 'test/helpers/cso-eval-oracles.ts'],
|
||||
@@ -144,6 +145,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/parity-baseline-v1.44.1.json': ['test/helpers/capture-parity-baseline.test.ts', 'test/parity-baseline-integrity.test.ts'],
|
||||
'test/fixtures/parity-baseline-v1.64.1.0.json': ['test/parity-suite.test.ts', 'test/qa-lazy-sections.test.ts', 'test/skill-size-budget.test.ts'],
|
||||
'test/fixtures/parity-baseline-v1.81.0.0.json': ['test/skill-size-budget.test.ts'],
|
||||
'test/fixtures/plan-ceo-floor-toolonly-create.json': ['test/plan-create-toolonly-session.test.ts', 'test/plan-floor-permission.test.ts'],
|
||||
'test/fixtures/plan-count-design-questionless-report.md': ['test/plan-count-empty-review.test.ts'],
|
||||
'test/fixtures/plan-create-cropped-title-batching.json': ['test/plan-create-combined-permission.test.ts'],
|
||||
'test/fixtures/plan-ceo-floor-unasked-decisions-872e154.json': ['test/plan-ceo-floor-interactive-boundary.test.ts'],
|
||||
@@ -163,6 +165,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/session-update-heartbeat.sh': ['test/session-update-heartbeat.test.ts'],
|
||||
'test/fixtures/shard-cli-parity/**': ['test/shard-cli-parity.test.ts'],
|
||||
'test/fixtures/shard-equivalence/**': ['test/shard-engine-equivalence.test.ts'],
|
||||
'test/fixtures/shared-libs-claude-version-probe.json': ['test/shared-libs-fixture.test.ts'],
|
||||
'test/fixtures/shared-libs-no-change-ci-public.json': ['test/shared-libs-fixture.test.ts'],
|
||||
'test/fixtures/shared-libs-plan-callers-fixes-behavior-local.json': ['test/shared-libs-plan-actor.test.ts'],
|
||||
'test/fixtures/shared-libs-plan-callers-future-risk-37195203538.json': ['test/shared-libs-plan-actor.test.ts'],
|
||||
@@ -173,6 +176,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/shared-libs-plan-callers-slash-scope-37174266054.json': ['test/shared-libs-plan-actor.test.ts'],
|
||||
'test/fixtures/shared-libs-plan-callers-verbless-scope-37186854666.json': ['test/shared-libs-plan-actor.test.ts'],
|
||||
'test/fixtures/shared-libs/**': ['test/shared-libs-checker-interface-evidence.test.ts', 'test/shared-libs-fixture.test.ts', 'test/shared-libs-plan-actor.test.ts', 'test/shared-libs-revalidation-prompt.test.ts', 'test/shared-libs-review-start-evidence.test.ts', 'test/shared-libs-source-reads.test.ts', 'test/shared-libs-stage-actor.test.ts'],
|
||||
'test/fixtures/ship-local-hook-reply.json': ['test/ship-hook-actor.test.ts'],
|
||||
'test/fixtures/ship-queue/gbrain-stamp-pair.json': ['test/helpers/ship-queue-fixture.ts', 'test/tree-receipt-gbrain-pair.test.ts'],
|
||||
'test/fixtures/ship-triage/**': ['test/ship-triage-labels.test.ts'],
|
||||
'test/fixtures/skill-md-size-allowlist.json': ['test/skill-md-size-guard.test.ts'],
|
||||
|
||||
@@ -410,6 +410,8 @@ export function currentFilePermissionEpoch(file: string | undefined, expected: s
|
||||
if (!stat.isFile() || stat.size > MAX_RECORD_BYTES || transcript.status !== 'ready') return null;
|
||||
const r = JSON.parse(fs.readFileSync(file,'utf8'));
|
||||
const sessions = new Set([...transcript.calls.map(c=>c.sessionId),...transcript.assistantMessages.map(m=>m.sessionId)]);
|
||||
// A parent that has only used tools so far (no text, no question) is identified by its public tool events.
|
||||
if (!sessions.size) readPlanCountTranscript(config, cwd, event => sessions.add(event.sessionId));
|
||||
const time = Date.parse(r.timestamp);
|
||||
const validId = (id: unknown) => typeof id === 'string' && id.startsWith(r.sessionId+':') && identifier(id.slice(r.sessionId.length+1));
|
||||
if (r.cwd !== cwd || r.expected !== expected || !identifier(r.sessionId) || sessions.size !== 1 || !sessions.has(r.sessionId) ||
|
||||
|
||||
@@ -41,7 +41,8 @@ export interface PlanSkillFloorObservation {
|
||||
| 'silent_write'
|
||||
| 'exited'
|
||||
| 'timeout'
|
||||
| 'assessment_error';
|
||||
| 'assessment_error'
|
||||
| 'permission_unanswered';
|
||||
summary: string;
|
||||
/** Public current viewport for an accepted finding; terminal tail otherwise. */
|
||||
evidence: string;
|
||||
@@ -190,6 +191,7 @@ export async function runPlanSkillFloorCheck(opts: PlanSkillFloorOptions): Promi
|
||||
since: 0, commandStartedAt: 0, targetDelivery: undefined, saveSnapshot: undefined, nativeCandidates: [],
|
||||
validatedPendingQuestion: undefined, sampledAt: undefined, lastCheckpointAt: 0, lastCheckpointState: '',
|
||||
artifactError: undefined, finished: false, started: false, start: 0, deadlineAt: 0, screenDeadlineAt: 0,
|
||||
permissionUnansweredSince: undefined,
|
||||
} as unknown as FloorRun;
|
||||
return runPtySession<PlanSkillFloorObservation>({
|
||||
driver, budgetMs: timeoutMs,
|
||||
@@ -266,6 +268,8 @@ interface FloorRun {
|
||||
start: number;
|
||||
deadlineAt: number;
|
||||
screenDeadlineAt: number;
|
||||
/** When the current permission render was first left unanswered by this actor. */
|
||||
permissionUnansweredSince: number | undefined;
|
||||
}
|
||||
|
||||
async function floorStart(run: FloorRun, session: ClaudePtySession): Promise<undefined> {
|
||||
@@ -388,19 +392,27 @@ async function floorTick(run: FloorRun, session: ClaudePtySession): Promise<PtyS
|
||||
? planFloorDXPane(run.viewport, call) !== null : matchesNativePlanQuestion(run.viewport, call, run.planningDirectory));
|
||||
run.pendingQuestion = matching.length === 1 ? matching[0] : undefined;
|
||||
floorCheckpoint(run, session);
|
||||
if (floorPermission(run, session)) return 'continue';
|
||||
if (floorPermission(run, session)) {
|
||||
const waited = run.permissionUnansweredSince === undefined ? 0 : driver.now() - run.permissionUnansweredSince;
|
||||
return waited < PERMISSION_UNANSWERED_MS ? 'continue' : finish({ auqObserved: false, outcome: 'permission_unanswered',
|
||||
summary: `PERMISSION_DIALOG_UNANSWERED: a Claude Code permission dialog stayed open for ${waited}ms without an owned grant`,
|
||||
evidence: run.viewport.slice(-3000), elapsedMs: driver.now() - run.startedAt });
|
||||
}
|
||||
const question = floorQuestion(run, session, currentCalls);
|
||||
if (question) return question;
|
||||
return floorAssess(run, session, visible);
|
||||
}
|
||||
|
||||
/** This actor answers only owned file grants; any other permission render fails fast with a named cause. */
|
||||
const PERMISSION_UNANSWERED_MS = 20_000;
|
||||
|
||||
/** An authorized owned file write enables the review, but never supplies its
|
||||
* finding question. True when an active permission render owns this tick. */
|
||||
function floorPermission(run: FloorRun, session: ClaudePtySession): boolean {
|
||||
const { fixture, viewport } = run;
|
||||
const permissionIsActiveRender = !run.pendingQuestion &&
|
||||
(isPermissionDialogVisible(viewport) || isCroppedEditPermissionVisible(viewport));
|
||||
if (!permissionIsActiveRender) return false;
|
||||
if (!permissionIsActiveRender) { run.permissionUnansweredSince = undefined; return false; }
|
||||
// Exclude the permission's old render after approval.
|
||||
const owned = currentFilePermissionBinding(run.ownedFilePermissions, fixture.cwd,
|
||||
session.hermeticConfigDir, run.startedAt, run.transcript, viewport);
|
||||
@@ -412,8 +424,10 @@ function floorPermission(run: FloorRun, session: ClaudePtySession): boolean {
|
||||
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
|
||||
ordinaryOwnedTarget = parent === fs.realpathSync(fixture.cwd) && (!target || target.isFile());
|
||||
} catch { /* No authority for a linked, foreign, or unreadable target. */ }
|
||||
if (owned && ordinaryOwnedTarget && owned.binding.guard(viewport, session.visibleText(), owned.epoch) === 'grant')
|
||||
session.send('1\r');
|
||||
const decision = owned && ordinaryOwnedTarget ? owned.binding.guard(viewport, session.visibleText(), owned.epoch) : null;
|
||||
if (decision === 'grant') session.send('1\r');
|
||||
// A grant or its lingering render is answered; anything else waits on no one.
|
||||
run.permissionUnansweredSince = decision ? undefined : run.permissionUnansweredSince ?? run.driver.now();
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ export function qaDeadlineShellPolicy(directory: string, guard: string, browse:
|
||||
${allowed.slice(1).join('\n')}
|
||||
Use Read/Glob for local inspection and Write only inside ${path.join(directory, 'qa-reports')}; the report and screenshots directories already exist. Memory files and learning stores outside that directory are not authorized. Write the initial charters and final report to the same caller-owned file ${path.join(directory, 'qa-reports/qa-only-report.md')}, not a separate charter file.
|
||||
Every other Bash call must be exactly one invocation of bun ${quote(guard)}, with literal arguments, no outer assignments, substitutions, globs, pipes, redirects, prefixes, suffixes or shell operators. The exact runtime path ${quote(process.execPath)} may replace bun; no other launcher is accepted.
|
||||
Start exactly once with: bun ${quote(guard)} start ${quote(file)} 30
|
||||
Start exactly once, and only after you have read the workflow sections you need and your Write of the initial charter to ${path.join(directory, 'qa-reports/qa-only-report.md')} has completed; the 30-second deadline begins at start: bun ${quote(guard)} start ${quote(file)} 30
|
||||
Then use bun ${quote(guard)} status ${quote(file)} or bun ${quote(guard)} run ${quote(file)} -- COMMAND ARGS. Use the absolute guard path directly, not $G.
|
||||
For browser scripts, put all assignments, pipelines and scripts INSIDE the child: bun ${quote(guard)} run ${quote(file)} -- bash -c 'B="${browse}"; "$B" goto URL; "$B" snapshot -i'. Literal argv and single/double quoted literal arguments are accepted; expansions are only allowed inside the single-quoted child script.
|
||||
Do not write, reset, replace, chmod or remove deadline.json, invoke the guard recursively, or print QA_DEADLINE receipts yourself. Browser cleanup also goes inside the guard; after expiry only local report bookkeeping is allowed. Fallback screenshots must be saved directly in the owned screenshots directory; retain them and write the report after expiry without new browser calls. This fixture does not accept a separate post-expiry shell copy from Aside's session directory; mark that artifact unavailable rather than launching new browser work.
|
||||
|
||||
@@ -506,11 +506,16 @@ export function isInternalClaudeGitRequest(request: SourceRequest, commands: str
|
||||
const [gc, gcAuto, maint, maintAuto, ...afterMaintenance] = tail;
|
||||
const hostPrefixes = [[...head, ...tail],
|
||||
[...head, '-c', 'format.pretty=medium', gc!, gcAuto!, maint!, maintAuto!, ...hooks292, ...afterMaintenance]];
|
||||
// 2.1.292's version probe: core.askPass= first, no pathspec/quoting configs,
|
||||
// and only --version after the hook list. It qualifies as this exact argv.
|
||||
const versionProbe = ['-c', 'core.askPass=', ...head, '-c', 'format.pretty=medium', gc!, gcAuto!, maint!, maintAuto!, ...hooks292,
|
||||
'-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=', '--version'];
|
||||
// Require direct process ancestry AND the exact observed host prefix AND no
|
||||
// matching model request. A shell/model-issued unguarded Git call still fails.
|
||||
return request.tool === 'git' && !!request.ppid &&
|
||||
/(?:^|[/\\])claude(?:\.exe)?$/.test(request.parentExecutable || '') &&
|
||||
hostPrefixes.some(prefix => JSON.stringify(request.args.slice(0, prefix.length)) === JSON.stringify(prefix)) &&
|
||||
(hostPrefixes.some(prefix => JSON.stringify(request.args.slice(0, prefix.length)) === JSON.stringify(prefix)) ||
|
||||
JSON.stringify(request.args) === JSON.stringify(versionProbe)) &&
|
||||
!commands.some(command => command.includes('core.safecrlf=false') || command.includes('protocol.ext.allow=never'));
|
||||
}
|
||||
|
||||
|
||||
@@ -130,7 +130,7 @@ export function shipHookFailures(fixture: ReturnType<typeof createShipHookFixtur
|
||||
} else {
|
||||
check(evidence.localHook === localPolicy, 'existing local policy changed');
|
||||
check(evidence.interactions.length === 0, 'existing local policy requires manual integration, not consent to overwrite');
|
||||
check(/manual/i.test(result.output), 'manual integration was not reported');
|
||||
check(/manual|\bintegrat\w*\b[^.\n]{0,40}\byourself\b|\bcombine\b[^\n]{0,160}\bby hand\b/i.test(result.output), 'manual integration was not reported');
|
||||
}
|
||||
}
|
||||
return { failures, evidence: { ...evidence, callback } };
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import {expect,test} from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {createFilePermissionRecorder,recordFilePermission,currentFilePermissionEpoch} from './helpers/plan-count-file-permission';
|
||||
import {readPlanCountTranscript} from './helpers/plan-count-transcript';
|
||||
import {createPlanCountPermissionGuard} from './helpers/claude-pty-runner';
|
||||
import captured from './fixtures/plan-ceo-floor-toolonly-create.json';
|
||||
|
||||
// CI 39161502626 slice 6: the parent ran four tools, then its first Write asked to
|
||||
// create the owned working plan before any assistant text or question existed.
|
||||
function fixture(){
|
||||
const root=fs.mkdtempSync(path.join(os.tmpdir(),'gstack-toolonly-'));const cwd=path.join(root,'owned'),config=path.join(root,'config');fs.mkdirSync(cwd);
|
||||
const expected=path.join(cwd,captured.basename),sid=captured.sessionId,id=captured.pendingId.slice(sid.length+1);
|
||||
const journal=path.join(config,'projects','owned',sid+'.jsonl');fs.mkdirSync(path.dirname(journal),{recursive:true});
|
||||
const startedAt=Date.now()-1000;const recorder=createFilePermissionRecorder(cwd,config,expected)!;
|
||||
const native=(role:string,content:any[],session=sid)=>({cwd,sessionId:session,isSidechain:false,timestamp:new Date().toISOString(),message:{role,content}});
|
||||
const tools=(session=sid)=>captured.publicTools.map(event=>event.kind==='use'
|
||||
? native('assistant',[{type:'tool_use',id:event.toolUseId,name:event.name,input:event.name==='Read'?{file_path:path.join(cwd,'PLAN.md')}:{command:'true'}}],session)
|
||||
: native('user',[{type:'tool_result',tool_use_id:event.toolUseId,content:'ok'}],session));
|
||||
const write=(rows:any[],file=journal)=>{fs.mkdirSync(path.dirname(file),{recursive:true});fs.writeFileSync(file,rows.map(r=>JSON.stringify(r)).join('\n')+'\n');};
|
||||
write(tools());
|
||||
recordFilePermission(JSON.stringify({hook_event_name:'PreToolUse',tool_name:'Write',session_id:sid,tool_use_id:id,cwd,transcript_path:journal,
|
||||
tool_input:{file_path:expected,content:captured.writeContent}}),recorder.file,cwd,config,expected);
|
||||
const read=(screen=captured.screen)=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,readPlanCountTranscript(config,cwd),screen);
|
||||
return {root,cwd,config,sid,id,journal,native,tools,write,read,close(){recorder.dispose();fs.rmSync(root,{recursive:true,force:true});}};
|
||||
}
|
||||
|
||||
test('a tool-only parent session authenticates its captured owned Create once',()=>{
|
||||
const f=fixture();try{
|
||||
expect(captured.recorded).toMatchObject({outcome:'timeout',transcriptCalls:0,transcriptAssistantMessages:0});
|
||||
const transcript=readPlanCountTranscript(f.config,f.cwd);
|
||||
expect(transcript).toMatchObject({status:'ready',calls:[],assistantMessages:[]});
|
||||
expect(f.read()?.pendingId).toBe(captured.pendingId);
|
||||
const guard=createPlanCountPermissionGuard();
|
||||
expect(guard(captured.screen,'',f.read())).toBe('grant');
|
||||
expect(guard(captured.screen,'',f.read())).toBe('handled');
|
||||
}finally{f.close();}
|
||||
});
|
||||
|
||||
test('tool-only identity still requires exactly one parent session that owns the pending Write',()=>{
|
||||
const f=fixture();try{
|
||||
const foreign='11111111-2222-4333-8444-555555555555',foreignJournal=path.join(f.config,'projects','owned',foreign+'.jsonl');
|
||||
f.write(f.tools(foreign),foreignJournal);
|
||||
expect(f.read()).toBeNull();
|
||||
f.write([]);
|
||||
expect(f.read()).toBeNull();
|
||||
fs.rmSync(foreignJournal);
|
||||
expect(f.read()).toBeNull();
|
||||
f.write([...f.tools(),f.native('assistant',[{type:'tool_use',id:'other-write',name:'Write',input:{file_path:path.join(f.cwd,'other.md'),content:'x'}}])]);
|
||||
expect(f.read()).toBeNull();
|
||||
}finally{f.close();}
|
||||
});
|
||||
|
||||
test('a tool-only session gets no credit for a changed preview or a foreign create prompt',()=>{
|
||||
const f=fixture();try{
|
||||
expect(f.read(captured.screen.replace('Validate first (recommended)','Validate later (recommended)'))).toBeNull();
|
||||
expect(f.read(captured.screen.replaceAll(captured.basename,'foreign-plan.md'))).toBeUndefined();
|
||||
}finally{f.close();}
|
||||
});
|
||||
@@ -17,6 +17,7 @@ import {readPlanFloorTarget} from './helpers/plan-floor-target';
|
||||
import {createFilePermissionRecorder, recordFilePermission, currentFilePermissionBinding, readPendingWriteInput, isCroppedEditPermissionVisible} from './helpers/plan-count-file-permission';
|
||||
import captured from './fixtures/plan-floor-permission-fb10.json';
|
||||
import croppedEdit from './fixtures/plan-edit-cropped-permission-1579.json';
|
||||
import toolOnlyCreate from './fixtures/plan-ceo-floor-toolonly-create.json';
|
||||
import {FORCING_FLOOR_CEO, FORCING_FLOOR_ENG, FORCING_FLOOR_DESIGN, FORCING_FLOOR_DEVEX} from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
@@ -41,7 +42,7 @@ const render = (question: typeof QUESTIONS.ceo) => ['☐ '+question.header,quest
|
||||
...question.options.flatMap((o,i)=>[`${i?' ':'❯'} ${i+1}. ${o.label}`,o.description]),
|
||||
'Enter to select · ↑/↓ to navigate · Esc to cancel'].join('\n');
|
||||
const FINDING = render(QUESTIONS.ceo);
|
||||
type Mode = 'planning-owned' | 'planning-foreign' | 'cropped-edit' | 'cropped-edit-missing' | 'cropped-edit-changed' | 'cropped-edit-completed' | 'captured' | 'owned' | 'owned-no-question' | 'foreign' | 'wrong-session' | 'missing-native' | 'linked-target' |
|
||||
type Mode = 'captured-unbound-create' | 'planning-owned' | 'planning-foreign' | 'cropped-edit' | 'cropped-edit-missing' | 'cropped-edit-changed' | 'cropped-edit-completed' | 'captured' | 'owned' | 'owned-no-question' | 'foreign' | 'wrong-session' | 'missing-native' | 'linked-target' |
|
||||
'native-question' | 'scope' | 'prose' | 'finding' | 'routing' | 'unrelated' | 'partial' | 'quoted' | 'foreign-question' |
|
||||
'stale-question' | 'answered-question' | 'failed-question' | 'mismatched-use' | 'duplicate-use' | 'judge-error' | 'mode' | 'pending-hook' | 'failed-hook' | 'packet' | 'prose-quoted' | 'prose-partial' | 'prose-foreign' | 'prose-stale' | 'product-type' | 'product-type-undeclared' |
|
||||
'unmatched-hook' | 'invalid-hook' | 'missing-hook' | 'idle-hook' | 'transition-hook' | 'unmatched-native' | 'dx-setup' | 'dx-editor-hint' | 'dx-no-finding' | 'dx-undeclared' | 'dx-cropped' | 'dx-unrelated' | 'dx-uncertain' | 'dx-changing-call';
|
||||
@@ -131,7 +132,7 @@ async function exercise(mode: Mode, kind: keyof typeof SEEDS = 'ceo', capture?:
|
||||
const pendingFilePermissionFiles = expected ? [(() => {
|
||||
const recorder = createFilePermissionRecorder(opts.cwd, config, expected)!;
|
||||
recorders.push(recorder);
|
||||
if (mode !== 'missing-native' && mode !== 'cropped-edit-missing') recordFilePermission(JSON.stringify({
|
||||
if (mode !== 'missing-native' && mode !== 'cropped-edit-missing' && mode !== 'captured-unbound-create') recordFilePermission(JSON.stringify({
|
||||
hook_event_name:'PreToolUse', tool_name:crop?'Edit':'Write', session_id:mode === 'wrong-session' ? 'foreign' : sid,
|
||||
tool_use_id:'write1', cwd:opts.cwd,
|
||||
transcript_path:mode === 'wrong-session' ? path.join(config,'projects','owned','foreign.jsonl') : journal,
|
||||
@@ -144,6 +145,7 @@ async function exercise(mode: Mode, kind: keyof typeof SEEDS = 'ceo', capture?:
|
||||
const permissionPath = mode === 'foreign' ? path.join(dir, 'foreign', path.basename(expected ?? 'report.md')) : expected;
|
||||
const permission = crop ? croppedEdit.screen.replaceAll(path.basename(croppedEdit.hook.expected),path.basename(expected))
|
||||
.replace('empathy narrative',mode==='cropped-edit-changed'?'different narrative':'empathy narrative') : mode === 'captured' ? captured.rawPermission :
|
||||
mode === 'captured-unbound-create' ? toolOnlyCreate.screen.replaceAll(toolOnlyCreate.basename, path.basename(expected)) :
|
||||
`Create file\n${permissionPath}\n────────────────\n 1 # Working review\n────────────────\nDo you want to create ${path.basename(permissionPath ?? 'report.md')}?\n❯ 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session\n 3. No\nEsc to cancel · Tab to amend`;
|
||||
return {
|
||||
hermeticConfigDir: config, pendingFilePermissionFiles, pendingQuestionFile:hookRecorder?.file,
|
||||
@@ -278,7 +280,19 @@ test('captured file permission never satisfies the floor through a waiting judge
|
||||
expect(captured.originalOutcome).toBe('auq_observed');
|
||||
expect(runner.isPermissionDialogVisible(captured.rawPermission)).toBe(true);
|
||||
const e = await exercise('captured');
|
||||
expect(e.result.outcome).toBe('timeout'); expect(e.result.auqObserved).toBe(false);
|
||||
expect(e.result.outcome).toBe('permission_unanswered'); expect(e.result.auqObserved).toBe(false);
|
||||
// gstack-timing-ok: injected exercise clock; failing within the 20 s bound instead of the timeout is the contract
|
||||
expect(e.result.summary).toStartWith('PERMISSION_DIALOG_UNANSWERED: '); expect(e.result.elapsedMs).toBeLessThan(50_000);
|
||||
expect(e.sent).toEqual(['/plan-ceo-review PLAN.md\r']); expect(e.judgments).toHaveLength(0);
|
||||
});
|
||||
test('the captured CI 39161502626 Create left without an owned grant fails fast with a named cause', async () => {
|
||||
expect(toolOnlyCreate.recorded.outcome).toBe('timeout');
|
||||
expect(runner.isPermissionDialogVisible(toolOnlyCreate.screen)).toBe(true);
|
||||
const e = await exercise('captured-unbound-create');
|
||||
expect(e.result).toMatchObject({outcome:'permission_unanswered', auqObserved:false});
|
||||
expect(e.result.summary).toStartWith('PERMISSION_DIALOG_UNANSWERED: ');
|
||||
// gstack-timing-ok: injected exercise clock; failing within the 20 s bound instead of the timeout is the contract
|
||||
expect(e.result.elapsedMs).toBeLessThan(50_000);
|
||||
expect(e.sent).toEqual(['/plan-ceo-review PLAN.md\r']); expect(e.judgments).toHaveLength(0);
|
||||
});
|
||||
test('one owned native Write grant enables the actual later question without answering it', async () => {
|
||||
@@ -298,12 +312,12 @@ for (const mode of ['planning-owned','planning-foreign'] as const) test(`actual
|
||||
test.each(['cropped-edit','cropped-edit-missing','cropped-edit-changed','cropped-edit-completed'] as Mode[])('%s routes through the actual floor only after exact ownership',async mode=>{
|
||||
const e=await exercise(mode,'devex');
|
||||
expect(e.sent).toEqual(mode==='cropped-edit'?['/plan-devex-review PLAN.md\r','1\r']:['/plan-devex-review PLAN.md\r']);
|
||||
expect(e.result.outcome).toBe(mode==='cropped-edit'?'auq_observed':'timeout');
|
||||
expect(e.result.outcome).toBe(mode==='cropped-edit'?'auq_observed':'permission_unanswered');
|
||||
expect(e.judgments).toHaveLength(mode==='cropped-edit'?1:0);
|
||||
if(mode==='cropped-edit')expect(e.saved.observation.pendingQuestion.answered).toBe(false);
|
||||
});
|
||||
test.each(['owned-no-question','foreign','wrong-session','missing-native','linked-target'] as Mode[])('%s cannot supply finding credit', async mode => {
|
||||
const e = await exercise(mode); expect(e.result.outcome).toBe('timeout');
|
||||
const e = await exercise(mode); expect(e.result.outcome).toBe(mode === 'owned-no-question' ? 'timeout' : 'permission_unanswered');
|
||||
expect(e.sent).toEqual(mode === 'owned-no-question' ? ['/plan-ceo-review PLAN.md\r','1\r'] : ['/plan-ceo-review PLAN.md\r']);
|
||||
expect(e.judgments).toHaveLength(0);
|
||||
});
|
||||
|
||||
@@ -98,6 +98,15 @@ test('R70 retained public ordering has no completed report Write before its cloc
|
||||
expect(() => assertQaBrowserPreparation(events, { directory: f.directory, guard })).toThrow('report Write must complete before guard start/baseline');
|
||||
});
|
||||
|
||||
test('the deadline prompt orders the charter Write before the one 30-second start', () => {
|
||||
const { policy } = fixture();
|
||||
const report = path.join(path.dirname(policy.file), 'qa-only-report.md');
|
||||
const start = policy.prompt.split('\n').find(line => line.startsWith('Start exactly once'))!;
|
||||
expect(start).toContain(`your Write of the initial charter to ${report} has completed`);
|
||||
expect(start.endsWith(`bun ${quote(guard)} start ${quote(policy.file)} 30`)).toBe(true);
|
||||
expect(start.indexOf('Write of the initial charter')).toBeLessThan(start.indexOf(' start '));
|
||||
});
|
||||
|
||||
test.each(['early', 'failed', 'unacknowledged', 'wrong-path', 'after-start', 'delayed-ack', 'subagent',
|
||||
'crossed-parent', 'duplicate-use', 'duplicate-result', 'wrong-result-id', 'empty', 'malformed-content', 'malformed-result'])
|
||||
('preparation credit requires an owned acknowledged nonempty Write: %s', scenario => {
|
||||
|
||||
@@ -14,6 +14,7 @@ import { collectorOutcomeCounts } from '../scripts/test-paid-shards';
|
||||
import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
import nativeNoChangeCases from './fixtures/shared-libs-no-change-ci-public.json';
|
||||
import r44 from './fixtures/shared-libs-index-flags-r44-packets.json';
|
||||
import claudeVersionProbe from './fixtures/shared-libs-claude-version-probe.json';
|
||||
import skipDescriptions from './fixtures/shared-libs-index-flags-skip-description-public.json';
|
||||
import { seedPathReviewPrerequisites, checkPathReviewPrerequisites } from './helpers/shared-libs-path-fixture';
|
||||
|
||||
@@ -1801,6 +1802,24 @@ describe('retained native runtime callback failures', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('recognizes the captured Claude Code 2.1.292 version probe as that exact argv only', () => {
|
||||
const request = claudeVersionProbe.public_request, commands = claudeVersionProbe.matching_model_commands;
|
||||
expect(request.args.slice(0, 2)).toEqual(['-c', 'core.askPass=']);
|
||||
expect(request.args.at(-1)).toBe('--version');
|
||||
expect(commands).toEqual([]);
|
||||
expect(isInternalClaudeGitRequest(request, commands)).toBe(true);
|
||||
expect(isGuardedGitRequest(request)).toBe(false);
|
||||
for (const parentExecutable of ['/usr/bin/bash', '/runtime/notclaude.exe', '']) {
|
||||
expect(isInternalClaudeGitRequest({ ...request, parentExecutable }, commands)).toBe(false);
|
||||
}
|
||||
expect(isInternalClaudeGitRequest({ ...request, ppid: undefined }, commands)).toBe(false);
|
||||
for (const args of [[...request.args.slice(0, -1), 'status'], [...request.args.slice(0, -1), 'log', '-1'],
|
||||
[...request.args, 'status'], request.args.slice(2), request.args.filter(arg => arg !== 'hook.post-index-change.enabled=false')]) {
|
||||
expect(isInternalClaudeGitRequest({ ...request, args }, commands), args.slice(-2).join(' ')).toBe(false);
|
||||
}
|
||||
expect(isInternalClaudeGitRequest(request, [`git ${request.args.join(' ')}`])).toBe(false);
|
||||
});
|
||||
|
||||
test('answers the exact captured two-question skip-only request without altering its input', async () => {
|
||||
const input = structuredClone(nativeCallbackReceipts.actor.public_question), before = structuredClone(input);
|
||||
const observed: unknown[] = [];
|
||||
|
||||
@@ -6,7 +6,8 @@ import { spawnSync } from 'node:child_process';
|
||||
import type { HookCallback, Query, SDKMessage } from '@anthropic-ai/claude-agent-sdk';
|
||||
import type { QueryProvider } from './helpers/agent-sdk-runner';
|
||||
import type { EvalTestEntry } from './helpers/eval-store';
|
||||
import { createShipHookFixture, runShipHookActor, type ShipHookCase } from './helpers/ship-hook-actor';
|
||||
import { createShipHookFixture, runShipHookActor, shipHookFailures, type ShipHookCase } from './helpers/ship-hook-actor';
|
||||
import capturedLocalHookReply from './fixtures/ship-local-hook-reply.json';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { DEFAULT_SHARD_TIMEOUT_MS, retriesForFiles } from '../scripts/test-paid-shards';
|
||||
|
||||
@@ -184,3 +185,16 @@ test('fixture input is the exact generated guard and does not link to live regis
|
||||
expect(fixture.snapshot().workflowSha256).toMatch(/^[a-f0-9]{64}$/);
|
||||
} finally { fs.rmSync(fixture.root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('local-hook preservation accepts the captured manual-integration wording and rejects replies without it', () => {
|
||||
const fixture = createShipHookFixture('ship-local-hook-preservation');
|
||||
const reported = (output: string) => !shipHookFailures(fixture, { exitReason: 'success', output }).failures.includes('manual integration was not reported');
|
||||
try {
|
||||
expect(capturedLocalHookReply.recordedFailures).toEqual(['manual integration was not reported']);
|
||||
expect(reported(capturedLocalHookReply.assistant)).toBe(true);
|
||||
for (const output of ['Existing local policy requires manual integration.', 'The rules say you need to integrate the hook yourself.',
|
||||
'Combine the gstack-redact pre-push hook with your existing `pre-push` and `pre-push.local` by hand.']) expect(reported(output), output).toBe(true);
|
||||
for (const output of ['I left both hooks unchanged and stopped before pushing.', 'I integrated the hook myself.',
|
||||
'The hooks were combined by hand earlier.', 'Do it yourself. I did not integrate anything.', 'No changes; nothing to combine.\nDone by hand.']) expect(reported(output), output).toBe(false);
|
||||
} finally { fs.rmSync(fixture.root, { recursive: true, force: true }); }
|
||||
});
|
||||
Reference in new issue
Block a user