CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-11-10 02:30:39 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(evals): owned plan Edit menu binds to exactly one pending Edit, including unpublished ones

Browse Source 
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-18 22:30:39 +00:00
1 parent 6f327370b4
commit d0b46d7058
6 files changed
+339 -68
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

CHANGELOG.md
+1
View File
@@ -19,6 +19,7 @@ The second Opus gate (run 39161502626) went 127/132. Each of the five reds is no
- **tpa-absent-linux** (the third gate, run 39166935531): Opus offered "step-by-step instructions to do it yourself" instead of saying "manual". Beside "manual", one sentence offering instructions or steps the user follows themselves now counts as the manual option.
- **qa-only-no-fix:** the fixture prompt now says the 30-second guard starts only after the charter Write has completed, and that local inspection outside the listed calls uses Read/Glob, not `ls`, `wc`, `cat` or `find` (its own `ls -la` entries had read as general permission). The budget and the assertion are unchanged.
- **plan-ceo-finding-floor:** Opus saved its D1 decision to the owned working plan before printing any text, as the 0D Save step requires. The permission check identified the parent session only from text or questions, so it never granted that Create and the run timed out after 600 s. A parent with only tool events is now identified by them, still as exactly one session that owns the pending Write. The floor runner also fails within 20 s with `PERMISSION_DIALOG_UNANSWERED` when a permission dialog gets no owned grant.
- **plan-ceo-finding-floor, Edit menu** (3 of 10 measured trials): after that Create, Opus edited the plan and the cropped Edit menu stayed unanswered. Twice Claude Code showed the menu before publishing the Edit, and once three Edits were pending at once, so the check found no single published Edit. The plan's own PreToolUse input now covers an unpublished Edit, as it already did for Write, and the visible diff must match exactly one pending Edit on the owned plan. An Edit to another file, a diff matching none or several pending Edits, and a foreign session are still refused.
### If you…
- **…compare pass rates across the switch:** Fable trials carry `harness_version: 5` and Opus trials `6`; they are separate series.
scripts/typecheck-test-baseline.json
-1
View File
@@ -226,7 +226,6 @@
"test/helpers/hermetic-env.test.ts\tTS7053\tElement implicitly has an 'any' type because expression of type 'string' can't be used to index type '{ ... 6 more ...; EVALS_SELECTION_JSON: string; GITHUB_APP_PRIVATE_KEY: string; GITHUB_CLIENT_SECRET: string; GITHUB_PAT: string; GITHUB_PERSONAL_ACCESS_TOKEN: string; GITHUB_TOKEN: string; NODE_ENV?: string | undefined; TZ?: string | undefined; }'. No index signature with a parameter of type 'string' was found on type '{ ... 6 more ...; EVALS_SELECTION_JSON: string; GITHUB_APP_PRIVATE_KEY: string; GITHUB_CLIENT_SECRET: string; GITHUB_PAT: string; GITHUB_PERSONAL_ACCESS_TOKEN: string; GITHUB_TOKEN: string; NODE_ENV?: string | undefined; TZ?: string | undefined; }'.": 1,
"test/helpers/office-hours-completion.ts\tTS18047\t'disposition' is possibly 'null'.": 2,
"test/helpers/plan-count-file-permission.ts\tTS18048\t'event.input' is possibly 'undefined'.": 3,
"test/helpers/plan-count-file-permission.ts\tTS18048\t'input' is possibly 'undefined'.": 12,
"test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'questions' does not exist on type 'AskUserQuestionFingerprint'.": 4,
"test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'selectedOptions' does not exist on type 'AskUserQuestionFingerprint'.": 4,
"test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'toolUseId' does not exist on type 'AskUserQuestionFingerprint'.": 8,
test/fixtures/plan-ceo-floor-edit-prepublication.json
Vendored
+115
View File
@@ -0,0 +1,115 @@
{
"source": "Ship-measure round baseline at 6f32737 (local, diagnostic): plan-ceo-finding-floor reds t04, t07, t08 ended permission_unanswered on the owned working-plan Edit menu.",
"note": "t04 Edit inputs are the published native inputs. t07/t08 Edits were never published before the deadline; their inputs are reconstructed from the captured diff rows (every removed row matched the captured Write bytes exactly) and their tool IDs are placeholders.",
"cases": [
{
"trial": "t04",
"source": "ship-measure plan-ceo-finding-floor baseline t04: t04/eval/t1/shards/skill-e2e-plan-ceo-finding-floor/pty-count/local-measure-plan-ceo-finding-floor-baseline-t4-39512/plan-ceo-review-1792361226070-y2Y3JE",
"sessionId": "b4f2ae78-2b26-4287-934b-f7b48e95e3c5",
"basename": "gstack-test-plan-ceo-floor.md",
"recorded": {
"outcome": "permission_unanswered",
"transcriptStatus": "ready",
"transcriptCalls": 0,
"transcriptAssistantMessages": 1,
"pendingWriteInputs": [],
"publishedEdits": 3
},
"screen": " 57 ```text\n 58 -Commitment | Source/approval or pending | Current | A | B \n - | C \n 59 -Developer tier is the product | user: preserve scope | yes | yes | yes, gated on \n -evidence | yes, as experiment arm \n 60 -Talk to developers first | pending | no | no | yes, 8-12 inte\n -rviews | no \n 61 -Baseline funnel numbers | pending | no | no | yes \n - | yes \n 62 -Concrete success metric | pending | \"more signups\" | \"more signups\" | activation + p\n -aid conv. | activation + paid conv. \n 63 -Price change reversible | pending | no | no | decided after \n -evidence | yes, cohort-limited \n 64 -Time before tier ships | pending | now | now | +~2 weeks \n - | +~4-6 weeks \n 58 +Commitment | Source/approval or pending | Current | A (validate) | B (as\n + written) | C (experiment) \n 59 +Developer tier is the product | user: preserve scope | yes | yes, gated on evidence | yes \n + | yes, as experiment arm \n 60 +Talk to developers first | pending | no | yes, 8-12 interviews | no \n + | no \n 61 +Baseline funnel numbers | pending | no | yes | no \n + | yes \n 62 +Concrete success metric | pending | \"more signups\" | activation + paid conv. | \"more\n + signups\" | activation + paid conv. \n 63 +Price change reversible | pending | no | decided after evidence | no \n + | yes, cohort-limited \n 64 +Time before tier ships | pending | now | +~2 weeks | now \n + | +~4-6 weeks \n 65 ```\n 66 \n 67 Question: D1 — P1: Do we ship the cheaper tier on gut feel, or validate that price is the barrier first?\n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n Do you want to make this edit to gstack-test-plan-ceo-floor.md?\n ❯ 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel · Tab to amend\n",
"screenSha256": "de5a50a2af7bf2eb8ee5de4663198882ca7a9bc84c93c57934d00d34b3e3f724",
"writeContent": "# CEO Review: Launch a \"developer-friendly\" pricing tier\n\nPlan under review: `PLAN.md` (commit 897aa82). Reviewer: /plan-ceo-review, 2026-10-18.\nReview depth: strategy-only (the plan has no implementation design; there is no code in the repo).\n\n## Context\nThe plan proposes a cheaper \"developer-friendly\" tier to increase developer adoption, measured by\n\"more signups\". Its own premise section says no developer has been asked whether price is a barrier;\nthe team's belief is that it \"feels like\" it should be cheaper (PLAN.md:9-11).\n\n## Pre-review system audit\n- Repo: one commit (`897aa82 Seed review plan`), files `CLAUDE.md`, `PLAN.md`. No remote, no stash, no TODOS.md, no TODO/FIXME markers.\n- No design doc. /office-hours offer skipped per user instruction.\n- Prior learnings: none. Branch decisions: none. Brain context: cold.\n- UI scope: none named in the plan. Section 11 likely skipped.\n- Landscape (WebSearch; Aside unavailable): dev-tool pricing has converged on ~$20/mo standard with thinning free tiers; ~74% of suppliers use usage-based models; developers dislike billing unpredictability more than list price (sources: costbench.com/software/developer-tools, agentdeals.dev/ai-coding-pricing-2026, nexdigm.com developer-platform-pricing-benchmarking).\n - Layer 1: dev adoption is usually won by a frictionless free/self-serve path, time-to-first-value and predictable billing, not by a lower sticker price.\n - Layer 3: if signups are low because of onboarding friction, docs, or the free tier's shape, a cheaper paid tier does nothing and cuts revenue from developers who would have paid.\n\n## 0A. Premise challenge\n- Stated problem: low developer adoption. Real problem: unknown. The plan has not established *why* developers don't adopt.\n- Proxy risk: \"price is the barrier\" is an untested hypothesis (PLAN.md:10-11). The plan solves a proxy.\n- Success metric \"more signups\" (PLAN.md:6) has no baseline, target, window, or segment, and signups are not adoption (activation/retention). Any change will \"succeed\" or \"fail\" by noise.\n- Do-nothing cost: unknown without baseline funnel numbers. Cost of being wrong: permanent price anchoring (price cuts are hard to reverse), cannibalized revenue from existing paying devs, and the real barrier stays unfixed.\n\n## 0B. Existing leverage\nNo code in the repo. Reusable assets the plan should name: existing signup/activation funnel analytics, current customer list (for interviews), billing system's plan/coupon support (for a reversible price test).\n\nREALITY: premise-table finding unverified=2 (\"price is a barrier\", \"cheaper tier increases adoption\"); 0 receipts PLAN.md:9\nREALITY: already-done pass single seed commit, no prior pricing work in history PLAN.md:1\nREALITY: surface-check pass plan names no CLI commands, flags or routes PLAN.md:1\nREALITY: binding-decisions pass gstack-decision-search returned none; plan has no Binding decisions section PLAN.md:1\nREALITY: numbers finding no numbers at all; \"more signups\" has no baseline or target [re-measure before gate] PLAN.md:6\nREALITY: deferred-asks pass no linked issue PLAN.md:1\nREALITY: pr-count pass no code deliverable; no ship-policy.json, default one PR PLAN.md:1\n\n## 0C. Dream state\n```\n CURRENT STATE THIS PLAN 12-MONTH IDEAL\n Adoption \"low\" (unmeasured); Ship a cheaper tier on gut Known #1 adoption barrier, fixed;\n no dev interviews; no baseline feel; measure \"more signups\" activation + paid conversion\n funnel ---> (no baseline) ---> measured weekly; pricing set from\n evidence, reversible experiments\n```\nAs written the plan moves sideways: it may raise signups while lowering revenue, and teaches nothing either way.\n\n## Decision ledger\n\n| ID and owner | Contract and evidence | Current | Proposed | Status | Exact approval and scope |\n|---|---|---|---|---|---|\n| MODE (user) | Review mode | HOLD SCOPE | n/a | approved | User instruction: \"For review-mode questions choose HOLD SCOPE (CEO)\" |\n| P1 (plan owner) | Premise/approach: is price the barrier? PLAN.md:9-11 | Ship cheaper tier on team intuition | A) as written; B) validate first (interviews + funnel baseline), tier conditional; C) reversible priced experiment | unresolved | pending D1 |\n\n## currentDecision (P1)\nCommitment comparison:\n\n```text\nCommitment | Source/approval or pending | Current | A | B | C\nDeveloper tier is the product | user: preserve scope | yes | yes | yes, gated on evidence | yes, as experiment arm\nTalk to developers first | pending | no | no | yes, 8-12 interviews | no\nBaseline funnel numbers | pending | no Line truncated
"edits": [
{
"toolUseId": "toolu_01ERak43Ep5qr7euvXoKxHe5",
"timestamp": "2026-10-18T22:08:15.794Z",
"published": true,
"input": {
"old_string": "```text\nCommitment | Source/approval or pending | Current | A | B | C\nDeveloper tier is the product | user: preserve scope | yes | yes | yes, gated on evidence | yes, as experiment arm\nTalk to developers first | pending | no | no | yes, 8-12 interviews | no\nBaseline funnel numbers | pending | no | no | yes | yes\nConcrete success metric | pending | \"more signups\" | \"more signups\" | activation + paid conv. | activation + paid conv.\nPrice change reversible | pending | no | no | decided after evidence | yes, cohort-limited\nTime before tier ships | pending | now | now | +~2 weeks | +~4-6 weeks\n```",
"new_string": "```text\nCommitment | Source/approval or pending | Current | A (validate) | B (as written) | C (experiment)\nDeveloper tier is the product | user: preserve scope | yes | yes, gated on evidence | yes | yes, as experiment arm\nTalk to developers first | pending | no | yes, 8-12 interviews | no | no\nBaseline funnel numbers | pending | no | yes | no | yes\nConcrete success metric | pending | \"more signups\" | activation + paid conv. | \"more signups\" | activation + paid conv.\nPrice change reversible | pending | no | decided after evidence | no | yes, cohort-limited\nTime before tier ships | pending | now | +~2 weeks | now | +~4-6 weeks\n```",
"replace_all": false
}
},
{
"toolUseId": "toolu_01XC8JDRFNTi4im7Fa8HbgKh",
"timestamp": "2026-10-18T22:08:17.443Z",
"published": true,
"input": {
"old_string": "A) as written; B) validate first (interviews + funnel baseline), tier conditional; C) reversible priced experiment",
"new_string": "A) validate first (interviews + funnel baseline), tier conditional; B) ship as written; C) reversible priced experiment",
"replace_all": false
}
},
{
"toolUseId": "toolu_01H3Q3ia3Zr5F4ssU6p1Ym1X",
"timestamp": "2026-10-18T22:08:18.689Z",
"published": true,
"input": {
"old_string": "Recommendation: B because",
"new_string": "Recommendation: A because",
"replace_all": false
}
}
]
},
{
"trial": "t07",
"source": "ship-measure plan-ceo-finding-floor baseline t07: t07/eval/t1/shards/skill-e2e-plan-ceo-finding-floor/pty-count/local-measure-plan-ceo-finding-floor-baseline-t7-39512/plan-ceo-review-1792361461841-1z5Jvz",
"sessionId": "22c9e376-7347-48da-90f2-2624d98e8d4c",
"basename": "gstack-test-plan-ceo-floor.md",
"recorded": {
"outcome": "permission_unanswered",
"transcriptStatus": "ready",
"transcriptCalls": 0,
"transcriptAssistantMessages": 1,
"pendingWriteInputs": [],
"publishedEdits": 0
},
"screen": " 72 Question: D1 — P1: The plan's premise (price is the barrier) has zero developer evidence. How should the plan treat\n it?\n 73 +Project/branch/task: CEO review of PLAN.md on main, a cheaper \"developer-friendly\" pricing tier. \n 74 +ELI10: The team wants to make a cheaper plan for developers because it feels right, but nobody asked developers if \n +price is what stops them. If price isn't the real blocker, we give away revenue and signups barely move, and price \n +cuts are hard to take back publicly. \n 75 +Stakes if we pick wrong: we cut price, signups stay flat, revenue per user drops, and we still don't know why devs \n +leave. \n 76 +Recommendation: B because it keeps the tier in scope but spends ~2 weeks of interviews and funnel data before an ir\n +reversible public price change. \n 77 +Note: options differ in kind, not coverage — no completeness score. \n 78 +Net: speed of launching on a hunch vs. two weeks to know whether price is the lever at all. \n 79 Header: Premise\n 80 A) Ship tier as written\n 75 -Keep the plan: launch a cheaper developer tier now on team intuition. Effort S, risk high, no verification of the p\n -remise. \n 81 +Launch a cheaper dev tier now on team intuition. Effort S, risk high, zero premise verification. ✅ Fastest path, t\n +he team can ship it within days of deciding a price. ✅ No research overhead or extra people needed before launch. \n +❌ Public price cuts are hard to reverse if signups do not move at all. \n 82 B) Validate, then ship tier (recommended)\n 77 -Keep the tier in scope but gate its price/shape on 8-12 dev interviews plus a funnel baseline, and replace \"more si\n -gnups\" with a measured target. Effort M, risk low. \n 83 +Keep the tier, but set its price and shape from 8-12 dev interviews plus a funnel baseline; replace \"more signups\" \n +with a measured target. Effort M, risk low. ✅ Tier still ships, now aimed at the step where devs actually drop off\n +. ✅ Gives a real baseline so the launch can be judged a win or loss. ❌ Delays launch roughly two weeks (human: ~2\n + wks / CC: ~1 day of synthesis). \n 84 C) Reframe to adoption friction\n 79 -Treat pricing as one lever in a broader developer-activation program (onboarding, docs, free usage). Effort L, risk\n - medium; widens scope beyond the plan. \n 85 +Treat price as one lever in a broader dev-activation program (onboarding, docs, free usage). Effort L, risk medium.\n + ✅ Attacks adoption directly instead of a single assumed cause. ✅ Likely the larger long-term win if friction, no\n +t price, is the blocker. ❌ Widens scope beyond the supplied plan, which the user asked to preserve. \n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n Do you want to make this edit to gstack-test-plan-ceo-floor.md?\n ❯ 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel · Tab to amend\n",
"screenSha256": "6571c111d303107f9eaf63604179077845fa0156b177f2da66dccfa77233f68f",
"writeContent": "# CEO Review Working Plan: \"Developer-friendly\" pricing tier\n\nSource under review: `PLAN.md` (lines 19-29). Review: /plan-ceo-review, 2026-10-18, branch `main`.\nReview depth: strategy-only (plan is product/pricing strategy; no implementation design requested, no code in repo).\n\n## Plan under review (verbatim intent)\n- Goal: Increase developer adoption. (PLAN.md:22)\n- Success metric: More signups. (PLAN.md:25)\n- Premise: No developers consulted; team \"feels like\" pricing should be cheaper. (PLAN.md:28-29)\n\n## Settled by user instruction (not review findings)\n- /office-hours prerequisite: skipped (CLAUDE.md:9).\n- Routing setup, cross-project recall, outside reviewers: declined (CLAUDE.md:11).\n- Review mode: HOLD SCOPE (CLAUDE.md:13). Product scope preserved.\n- Explicitly NOT approved: premise, approach, remedies (CLAUDE.md:15).\n\n## Pre-review system audit\n- Git: single commit `95e8648 Seed review plan`; no stashes; no diff vs `main`; no remote.\n- Files: CLAUDE.md, PLAN.md only. No TODOS.md, no architecture docs, no design doc, no pricing code, no billing integration, no analytics.\n- Prior learnings: none. Prior decisions: none. Brain context: cold.\n- UI scope: plan implies a pricing page / checkout change but names none. DESIGN_SCOPE: unknown, pending approach.\n\n## Landscape check (WebSearch; Aside unavailable)\n- [Layer 1] Dev tools grow bottom-up: devs self-validate on a free/usage tier before advocating a paid team plan. Price is rarely the first barrier; activation friction is.\n- [Layer 2] 2026 guidance: gate by usage intensity (API calls, repos, seats), not raw price; the pricing page is a trust signal and devs punish opaque pricing. Sources: getmonetizely.com dev-tool pricing articles (vendor content, treat as weak evidence).\n- [Layer 3] First principles: \"cheaper\" only helps if price is the binding constraint at the step where devs drop out. Without funnel data, a cut price can lower revenue per user while signups stay flat, and price cuts are hard to reverse publicly.\n\n## 0A. Premise challenge (finding, pending decision)\n- Real problem: unknown. \"Developer adoption\" is not measured; the plan does not say where devs drop (discover, sign up, activate, convert, retain).\n- Plan solves a proxy: price is assumed to be the barrier with zero evidence (PLAN.md:28-29).\n- Success metric \"more signups\" (PLAN.md:25) has no baseline, target, window or quality bar. A cheaper tier can inflate signups while activation and revenue fall.\n- Do-nothing cost: unknown, because current adoption numbers are not stated.\n\n## 0B. Existing leverage\n- No code in repo to reuse. Anything about billing, plans, pricing page, analytics: implementation owner must prove what exists.\n\nREALITY: premise-table finding unverified=3 (price-is-barrier, devs-want-cheaper, signups-track-adoption) PLAN.md:28\nREALITY: already-done pass no remote/base history beyond seed commit; nothing shipped PLAN.md:19\nREALITY: surface-check n/a plan names no CLI, flag or route PLAN.md:19\nREALITY: binding-decisions pass none recorded; CLAUDE.md scope instructions honored CLAUDE.md:13\nREALITY: numbers finding plan has zero numbers: no price, baseline, target or window PLAN.md:25\nREALITY: deferred-asks pass no linked issue PLAN.md:19\nREALITY: pr-count pass no ship-policy; strategy-only, no PRs planned PLAN.md:19\n\n## 0C. Dream state\n```\n CURRENT STATE THIS PLAN 12-MONTH IDEAL\n Unknown dev adoption, ---> Cheaper tier on a hunch, ---> Measured dev funnel; pricing\n no funnel data, no dev success = \"more signups\" tied to the real drop-off step;\n interviews (unmeasured) devs self-serve to paid teams\n```\nAs written, the plan moves sideways: it changes price without knowing whether price is the constraint.\n\n## Decision ledger\n\n| ID and owner | Contract and evidence | Current | Proposed | Status | Exact approval and scope |\n|---|---|---|---|---|---|\n| MODE (user) | CLAUDE.md:13 | HOLD SCOPE | - | approved | CLAUDE.md:13 \"choose HOLD SCOPE (CEO)\" |\n| P1 premise/approach (plan owner) | PLAN.md:28-29 no dev evidence | Launch cheaper tier on team intuition | A) as written; B) validate premise first, tier gated on evidence; C) reframe to adoption-friction program | unresolved | pending D1 |\n\n## currentDecision (P1)\nCommitment comparison:\n\nCommitment | Source/approval or pending | Current | A | B | C\n---|---|---|---|---|---\nShip a dev-friendly pricing tier | PLAN.md:19, scope preserved | yes | yes, now | yes, shaped by evidence | maybe, one lever among several\nTalk to developers before pricing | pending | no | no | yes, 8-12 interviews | yes\nFunnel baseline before launch | pending | no | no | yes | yes\nSuccess metric | pending | \"more signups\" | unchanged | activation-qualified signups w/ baseline + target | activation + paid conversion\nEffort | pending | - | S | M | L\n\Line truncated
"edits": [
{
"toolUseId": "toolu_t07_unpublished_edit",
"timestamp": null,
"published": false,
"reconstructed": true,
"input": {
"old_string": "Header: Premise\nA) Ship tier as written\nKeep the plan: launch a cheaper developer tier now on team intuition. Effort S, risk high, no verification of the premise.\nB) Validate, then ship tier (recommended)\nKeep the tier in scope but gate its price/shape on 8-12 dev interviews plus a funnel baseline, and replace \"more signups\" with a measured target. Effort M, risk low.\nC) Reframe to adoption friction\nTreat pricing as one lever in a broader developer-activation program (onboarding, docs, free usage). Effort L, risk medium; widens scope beyond the plan.",
"new_string": "Project/branch/task: CEO review of PLAN.md on main, a cheaper \"developer-friendly\" pricing tier.\nELI10: The team wants to make a cheaper plan for developers because it feels right, but nobody asked developers if price is what stops them. If price isn't the real blocker, we give away revenue and signups barely move, and price cuts are hard to take back publicly.\nStakes if we pick wrong: we cut price, signups stay flat, revenue per user drops, and we still don't know why devs leave.\nRecommendation: B because it keeps the tier in scope but spends ~2 weeks of interviews and funnel data before an irreversible public price change.\nNote: options differ in kind, not coverage — no completeness score.\nNet: speed of launching on a hunch vs. two weeks to know whether price is the lever at all.\nHeader: Premise\nA) Ship tier as written\nLaunch a cheaper dev tier now on team intuition. Effort S, risk high, zero premise verification. ✅ Fastest path, the team can ship it within days of deciding a price. ✅ No research overhead or extra people needed before launch. ❌ Public price cuts are hard to reverse if signups do not move at all.\nB) Validate, then ship tier (recommended)\nKeep the tier, but set its price and shape from 8-12 dev interviews plus a funnel baseline; replace \"more signups\" with a measured target. Effort M, risk low. ✅ Tier still ships, now aimed at the step where devs actually drop off. ✅ Gives a real baseline so the launch can be judged a win or loss. ❌ Delays launch roughly two weeks (human: ~2 wks / CC: ~1 day of synthesis).\nC) Reframe to adoption friction\nTreat price as one lever in a broader dev-activation program (onboarding, docs, free usage). Effort L, risk medium. ✅ Attacks adoption directly instead of a single assumed cause. ✅ Likely the larger long-term win if friction, not price, is the blocker. ❌ Widens scope beyond the supplied plan, which the user asked to preserve.",
"replace_all": false
}
}
]
},
{
"trial": "t08",
"source": "ship-measure plan-ceo-finding-floor baseline t08: t08/eval/t1/shards/skill-e2e-plan-ceo-finding-floor/pty-count/local-measure-plan-ceo-finding-floor-baseline-t8-39512/plan-ceo-review-1792361461901-BWluIb",
"sessionId": "77d807e3-5777-405c-9b05-143b4169bae6",
"basename": "gstack-test-plan-ceo-floor.md",
"recorded": {
"outcome": "permission_unanswered",
"transcriptStatus": "ready",
"transcriptCalls": 0,
"transcriptAssistantMessages": 1,
"pendingWriteInputs": [],
"publishedEdits": 0
},
"screen": " 67 -Talk to developers first | pending | no | no | yes, 10-15\n - interviews + funnel read | no (data from experiment instead) \n 68 -Funnel instrumentation | pending | unknown | not required | read exist\n -ing funnel, add gaps | required (A/B split) \n 69 -Price-point decision basis | pending | team intuition | team intuition | interview \n -+ funnel evidence | experiment result \n 70 -Reversibility | pending | low | low | high (noth\n -ing shipped until gate) | medium (exposed cohort keeps price) \n 71 -Time to tier in market | pending | fastest | fastest | +1-2 weeks\n - | +1 week setup, 2-4 week run \n 72 -Success metric (D2) | pending, separate row | \"More signups\" | unchanged, pending | unchanged,\n - pending | unchanged, pending \n 65 +Commitment | Source/approval or pending | Current | A (discovery gate) \n + | B (experiment) | C (as written) \n 66 +Developer-friendly tier exists | PLAN.md scope (preserved) | planned | yes, gated on discovery evidenc\n +e | yes, as experiment arm | yes \n 67 +Talk to developers first | pending | no | yes, 10-15 interviews + funnel \n +read | no (data from experiment instead) | no \n 68 +Funnel instrumentation | pending | unknown | read existing funnel, add gaps \n + | required (randomized split) | not required \n 69 +Price-point decision basis | pending | team intuition | interview + funnel evidence \n + | experiment result | team intuition \n 70 +Reversibility | pending | low | high (nothing shipped until gat\n +e) | medium (exposed cohort keeps price) | low \n 71 +Time to tier in market | pending | fastest | +1-2 weeks \n + | +1 week setup, 2-4 week run | fastest \n 72 +Success metric (D2) | pending, separate row | \"More signups\" | unchanged, pending \n + | unchanged, pending | unchanged, pending \n 73 ```\n 74 \n 75 Question: D1 — D1-PREMISE: Should the pricing tier ship on team intuition, or after testing whether price is actual\n ly the barrier? \n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n Do you want to make this edit to gstack-test-plan-ceo-floor.md?\n ❯ 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel · Tab to amend\n",
"screenSha256": "13dd1c4e9c135a5e66b511ee9eb3421e7c4a1797e883f88af3eaa0876d72ef4f",
"writeContent": "# CEO Review Working Plan: \"Developer-friendly\" pricing tier\n\nSource under review: `PLAN.md` (commit 47a67a6). Skill: /plan-ceo-review. Branch: main.\nReview depth: strategy-only (plan contains no implementation design; user asked for a plan review, not implementation).\nMode: HOLD SCOPE (pre-answered in CLAUDE.md: \"For review-mode questions choose HOLD SCOPE (CEO)\"). Mode handoff runs after premise/approach decisions.\n\n## Pre-review system audit\n- `git log`: one commit, `47a67a6 Seed review plan`. No stashes, no remote, no base diff.\n- No TODOS.md, no architecture docs, no code. Plan is pure product/pricing strategy.\n- No /office-hours design doc; offer skipped per CLAUDE.md instruction.\n- Prior learnings: 0. Cross-project recall: false (pre-declined). Brain digests: none. Decision log: empty.\n- DESIGN_SCOPE: no UI in plan as written (pricing page changes would be UI, but not specified). Section 11 likely skipped.\n- Landscape (WebSearch; Aside not installed):\n - [L1] Tried-and-true: generous free tier for individual adoption, paid tiers gate team/collab/enterprise features.\n - [L2] 2026 data: 88% of tracked dev tools already have free tiers; Q1 2026 saw net tightening (173 negative vs 32 positive changes). Upgrade drivers skew to features (39%) and usage (28%); barriers are budget (46%) and ROI uncertainty (35%).\n - [L3] First principles: if 88% of competitors already have a free tier, \"cheaper\" is table stakes, not differentiation. If the real barrier is activation friction or unclear value, a price cut lowers revenue and moves no signups.\n\n## Step 0A: Premise challenge\n- **Stated problem:** low developer adoption. **Proposed fix:** a cheaper tier.\n- **Evidence for the causal link:** none. PLAN.md:\"Premise\" says verbatim: \"We haven't talked to any developers about whether the current pricing is actually a barrier. The team agreed it 'feels like' it should be cheaper.\"\n- **Proxy vs. direct:** price is a guessed proxy for the real pain. Unknown whether devs drop at the pricing page, at signup, at first-run/activation, or never arrive (awareness).\n- **Success metric \"More signups\":** no baseline, no target, no time window, no segment (\"developer\" undefined), no guardrail (revenue/ARPU cannibalization, paid-to-cheap downgrades). Signups is also a vanity metric if activation is the bottleneck.\n- **Do-nothing cost:** unknown, because current adoption numbers are not cited.\n- **Cost of being wrong:** pricing is a semi-one-way door. Cutting price is easy; raising it back later angers users and costs trust. Existing paying customers may downgrade.\n\n## Step 0B: Existing leverage\nNo code in repo. Reusable assets to confirm (owner must prove): current pricing page analytics, signup funnel events, existing customer list for interviews, billing system's ability to add a plan/coupon without migration.\n\n### Reality rows\n| premise | claim | receipt | verdict |\n|---|---|---|---|\n| P1 | Current pricing is a barrier to developers | PLAN.md:13-14 (admits no evidence) | unverified |\n| P2 | A cheaper tier increases adoption | PLAN.md:4-5 | unverified |\n| P3 | Signups measure adoption | PLAN.md:7-8 | unverified |\nunverified=3\n\nREALITY: premise-table finding 3 of 3 premises unverified; plan admits no developer conversations PLAN.md:13\nREALITY: already-done pass single-commit repo, no prior pricing work, no remote to fetch PLAN.md:1\nREALITY: surface-check n/a-equivalent pass no CLI commands, flags or routes named in plan PLAN.md:1\nREALITY: binding-decisions pass decision log empty; no `## Binding decisions` section PLAN.md:1\nREALITY: numbers finding plan contains zero numbers: no baseline, target, price point or window PLAN.md:7-8\nREALITY: deferred-asks pass no linked issue PLAN.md:1\nREALITY: pr-count pass no .gstack/ship-policy.json; default one PR per wave; no code planned PLAN.md:1\n\n## Step 0C: Dream state\n```\n CURRENT STATE THIS PLAN 12-MONTH IDEAL\n Unknown adoption baseline. --> Cheaper tier shipped on --> Devs adopt because time-to-first-value\n No dev interviews. team intuition. Signups is minutes; pricing matches how they\n Price assumed to be the may tick up or not; no grow (free solo -> paid team); funnel\n barrier. way to tell why. instrumented; price set from evidence.\n```\nAs written, the plan moves toward the ideal only if price is in fact the barrier. Nothing in the plan can confirm or refute that.\n\n## Decision ledger\n| ID and owner | Contract and evidence | Current | Proposed | Status | Exact approval and scope |\n|---|---|---|---|---|---|\n| MODE (admin) | CLAUDE.md \"choose HOLD SCOPE (CEO)\" | HOLD SCOPE | none | approved | CLAUDE.md instruction; review mode only, approves no plan changes |\n| D1-PREMISE (owner: plan author / PM) | PLAN.md:13-14 admits pricing-as-barrier is untested | A) launch tier on team intuition | Line truncated
"edits": [
{
"toolUseId": "toolu_t08_unpublished_edit",
"timestamp": null,
"published": false,
"reconstructed": true,
"input": {
"old_string": "Commitment | Source/approval or pending | Current | A | B | C\nDeveloper-friendly tier exists | PLAN.md scope (preserved) | yes | yes | yes, gated on discovery evidence | yes, as experiment arm\nTalk to developers first | pending | no | no | yes, 10-15 interviews + funnel read | no (data from experiment instead)\nFunnel instrumentation | pending | unknown | not required | read existing funnel, add gaps | required (A/B split)\nPrice-point decision basis | pending | team intuition | team intuition | interview + funnel evidence | experiment result\nReversibility | pending | low | low | high (nothing shipped until gate) | medium (exposed cohort keeps price)\nTime to tier in market | pending | fastest | fastest | +1-2 weeks | +1 week setup, 2-4 week run\nSuccess metric (D2) | pending, separate row | \"More signups\" | unchanged, pending | unchanged, pending | unchanged, pending",
"new_string": "Commitment | Source/approval or pending | Current | A (discovery gate) | B (experiment) | C (as written)\nDeveloper-friendly tier exists | PLAN.md scope (preserved) | planned | yes, gated on discovery evidence | yes, as experiment arm | yes\nTalk to developers first | pending | no | yes, 10-15 interviews + funnel read | no (data from experiment instead) | no\nFunnel instrumentation | pending | unknown | read existing funnel, add gaps | required (randomized split) | not required\nPrice-point decision basis | pending | team intuition | interview + funnel evidence | experiment result | team intuition\nReversibility | pending | low | high (nothing shipped until gate) | medium (exposed cohort keeps price) | low\nTime to tier in market | pending | fastest | +1-2 weeks | +1 week setup, 2-4 week run | fastest\nSuccess metric (D2) | pending, separate row | \"More signups\" | unchanged, pending | unchanged, pending | unchanged, pending",
"replace_all": false
}
}
]
}
]
}
test/helpers/free-fixtures-data.ts
+1
View File
@@ -145,6 +145,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
'test/fixtures/parity-baseline-v1.44.1.json': ['test/helpers/capture-parity-baseline.test.ts', 'test/parity-baseline-integrity.test.ts'],
'test/fixtures/parity-baseline-v1.64.1.0.json': ['test/parity-suite.test.ts', 'test/qa-lazy-sections.test.ts', 'test/skill-size-budget.test.ts'],
'test/fixtures/parity-baseline-v1.81.0.0.json': ['test/skill-size-budget.test.ts'],
'test/fixtures/plan-ceo-floor-edit-prepublication.json': ['test/plan-edit-cropped-permission.test.ts'],
'test/fixtures/plan-ceo-floor-toolonly-create.json': ['test/plan-create-prepublication.test.ts', 'test/plan-floor-permission.test.ts'],
'test/fixtures/plan-count-design-questionless-report.md': ['test/plan-count-empty-review.test.ts'],
'test/fixtures/plan-create-cropped-title-batching.json': ['test/plan-create-combined-permission.test.ts'],
test/helpers/plan-count-file-permission.ts
+130 -66
View File
@@ -81,8 +81,28 @@ export function readPendingWriteInput(file: string, expected: string, cwd: strin
}catch{return undefined;}
}
const digest = (bytes: Buffer) => createHash('sha256').update(bytes).digest('hex');
/** Private inputs of open owned Edits, keyed by native ID; the record holds their digest. */
function readPendingEditInputs(file: string, r: any): Record<string, any> {
if (typeof r.editInputsSha256 !== 'string') return {};
const bytes = boundedRegular(file + '.edit.json', MAX_WRITE_INPUT_BYTES);
if (digest(bytes) !== r.editInputsSha256) throw Error('changed Edit input witness');
return JSON.parse(bytes.toString('utf8'));
}
function writePendingEditInputs(file: string, state: any, inputs: Record<string, unknown>) {
delete state.editInputsSha256;
if (!Object.keys(inputs).length) { fs.rmSync(file + '.edit.json', {force:true}); return; }
const bytes = Buffer.from(JSON.stringify(inputs) + '\n');
if (bytes.length > MAX_WRITE_INPUT_BYTES) throw Error('oversized Edit input witness');
fs.writeFileSync(file + '.edit.json.tmp', bytes, {mode:0o600});
fs.renameSync(file + '.edit.json.tmp', file + '.edit.json');
state.editInputsSha256 = digest(bytes);
}
/** No stdout, permission decision, input rewrite or model context. Metadata stays
* content-free; only an owned Write gets a bounded private input sibling. */
* content-free; owned Write and Edit inputs get bounded private siblings. */
export function recordFilePermission(input: string, file: string, cwd: string, config: string, expected: string) {
try {
if (Buffer.byteLength(input) > MAX_WRITE_INPUT_BYTES) throw Error('oversized hook');
@@ -100,14 +120,17 @@ export function recordFilePermission(input: string, file: string, cwd: string, c
const id = `${e.session_id}:${e.tool_use_id}`;
const timestamp = new Date().toISOString();
const same = old.cwd === cwd && old.expected === expected && old.sessionId === e.session_id;
const state = { cwd, expected, sessionId: e.session_id, transcriptPath: e.transcript_path,
const edits = same ? readPendingEditInputs(file, old) : {};
const state: any = { cwd, expected, sessionId: e.session_id, transcriptPath: e.transcript_path,
seenIds: same && Array.isArray(old.seenIds) ? old.seenIds : [],
openIds: same && Array.isArray(old.openIds) ? old.openIds : [],
pendingId: same ? old.pendingId ?? null : null,
completedId: same ? old.completedId ?? null : null,
completedIds: same && Array.isArray(old.completedIds) ? old.completedIds : [],
timestamp,
...(same && old.overlappingWriteInput === true ? {overlappingWriteInput:true} : {}),
...(same && typeof old.writeInputSha256 === 'string' ? {writeInputSha256:old.writeInputSha256} : {}) };
...(same && typeof old.writeInputSha256 === 'string' ? {writeInputSha256:old.writeInputSha256} : {}),
...(same && typeof old.editInputsSha256 === 'string' ? {editInputsSha256:old.editInputsSha256} : {}) };
let clearWriteInput = false;
if (e.hook_event_name === 'PreToolUse') {
// Replayed requests, including failed and older completed IDs, never reopen.
@@ -121,12 +144,23 @@ export function recordFilePermission(input: string, file: string, cwd: string, c
fs.writeFileSync(file+'.tmp',JSON.stringify(old)+'\n',{mode:0o600});
fs.renameSync(file+'.tmp',file);
}
// A repeated Edit ID with different arguments loses its witness for good.
if (edits[id] && !isDeepStrictEqual(edits[id].input, e.tool_input)) {
delete edits[id];
writePendingEditInputs(file, old, edits);
fs.writeFileSync(file+'.tmp',JSON.stringify(old)+'\n',{mode:0o600});
fs.renameSync(file+'.tmp',file);
}
return;
}
if (state.seenIds.length >= 128) throw Error('too many file requests');
if(state.pendingId) state.overlappingWriteInput=true;
state.seenIds.push(id);
state.openIds.push(id);
state.pendingId = id;
if (e.tool_name === 'Edit' && typeof e.tool_input.old_string === 'string' && typeof e.tool_input.new_string === 'string')
edits[id] = {source:'PreToolUse',toolName:'Edit',cwd,expected,sessionId:e.session_id,
transcriptPath:e.transcript_path,pendingId:id,timestamp,input:e.tool_input};
delete state.writeInputSha256;
clearWriteInput = true;
if(e.tool_name==='Write' && typeof e.tool_input.content==='string') {
@@ -143,10 +177,15 @@ export function recordFilePermission(input: string, file: string, cwd: string, c
}
} else {
// An unrelated/late result cannot overwrite the current request epoch.
if (state.pendingId !== id) return;
state.pendingId = null;
delete state.writeInputSha256;
clearWriteInput = true;
// An earlier open request (parallel Edits) completes without moving it.
if (state.pendingId !== id && !state.openIds.includes(id)) return;
state.openIds = state.openIds.filter((open: string) => open !== id);
delete edits[id];
if (state.pendingId === id) {
state.pendingId = null;
delete state.writeInputSha256;
clearWriteInput = true;
}
if (e.hook_event_name === 'PostToolUse') {
state.completedId = id;
// Polling may miss automatically permitted edits between two menus.
@@ -154,6 +193,7 @@ export function recordFilePermission(input: string, file: string, cwd: string, c
if (!state.completedIds.includes(id)) state.completedIds.push(id);
}
}
writePendingEditInputs(file, state, edits);
fs.writeFileSync(file+'.tmp',JSON.stringify(state)+'\n',{mode:0o600});
fs.renameSync(file+'.tmp',file);
if(clearWriteInput) fs.rmSync(file+'.write.json',{force:true});
@@ -310,11 +350,64 @@ function currentCreatePreview(preview: string, r: any, config: string, cwd: stri
(!i||row.line===numbered[i-1]!.line+1)&&compact(row.text)===compact(source[row.line-1]!));
}
/** A cropped in-fixture Edit needs its sole current native request and exact visible diff. */
function currentEditPreview(preview: string, r: any, config: string, cwd: string, startedAt: number): boolean {
/** One Edit's exact visible diff against the current owned file bytes. */
function editPreviewMatches(preview: string, input: any, before: string): boolean {
if (typeof input?.old_string !== 'string' || !input.old_string || typeof input.new_string !== 'string' ||
(input.replace_all !== undefined && input.replace_all !== false)) return false;
const at = before.indexOf(input.old_string);
if (at < 0 || before.indexOf(input.old_string, at + input.old_string.length) !== -1) return false;
const after = before.slice(0, at) + input.new_string + before.slice(at + input.old_string.length);
const oldLines = before.split(/\r?\n/), newLines = after.split(/\r?\n/);
const firstLine = before.slice(0, at).split(/\r?\n/).length;
const oldLast = firstLine + input.old_string.split(/\r?\n/).length - 1;
const newLast = firstLine + input.new_string.split(/\r?\n/).length - 1;
const rows: Array<{line: number; kind: string; text: string; clipped?: boolean; gutter?: number}> = [];
let leading: {kind: string; text: string} | undefined;
for (const line of preview.split('\n')) {
if (!line.trim() || /^[╌─━]{3,}[ \t]*$/.test(line)) continue;
// A trimmed empty unchanged row keeps only its number.
const numbered = /^( {0,3}([1-9]\d*) )([ +\-])(.*)$/.exec(line) ?? /^( {0,3}([1-9]\d*))()[ \t]*$/.exec(line);
if (numbered) {
const kind = numbered[3] || ' ';
if (leading) {
// A wrapped first row has no coordinate. The next same-kind numbered
// row anchors its complete visible suffix to the preceding source line.
if (leading.kind !== kind || Number(numbered[2]) < 2) return false;
rows.push({line:Number(numbered[2])-1,...leading,clipped:true}); leading=undefined;
}
rows.push({line:Number(numbered[2]),kind,text:numbered[4] ?? '',gutter:numbered[1]!.length}); continue;
}
const last = rows.at(-1);
// An unchanged row wraps under its own gutter with a blank marker column.
if (last?.kind === ' ' && last.gutter && line.startsWith(' '.repeat(last.gutter + 1))) {
last.text += line.slice(last.gutter + 1); continue;
}
const wrapped = /^ {4,5}([+\-])(.*)$/.exec(line);
if (!wrapped) return false;
if (!last) {
if (leading && leading.kind !== wrapped[1]) return false;
leading={kind:wrapped[1]!,text:(leading?.text??'')+wrapped[2]!};
} else {
if (wrapped[1] !== last.kind) return false;
last.text += wrapped[2]!;
}
}
const compact = (value: string) => value.replace(/\s/g, '');
return !leading && rows.length >= 2 && rows.some(row => row.kind === '+' || row.kind === '-') &&
rows.every((row, i) => Number.isSafeInteger(row.line) && row.line > 0 &&
(!i || row.kind === '-' || rows[i-1]!.kind === '-' || row.line === rows[i-1]!.line + 1) &&
(row.kind === ' ' || (row.line >= firstLine && row.line <= (row.kind === '-' ? oldLast : newLast))) &&
row.line <= (row.kind === '-' ? oldLines : newLines).length &&
(row.clipped ? Boolean(compact(row.text)) && compact((row.kind === '-' ? oldLines : newLines)[row.line - 1]!).endsWith(compact(row.text)) :
compact(row.text) === compact((row.kind === '-' ? oldLines : newLines)[row.line - 1]!)));
}
/** A cropped in-fixture Edit binds its exact visible diff to exactly one pending
* owned request, published or witnessed by its own PreToolUse hook. Returns that ID. */
function currentEditPreview(preview: string, r: any, config: string, cwd: string, startedAt: number, file: string): string | null {
try {
if (path.dirname(r.expected) !== cwd || fs.realpathSync(cwd) !== cwd || fs.realpathSync(r.expected) !== r.expected ||
[config, path.dirname(r.transcriptPath), r.transcriptPath].some(p => fs.realpathSync(p) !== p)) return false;
[config, path.dirname(r.transcriptPath), r.transcriptPath].some(p => fs.realpathSync(p) !== p)) return null;
const pending = new Map<string, NativePublicToolEvent>(), seen = new Map<string, NativePublicToolEvent>();
const completed = new Set<string>();
let conflict = false;
@@ -331,61 +424,31 @@ function currentEditPreview(preview: string, r: any, config: string, cwd: string
completed.add(event.toolUseId); pending.delete(event.toolUseId);
}
}, r.transcriptPath);
const event = [...pending.values()][0], input = event?.input;
if (conflict || transcript.status !== 'ready' || pending.size !== 1 || event?.name !== 'Edit' ||
`${event.sessionId}:${event.toolUseId}` !== r.pendingId || !Number.isFinite(Date.parse(event.timestamp)) ||
Date.parse(event.timestamp) < startedAt || Date.parse(event.timestamp) > Date.now() ||
input?.file_path !== r.expected || typeof input.old_string !== 'string' || !input.old_string ||
typeof input.new_string !== 'string' || (input.replace_all !== undefined && input.replace_all !== false)) return false;
const bytes = boundedRegular(r.expected, MAX_WRITE_INPUT_BYTES), before = bytes.toString('utf8');
if (!Buffer.from(before).equals(bytes)) return false;
const at = before.indexOf(input.old_string);
if (at < 0 || before.indexOf(input.old_string, at + input.old_string.length) !== -1) return false;
const after = before.slice(0, at) + input.new_string + before.slice(at + input.old_string.length);
const oldLines = before.split(/\r?\n/), newLines = after.split(/\r?\n/);
const firstLine = before.slice(0, at).split(/\r?\n/).length;
const oldLast = firstLine + input.old_string.split(/\r?\n/).length - 1;
const newLast = firstLine + input.new_string.split(/\r?\n/).length - 1;
const rows: Array<{line: number; kind: string; text: string; clipped?: boolean; gutter?: number}> = [];
let leading: {kind: string; text: string} | undefined;
for (const line of preview.split('\n')) {
if (!line.trim() || /^[╌─━]{3,}[ \t]*$/.test(line)) continue;
// A trimmed empty unchanged row keeps only its number.
const numbered = /^( {0,3}([1-9]\d*) )([ +\-])(.*)$/.exec(line) ?? /^( {0,3}([1-9]\d*))()[ \t]*$/.exec(line);
if (numbered) {
const kind = numbered[3] || ' ';
if (leading) {
// A wrapped first row has no coordinate. The next same-kind numbered
// row anchors its complete visible suffix to the preceding source line.
if (leading.kind !== kind || Number(numbered[2]) < 2) return false;
rows.push({line:Number(numbered[2])-1,...leading,clipped:true}); leading=undefined;
}
rows.push({line:Number(numbered[2]),kind,text:numbered[4] ?? '',gutter:numbered[1]!.length}); continue;
}
const last = rows.at(-1);
// An unchanged row wraps under its own gutter with a blank marker column.
if (last?.kind === ' ' && last.gutter && line.startsWith(' '.repeat(last.gutter + 1))) {
last.text += line.slice(last.gutter + 1); continue;
}
const wrapped = /^ {4,5}([+\-])(.*)$/.exec(line);
if (!wrapped) return false;
if (!last) {
if (leading && leading.kind !== wrapped[1]) return false;
leading={kind:wrapped[1]!,text:(leading?.text??'')+wrapped[2]!};
} else {
if (wrapped[1] !== last.kind) return false;
last.text += wrapped[2]!;
}
if (conflict || transcript.status !== 'ready') return null;
// Any other pending file request (a Write, or an Edit elsewhere) keeps the menu waiting.
const candidates = new Map<string, {timestamp: string; input?: any}>();
for (const event of pending.values()) {
if (event.name !== 'Edit' || event.input?.file_path !== r.expected) return null;
candidates.set(`${event.sessionId}:${event.toolUseId}`, event);
}
const compact = (value: string) => value.replace(/\s/g, '');
return !leading && rows.length >= 2 && rows.some(row => row.kind === '+' || row.kind === '-') &&
rows.every((row, i) => Number.isSafeInteger(row.line) && row.line > 0 &&
(!i || row.kind === '-' || rows[i-1]!.kind === '-' || row.line === rows[i-1]!.line + 1) &&
(row.kind === ' ' || (row.line >= firstLine && row.line <= (row.kind === '-' ? oldLast : newLast))) &&
row.line <= (row.kind === '-' ? oldLines : newLines).length &&
(row.clipped ? Boolean(compact(row.text)) && compact((row.kind === '-' ? oldLines : newLines)[row.line - 1]!).endsWith(compact(row.text)) :
compact(row.text) === compact((row.kind === '-' ? oldLines : newLines)[row.line - 1]!)));
} catch { return false; }
// Claude Code can show the menu before it publishes the Edit; the owned hook input covers that delay.
const open: unknown[] = Array.isArray(r.openIds) ? r.openIds : [r.pendingId];
for (const [id, witness] of Object.entries(readPendingEditInputs(file, r))) {
if (witness?.pendingId !== id || witness.toolName !== 'Edit' || witness.sessionId !== r.sessionId || witness.cwd !== cwd ||
witness.expected !== r.expected || witness.transcriptPath !== r.transcriptPath || witness.input?.file_path !== r.expected ||
!open.includes(id) || completed.has(id.slice(r.sessionId.length + 1))) return null;
const published = candidates.get(id);
if (published && !isDeepStrictEqual(published.input, witness.input)) return null;
if (!published) candidates.set(id, witness);
}
const bytes = boundedRegular(r.expected, MAX_WRITE_INPUT_BYTES), before = bytes.toString('utf8');
if (!Buffer.from(before).equals(bytes)) return null;
const matches = [...candidates].filter(([, {timestamp, input}]) => {
const time = Date.parse(timestamp);
return Number.isFinite(time) && time >= startedAt && time <= Date.now() && editPreviewMatches(preview, input, before);
});
return matches.length === 1 && open.includes(matches[0]![0]) ? matches[0]![0] : null;
} catch { return null; }
}
/** Undefined leaves other permissions alone; null keeps this report pane waiting. */
@@ -424,8 +487,9 @@ export function currentFilePermissionEpoch(file: string | undefined, expected: s
r.completedIds.some((id: string) => id === r.pendingId || !r.seenIds.includes(id)) ||
(r.completedId === null ? r.completedIds.length !== 0 : r.completedIds.at(-1) !== r.completedId)) return null;
if(create && !currentCreatePreview(create.preview,r,config,cwd,startedAt,file)) return null;
if(edit && edit.directory === undefined && !currentEditPreview(edit.preview,r,config,cwd,startedAt)) return null;
return {pendingId:r.pendingId,completedId:r.completedId,completedIds:r.completedIds};
const editId = edit && edit.directory === undefined ? currentEditPreview(edit.preview,r,config,cwd,startedAt,file) : undefined;
if(editId === null) return null;
return {pendingId:editId ?? r.pendingId,completedId:r.completedId,completedIds:r.completedIds};
} catch { return null; }
}
test/plan-edit-cropped-permission.test.ts
+92 -1
View File
@@ -3,10 +3,11 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {createHash} from 'node:crypto';
import {createFilePermissionRecorder,currentFilePermissionEpoch,currentFilePermissionBinding} from './helpers/plan-count-file-permission';
import {createFilePermissionRecorder,currentFilePermissionEpoch,currentFilePermissionBinding,recordFilePermission,isCroppedEditPermissionVisible} from './helpers/plan-count-file-permission';
import {readPlanCountTranscript} from './helpers/plan-count-transcript';
import {createPlanCountPermissionGuard,isPermissionDialogVisible,classifyPlanCountFrame} from './helpers/claude-pty-runner';
import captured from './fixtures/plan-edit-cropped-permission-1579.json';
import editPrepublication from './fixtures/plan-ceo-floor-edit-prepublication.json';
function fixture(){
const dir=fs.mkdtempSync(path.join(os.tmpdir(),'cropped-edit-')),cwd=path.join(dir,'owned'),config=path.join(dir,'config');fs.mkdirSync(cwd);
@@ -129,3 +130,93 @@ test.each(['foreign-basename','foreign-header','quoted','fenced','auq','old-menu
expect(createPlanCountPermissionGuard()(screen,'',f.read(screen))).not.toBe('grant');
}finally{f.close();}
});
// Ship-measure plan-ceo-finding-floor baseline at 6f32737: after the owned Write was
// granted, t04 (three published Edits pending at once) and t07/t08 (Edit menu shown
// before Claude Code published the Edit) stalled on the owned working-plan Edit menu.
function edits(trial:string){
const c=editPrepublication.cases.find(c=>c.trial===trial)!;
const dir=fs.mkdtempSync(path.join(os.tmpdir(),'edit-prepub-')),cwd=path.join(dir,'owned'),config=path.join(dir,'config');fs.mkdirSync(cwd);
const expected=path.join(cwd,c.basename),sid=c.sessionId,journal=path.join(config,'projects','owned',sid+'.jsonl');
fs.mkdirSync(path.dirname(journal),{recursive:true});
const startedAt=Date.now()-1000,recorder=createFilePermissionRecorder(cwd,config,expected)!;
const native=(role:string,content:any[],session=sid)=>({cwd,sessionId:session,isSidechain:false,timestamp:new Date().toISOString(),message:{role,content}});
const hook=(kind:string,tool:string,id:string,input:any,extra={})=>recordFilePermission(JSON.stringify({hook_event_name:kind,tool_name:tool,session_id:sid,
tool_use_id:id,cwd,transcript_path:journal,tool_input:input,...extra}),recorder.file,cwd,config,expected);
const writeInput={file_path:expected,content:c.writeContent};
const base=[native('assistant',[{type:'tool_use',id:'owned-write',name:'Write',input:writeInput}]),
native('user',[{type:'tool_result',tool_use_id:'owned-write',content:'ok'}])];
hook('PreToolUse','Write','owned-write',writeInput);fs.writeFileSync(expected,c.writeContent);hook('PostToolUse','Write','owned-write',writeInput);
const write=(rows:any[]=base)=>fs.writeFileSync(journal,rows.map(r=>JSON.stringify(r)).join('\n')+'\n');write();
const pending=c.edits.map(e=>({id:e.toolUseId,input:{file_path:expected,...e.input}}));
const block=(e:{id:string;input:any})=>({type:'tool_use',id:e.id,name:'Edit',input:e.input});
const read=(screen=c.screen)=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,readPlanCountTranscript(config,cwd),screen);
return {c,dir,cwd,config,expected,sid,journal,recorder,native,hook,base,write,pending,block,read,startedAt,
close(){recorder.dispose();fs.rmSync(dir,{recursive:true,force:true});}};
}
test.each(['t07','t08'])('%s: an unpublished owned Edit is granted once from its own PreToolUse input',trial=>{
const f=edits(trial);try{
expect(f.c.recorded).toMatchObject({outcome:'permission_unanswered',transcriptStatus:'ready',publishedEdits:0});
expect(createHash('sha256').update(f.c.screen).digest('hex')).toBe(f.c.screenSha256);
expect(isCroppedEditPermissionVisible(f.c.screen)).toBe(true);
const [edit]=f.pending;expect(f.read()).toBeNull();
f.hook('PreToolUse','Edit',edit!.id,edit!.input);
expect(fs.readFileSync(f.recorder.file,'utf8')).not.toContain(edit!.input.new_string);
expect(fs.statSync(f.recorder.file+'.edit.json').mode&0o777).toBe(0o600);
const owned=currentFilePermissionBinding([{file:f.recorder.file,expected:f.expected}],f.cwd,f.config,f.startedAt,readPlanCountTranscript(f.config,f.cwd),f.c.screen);
expect(owned?.epoch.pendingId).toBe(`${f.sid}:${edit!.id}`);
const guard=createPlanCountPermissionGuard();expect(guard(f.c.screen,'',f.read())).toBe('grant');expect(guard(f.c.screen,'',f.read())).toBe('handled');
// Publication later must agree with the witness; completion retires it.
f.write([...f.base,f.native('assistant',[f.block(edit!)])]);expect(f.read()?.pendingId).toBe(`${f.sid}:${edit!.id}`);
f.write([...f.base,f.native('assistant',[f.block({...edit!,input:{...edit!.input,new_string:edit!.input.new_string+'\nhidden'}})])]);expect(f.read()).toBeNull();
f.write();f.hook('PostToolUse','Edit',edit!.id,edit!.input);
expect(fs.existsSync(f.recorder.file+'.edit.json')).toBe(false);expect(f.read()).toBeNull();
}finally{f.close();}
});
test.each(['before','after'])('t04: the menu binds to the one of three pending owned Edits its diff shows (later hooks %s the menu)',order=>{
const f=edits('t04');try{
expect(f.c.recorded).toMatchObject({outcome:'permission_unanswered',publishedEdits:3});
f.write([...f.base,f.native('assistant',f.pending.map(f.block))]);
for(const e of order==='before'?f.pending:f.pending.slice(0,1))f.hook('PreToolUse','Edit',e.id,e.input);
const [shown,...rest]=f.pending;
expect(f.read()?.pendingId).toBe(`${f.sid}:${shown!.id}`);
const guard=createPlanCountPermissionGuard();expect(guard(f.c.screen,'',f.read())).toBe('grant');expect(guard(f.c.screen,'',f.read())).toBe('handled');
// After the shown Edit lands, its stale render never binds to the other two.
const before=fs.readFileSync(f.expected,'utf8');
fs.writeFileSync(f.expected,before.replace(shown!.input.old_string,shown!.input.new_string));
f.hook('PostToolUse','Edit',shown!.id,shown!.input);
f.write([...f.base,f.native('assistant',f.pending.map(f.block)),f.native('user',[{type:'tool_result',tool_use_id:shown!.id,content:'ok'}])]);
if(order==='after')for(const e of rest)f.hook('PreToolUse','Edit',e.id,e.input);
expect(f.read()).toBeNull();
}finally{f.close();}
});
test.each(['foreign-file-published','foreign-file-hook','no-match','two-matches-witnessed','two-matches-published','unwitnessed','foreign-session','foreign-session-journal','changed-witness','sidechain'])
('owned Edit menu negative control: %s is refused',mode=>{
const f=edits(mode.includes('published')||mode==='unwitnessed'?'t04':'t07');try{
const [edit]=f.pending;let screen=f.c.screen;
if(mode==='foreign-file-published'){
f.write([...f.base,f.native('assistant',[...f.pending.map(f.block),f.block({id:'foreign-edit',input:{...edit!.input,file_path:path.join(f.dir,'foreign.md')}})])]);
for(const e of f.pending)f.hook('PreToolUse','Edit',e.id,e.input);
}
if(mode==='foreign-file-hook')f.hook('PreToolUse','Edit',edit!.id,{...edit!.input,file_path:path.join(f.dir,f.c.basename)});
if(mode==='no-match'){f.hook('PreToolUse','Edit',edit!.id,edit!.input);screen=screen.replace('Recommendation: B','Recommendation: A');}
if(mode==='two-matches-witnessed'){f.hook('PreToolUse','Edit',edit!.id,edit!.input);f.hook('PreToolUse','Edit','twin-edit',edit!.input);}
if(mode==='two-matches-published'){
f.write([...f.base,f.native('assistant',[...f.pending.map(f.block),f.block({id:'twin-edit',input:edit!.input})])]);
for(const e of f.pending)f.hook('PreToolUse','Edit',e.id,e.input);
}
if(mode==='unwitnessed'){f.write([...f.base,f.native('assistant',f.pending.map(f.block))]);for(const e of f.pending.slice(1))f.hook('PreToolUse','Edit',e.id,e.input);}
if(mode==='foreign-session'||mode==='foreign-session-journal'){
const foreign='11111111-2222-4333-8444-555555555555',journal=path.join(f.config,'projects','owned',foreign+'.jsonl');
if(mode==='foreign-session-journal')fs.writeFileSync(journal,JSON.stringify(f.native('assistant',[f.block(edit!)],foreign))+'\n');
f.hook('PreToolUse','Edit',edit!.id,edit!.input,{session_id:foreign,transcript_path:journal});
}
if(mode==='changed-witness'){f.hook('PreToolUse','Edit',edit!.id,edit!.input);fs.appendFileSync(f.recorder.file+'.edit.json',' ');}
if(mode==='sidechain')f.hook('PreToolUse','Edit',edit!.id,edit!.input,{agent_id:'helper'});
expect(f.read(screen)).toBeFalsy();
expect(createPlanCountPermissionGuard()(screen,'',f.read(screen))).not.toBe('grant');
}finally{f.close();}
});