diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b5de983..ba2f7419a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The second Opus gate (run 39161502626) went 127/132. Each of the five reds is no - **tpa-absent-linux** (the third gate, run 39166935531): Opus offered "step-by-step instructions to do it yourself" instead of saying "manual". Beside "manual", one sentence offering instructions or steps the user follows themselves now counts as the manual option. - **qa-only-no-fix:** the fixture prompt now says the 30-second guard starts only after the charter Write has completed, and that local inspection outside the listed calls uses Read/Glob, not `ls`, `wc`, `cat` or `find` (its own `ls -la` entries had read as general permission). The budget and the assertion are unchanged. - **plan-ceo-finding-floor:** Opus saved its D1 decision to the owned working plan before printing any text, as the 0D Save step requires. The permission check identified the parent session only from text or questions, so it never granted that Create and the run timed out after 600 s. A parent with only tool events is now identified by them, still as exactly one session that owns the pending Write. The floor runner also fails within 20 s with `PERMISSION_DIALOG_UNANSWERED` when a permission dialog gets no owned grant. +- **plan-ceo-finding-floor, Edit menu** (3 of 10 measured trials): after that Create, Opus edited the plan and the cropped Edit menu stayed unanswered. Twice Claude Code showed the menu before publishing the Edit, and once three Edits were pending at once, so the check found no single published Edit. The plan's own PreToolUse input now covers an unpublished Edit, as it already did for Write, and the visible diff must match exactly one pending Edit on the owned plan. An Edit to another file, a diff matching none or several pending Edits, and a foreign session are still refused. ### If you… - **…compare pass rates across the switch:** Fable trials carry `harness_version: 5` and Opus trials `6`; they are separate series. diff --git a/scripts/typecheck-test-baseline.json b/scripts/typecheck-test-baseline.json index 215ee7eb1..4cfacfe82 100644 --- a/scripts/typecheck-test-baseline.json +++ b/scripts/typecheck-test-baseline.json @@ -226,7 +226,6 @@ "test/helpers/hermetic-env.test.ts\tTS7053\tElement implicitly has an 'any' type because expression of type 'string' can't be used to index type '{ ... 6 more ...; EVALS_SELECTION_JSON: string; GITHUB_APP_PRIVATE_KEY: string; GITHUB_CLIENT_SECRET: string; GITHUB_PAT: string; GITHUB_PERSONAL_ACCESS_TOKEN: string; GITHUB_TOKEN: string; NODE_ENV?: string | undefined; TZ?: string | undefined; }'. No index signature with a parameter of type 'string' was found on type '{ ... 6 more ...; EVALS_SELECTION_JSON: string; GITHUB_APP_PRIVATE_KEY: string; GITHUB_CLIENT_SECRET: string; GITHUB_PAT: string; GITHUB_PERSONAL_ACCESS_TOKEN: string; GITHUB_TOKEN: string; NODE_ENV?: string | undefined; TZ?: string | undefined; }'.": 1, "test/helpers/office-hours-completion.ts\tTS18047\t'disposition' is possibly 'null'.": 2, "test/helpers/plan-count-file-permission.ts\tTS18048\t'event.input' is possibly 'undefined'.": 3, - "test/helpers/plan-count-file-permission.ts\tTS18048\t'input' is possibly 'undefined'.": 12, "test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'questions' does not exist on type 'AskUserQuestionFingerprint'.": 4, "test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'selectedOptions' does not exist on type 'AskUserQuestionFingerprint'.": 4, "test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'toolUseId' does not exist on type 'AskUserQuestionFingerprint'.": 8, diff --git a/test/fixtures/plan-ceo-floor-edit-prepublication.json b/test/fixtures/plan-ceo-floor-edit-prepublication.json new file mode 100644 index 000000000..ecdf9c197 --- /dev/null +++ b/test/fixtures/plan-ceo-floor-edit-prepublication.json @@ -0,0 +1,115 @@ +{ + "source": "Ship-measure round baseline at 6f32737 (local, diagnostic): plan-ceo-finding-floor reds t04, t07, t08 ended permission_unanswered on the owned working-plan Edit menu.", + "note": "t04 Edit inputs are the published native inputs. t07/t08 Edits were never published before the deadline; their inputs are reconstructed from the captured diff rows (every removed row matched the captured Write bytes exactly) and their tool IDs are placeholders.", + "cases": [ + { + "trial": "t04", + "source": "ship-measure plan-ceo-finding-floor baseline t04: t04/eval/t1/shards/skill-e2e-plan-ceo-finding-floor/pty-count/local-measure-plan-ceo-finding-floor-baseline-t4-39512/plan-ceo-review-1792361226070-y2Y3JE", + "sessionId": "b4f2ae78-2b26-4287-934b-f7b48e95e3c5", + "basename": "gstack-test-plan-ceo-floor.md", + "recorded": { + "outcome": "permission_unanswered", + "transcriptStatus": "ready", + "transcriptCalls": 0, + "transcriptAssistantMessages": 1, + "pendingWriteInputs": [], + "publishedEdits": 3 + }, + "screen": " 57 ```text\n 58 -Commitment | Source/approval or pending | Current | A | B \n - | C \n 59 -Developer tier is the product | user: preserve scope | yes | yes | yes, gated on \n -evidence | yes, as experiment arm \n 60 -Talk to developers first | pending | no | no | yes, 8-12 inte\n -rviews | no \n 61 -Baseline funnel numbers | pending | no | no | yes \n - | yes \n 62 -Concrete success metric | pending | \"more signups\" | \"more signups\" | activation + p\n -aid conv. | activation + paid conv. \n 63 -Price change reversible | pending | no | no | decided after \n -evidence | yes, cohort-limited \n 64 -Time before tier ships | pending | now | now | +~2 weeks \n - | +~4-6 weeks \n 58 +Commitment | Source/approval or pending | Current | A (validate) | B (as\n + written) | C (experiment) \n 59 +Developer tier is the product | user: preserve scope | yes | yes, gated on evidence | yes \n + | yes, as experiment arm \n 60 +Talk to developers first | pending | no | yes, 8-12 interviews | no \n + | no \n 61 +Baseline funnel numbers | pending | no | yes | no \n + | yes \n 62 +Concrete success metric | pending | \"more signups\" | activation + paid conv. | \"more\n + signups\" | activation + paid conv. \n 63 +Price change reversible | pending | no | decided after evidence | no \n + | yes, cohort-limited \n 64 +Time before tier ships | pending | now | +~2 weeks | now \n + | +~4-6 weeks \n 65 ```\n 66 \n 67 Question: D1 — P1: Do we ship the cheaper tier on gut feel, or validate that price is the barrier first?\n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n Do you want to make this edit to gstack-test-plan-ceo-floor.md?\n ❯ 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel · Tab to amend\n", + "screenSha256": "de5a50a2af7bf2eb8ee5de4663198882ca7a9bc84c93c57934d00d34b3e3f724", + "writeContent": "# CEO Review: Launch a \"developer-friendly\" pricing tier\n\nPlan under review: `PLAN.md` (commit 897aa82). Reviewer: /plan-ceo-review, 2026-10-18.\nReview depth: strategy-only (the plan has no implementation design; there is no code in the repo).\n\n## Context\nThe plan proposes a cheaper \"developer-friendly\" tier to increase developer adoption, measured by\n\"more signups\". Its own premise section says no developer has been asked whether price is a barrier;\nthe team's belief is that it \"feels like\" it should be cheaper (PLAN.md:9-11).\n\n## Pre-review system audit\n- Repo: one commit (`897aa82 Seed review plan`), files `CLAUDE.md`, `PLAN.md`. No remote, no stash, no TODOS.md, no TODO/FIXME markers.\n- No design doc. /office-hours offer skipped per user instruction.\n- Prior learnings: none. Branch decisions: none. Brain context: cold.\n- UI scope: none named in the plan. Section 11 likely skipped.\n- Landscape (WebSearch; Aside unavailable): dev-tool pricing has converged on ~$20/mo standard with thinning free tiers; ~74% of suppliers use usage-based models; developers dislike billing unpredictability more than list price (sources: costbench.com/software/developer-tools, agentdeals.dev/ai-coding-pricing-2026, nexdigm.com developer-platform-pricing-benchmarking).\n - Layer 1: dev adoption is usually won by a frictionless free/self-serve path, time-to-first-value and predictable billing, not by a lower sticker price.\n - Layer 3: if signups are low because of onboarding friction, docs, or the free tier's shape, a cheaper paid tier does nothing and cuts revenue from developers who would have paid.\n\n## 0A. Premise challenge\n- Stated problem: low developer adoption. Real problem: unknown. The plan has not established *why* developers don't adopt.\n- Proxy risk: \"price is the barrier\" is an untested hypothesis (PLAN.md:10-11). The plan solves a proxy.\n- Success metric \"more signups\" (PLAN.md:6) has no baseline, target, window, or segment, and signups are not adoption (activation/retention). Any change will \"succeed\" or \"fail\" by noise.\n- Do-nothing cost: unknown without baseline funnel numbers. Cost of being wrong: permanent price anchoring (price cuts are hard to reverse), cannibalized revenue from existing paying devs, and the real barrier stays unfixed.\n\n## 0B. Existing leverage\nNo code in the repo. Reusable assets the plan should name: existing signup/activation funnel analytics, current customer list (for interviews), billing system's plan/coupon support (for a reversible price test).\n\nREALITY: premise-table finding unverified=2 (\"price is a barrier\", \"cheaper tier increases adoption\"); 0 receipts PLAN.md:9\nREALITY: already-done pass single seed commit, no prior pricing work in history PLAN.md:1\nREALITY: surface-check pass plan names no CLI commands, flags or routes PLAN.md:1\nREALITY: binding-decisions pass gstack-decision-search returned none; plan has no Binding decisions section PLAN.md:1\nREALITY: numbers finding no numbers at all; \"more signups\" has no baseline or target [re-measure before gate] PLAN.md:6\nREALITY: deferred-asks pass no linked issue PLAN.md:1\nREALITY: pr-count pass no code deliverable; no ship-policy.json, default one PR PLAN.md:1\n\n## 0C. Dream state\n```\n CURRENT STATE THIS PLAN 12-MONTH IDEAL\n Adoption \"low\" (unmeasured); Ship a cheaper tier on gut Known #1 adoption barrier, fixed;\n no dev interviews; no baseline feel; measure \"more signups\" activation + paid conversion\n funnel ---> (no baseline) ---> measured weekly; pricing set from\n evidence, reversible experiments\n```\nAs written the plan moves sideways: it may raise signups while lowering revenue, and teaches nothing either way.\n\n## Decision ledger\n\n| ID and owner | Contract and evidence | Current | Proposed | Status | Exact approval and scope |\n|---|---|---|---|---|---|\n| MODE (user) | Review mode | HOLD SCOPE | n/a | approved | User instruction: \"For review-mode questions choose HOLD SCOPE (CEO)\" |\n| P1 (plan owner) | Premise/approach: is price the barrier? PLAN.md:9-11 | Ship cheaper tier on team intuition | A) as written; B) validate first (interviews + funnel baseline), tier conditional; C) reversible priced experiment | unresolved | pending D1 |\n\n## currentDecision (P1)\nCommitment comparison:\n\n```text\nCommitment | Source/approval or pending | Current | A | B | C\nDeveloper tier is the product | user: preserve scope | yes | yes | yes, gated on evidence | yes, as experiment arm\nTalk to developers first | pending | no | no | yes, 8-12 interviews | no\nBaseline funnel numbers | pending | no | no | yes | yes\nConcrete success metric | pending | \"more signups\" | \"more signups\" | activation + paid conv. | activation + paid conv.\nPrice change reversible | pending | no | no | decided after evidence | yes, cohort-limited\nTime before tier ships | pending | now | now | +~2 weeks | +~4-6 weeks\n```\n\nQuestion: D1 — P1: Do we ship the cheaper tier on gut feel, or validate that price is the barrier first?\nProject: PLAN.md on main, a strategy plan for a \"developer-friendly\" pricing tier.\nELI10: The team wants to make a cheaper plan because it feels like developers find us too expensive, but nobody has asked a developer. If price isn't the real reason developers don't sign up, a cheaper plan just earns less money and the real problem stays. Checking first costs about two weeks.\nStakes if we pick wrong: we permanently anchor a lower price, lose revenue from devs who'd have paid, and still don't fix why developers aren't adopting.\nRecommendation: B because it keeps the developer tier in scope but makes it conditional on a cheap test of the one claim the whole plan rests on.\nNote: options differ in kind, not coverage — no completeness score.\nNet: two weeks of interviews and funnel data versus shipping a hard-to-reverse price cut blind.\nHeader: Premise\nA) Validate first (recommended)\nRun 8-12 developer interviews plus a signup/activation funnel baseline, then build the tier only if price shows up as a top barrier. Effort S (human: ~2 weeks / CC: ~1 hour to draft script and metrics), risk low. ✅ Tests the single unverified claim the entire plan depends on before money moves. ✅ Produces a real baseline so \"success\" becomes measurable instead of noise. ❌ Delays any tier launch by roughly two weeks even if price turns out to be the barrier.\nB) Ship as written\nLaunch the cheaper tier now on the team's intuition, measuring \"more signups\". Effort M (human: ~1-2 weeks / CC: ~1 day billing/config), risk high. ✅ Fastest path to something in market; zero research overhead for the team. ✅ If intuition is right, adoption gains start roughly two weeks sooner. ❌ Price cuts are hard to reverse, and with no baseline you cannot tell if it worked.\nC) Priced experiment\nOffer the cheaper tier to a limited cohort (A/B on pricing page) against current pricing, with baseline and activation metrics. Effort M (human: ~4-6 weeks / CC: ~1 day setup), risk medium. ✅ Measures causal effect of price on signups and paid conversion directly. ✅ Reversible: only a cohort sees the lower price. ❌ Needs enough traffic for significance and still skips learning why devs don't adopt.\n", + "edits": [ + { + "toolUseId": "toolu_01ERak43Ep5qr7euvXoKxHe5", + "timestamp": "2026-10-18T22:08:15.794Z", + "published": true, + "input": { + "old_string": "```text\nCommitment | Source/approval or pending | Current | A | B | C\nDeveloper tier is the product | user: preserve scope | yes | yes | yes, gated on evidence | yes, as experiment arm\nTalk to developers first | pending | no | no | yes, 8-12 interviews | no\nBaseline funnel numbers | pending | no | no | yes | yes\nConcrete success metric | pending | \"more signups\" | \"more signups\" | activation + paid conv. | activation + paid conv.\nPrice change reversible | pending | no | no | decided after evidence | yes, cohort-limited\nTime before tier ships | pending | now | now | +~2 weeks | +~4-6 weeks\n```", + "new_string": "```text\nCommitment | Source/approval or pending | Current | A (validate) | B (as written) | C (experiment)\nDeveloper tier is the product | user: preserve scope | yes | yes, gated on evidence | yes | yes, as experiment arm\nTalk to developers first | pending | no | yes, 8-12 interviews | no | no\nBaseline funnel numbers | pending | no | yes | no | yes\nConcrete success metric | pending | \"more signups\" | activation + paid conv. | \"more signups\" | activation + paid conv.\nPrice change reversible | pending | no | decided after evidence | no | yes, cohort-limited\nTime before tier ships | pending | now | +~2 weeks | now | +~4-6 weeks\n```", + "replace_all": false + } + }, + { + "toolUseId": "toolu_01XC8JDRFNTi4im7Fa8HbgKh", + "timestamp": "2026-10-18T22:08:17.443Z", + "published": true, + "input": { + "old_string": "A) as written; B) validate first (interviews + funnel baseline), tier conditional; C) reversible priced experiment", + "new_string": "A) validate first (interviews + funnel baseline), tier conditional; B) ship as written; C) reversible priced experiment", + "replace_all": false + } + }, + { + "toolUseId": "toolu_01H3Q3ia3Zr5F4ssU6p1Ym1X", + "timestamp": "2026-10-18T22:08:18.689Z", + "published": true, + "input": { + "old_string": "Recommendation: B because", + "new_string": "Recommendation: A because", + "replace_all": false + } + } + ] + }, + { + "trial": "t07", + "source": "ship-measure plan-ceo-finding-floor baseline t07: t07/eval/t1/shards/skill-e2e-plan-ceo-finding-floor/pty-count/local-measure-plan-ceo-finding-floor-baseline-t7-39512/plan-ceo-review-1792361461841-1z5Jvz", + "sessionId": "22c9e376-7347-48da-90f2-2624d98e8d4c", + "basename": "gstack-test-plan-ceo-floor.md", + "recorded": { + "outcome": "permission_unanswered", + "transcriptStatus": "ready", + "transcriptCalls": 0, + "transcriptAssistantMessages": 1, + "pendingWriteInputs": [], + "publishedEdits": 0 + }, + "screen": " 72 Question: D1 — P1: The plan's premise (price is the barrier) has zero developer evidence. How should the plan treat\n it?\n 73 +Project/branch/task: CEO review of PLAN.md on main, a cheaper \"developer-friendly\" pricing tier. \n 74 +ELI10: The team wants to make a cheaper plan for developers because it feels right, but nobody asked developers if \n +price is what stops them. If price isn't the real blocker, we give away revenue and signups barely move, and price \n +cuts are hard to take back publicly. \n 75 +Stakes if we pick wrong: we cut price, signups stay flat, revenue per user drops, and we still don't know why devs \n +leave. \n 76 +Recommendation: B because it keeps the tier in scope but spends ~2 weeks of interviews and funnel data before an ir\n +reversible public price change. \n 77 +Note: options differ in kind, not coverage — no completeness score. \n 78 +Net: speed of launching on a hunch vs. two weeks to know whether price is the lever at all. \n 79 Header: Premise\n 80 A) Ship tier as written\n 75 -Keep the plan: launch a cheaper developer tier now on team intuition. Effort S, risk high, no verification of the p\n -remise. \n 81 +Launch a cheaper dev tier now on team intuition. Effort S, risk high, zero premise verification. ✅ Fastest path, t\n +he team can ship it within days of deciding a price. ✅ No research overhead or extra people needed before launch. \n +❌ Public price cuts are hard to reverse if signups do not move at all. \n 82 B) Validate, then ship tier (recommended)\n 77 -Keep the tier in scope but gate its price/shape on 8-12 dev interviews plus a funnel baseline, and replace \"more si\n -gnups\" with a measured target. Effort M, risk low. \n 83 +Keep the tier, but set its price and shape from 8-12 dev interviews plus a funnel baseline; replace \"more signups\" \n +with a measured target. Effort M, risk low. ✅ Tier still ships, now aimed at the step where devs actually drop off\n +. ✅ Gives a real baseline so the launch can be judged a win or loss. ❌ Delays launch roughly two weeks (human: ~2\n + wks / CC: ~1 day of synthesis). \n 84 C) Reframe to adoption friction\n 79 -Treat pricing as one lever in a broader developer-activation program (onboarding, docs, free usage). Effort L, risk\n - medium; widens scope beyond the plan. \n 85 +Treat price as one lever in a broader dev-activation program (onboarding, docs, free usage). Effort L, risk medium.\n + ✅ Attacks adoption directly instead of a single assumed cause. ✅ Likely the larger long-term win if friction, no\n +t price, is the blocker. ❌ Widens scope beyond the supplied plan, which the user asked to preserve. \n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n Do you want to make this edit to gstack-test-plan-ceo-floor.md?\n ❯ 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel · Tab to amend\n", + "screenSha256": "6571c111d303107f9eaf63604179077845fa0156b177f2da66dccfa77233f68f", + "writeContent": "# CEO Review Working Plan: \"Developer-friendly\" pricing tier\n\nSource under review: `PLAN.md` (lines 19-29). Review: /plan-ceo-review, 2026-10-18, branch `main`.\nReview depth: strategy-only (plan is product/pricing strategy; no implementation design requested, no code in repo).\n\n## Plan under review (verbatim intent)\n- Goal: Increase developer adoption. (PLAN.md:22)\n- Success metric: More signups. (PLAN.md:25)\n- Premise: No developers consulted; team \"feels like\" pricing should be cheaper. (PLAN.md:28-29)\n\n## Settled by user instruction (not review findings)\n- /office-hours prerequisite: skipped (CLAUDE.md:9).\n- Routing setup, cross-project recall, outside reviewers: declined (CLAUDE.md:11).\n- Review mode: HOLD SCOPE (CLAUDE.md:13). Product scope preserved.\n- Explicitly NOT approved: premise, approach, remedies (CLAUDE.md:15).\n\n## Pre-review system audit\n- Git: single commit `95e8648 Seed review plan`; no stashes; no diff vs `main`; no remote.\n- Files: CLAUDE.md, PLAN.md only. No TODOS.md, no architecture docs, no design doc, no pricing code, no billing integration, no analytics.\n- Prior learnings: none. Prior decisions: none. Brain context: cold.\n- UI scope: plan implies a pricing page / checkout change but names none. DESIGN_SCOPE: unknown, pending approach.\n\n## Landscape check (WebSearch; Aside unavailable)\n- [Layer 1] Dev tools grow bottom-up: devs self-validate on a free/usage tier before advocating a paid team plan. Price is rarely the first barrier; activation friction is.\n- [Layer 2] 2026 guidance: gate by usage intensity (API calls, repos, seats), not raw price; the pricing page is a trust signal and devs punish opaque pricing. Sources: getmonetizely.com dev-tool pricing articles (vendor content, treat as weak evidence).\n- [Layer 3] First principles: \"cheaper\" only helps if price is the binding constraint at the step where devs drop out. Without funnel data, a cut price can lower revenue per user while signups stay flat, and price cuts are hard to reverse publicly.\n\n## 0A. Premise challenge (finding, pending decision)\n- Real problem: unknown. \"Developer adoption\" is not measured; the plan does not say where devs drop (discover, sign up, activate, convert, retain).\n- Plan solves a proxy: price is assumed to be the barrier with zero evidence (PLAN.md:28-29).\n- Success metric \"more signups\" (PLAN.md:25) has no baseline, target, window or quality bar. A cheaper tier can inflate signups while activation and revenue fall.\n- Do-nothing cost: unknown, because current adoption numbers are not stated.\n\n## 0B. Existing leverage\n- No code in repo to reuse. Anything about billing, plans, pricing page, analytics: implementation owner must prove what exists.\n\nREALITY: premise-table finding unverified=3 (price-is-barrier, devs-want-cheaper, signups-track-adoption) PLAN.md:28\nREALITY: already-done pass no remote/base history beyond seed commit; nothing shipped PLAN.md:19\nREALITY: surface-check n/a plan names no CLI, flag or route PLAN.md:19\nREALITY: binding-decisions pass none recorded; CLAUDE.md scope instructions honored CLAUDE.md:13\nREALITY: numbers finding plan has zero numbers: no price, baseline, target or window PLAN.md:25\nREALITY: deferred-asks pass no linked issue PLAN.md:19\nREALITY: pr-count pass no ship-policy; strategy-only, no PRs planned PLAN.md:19\n\n## 0C. Dream state\n```\n CURRENT STATE THIS PLAN 12-MONTH IDEAL\n Unknown dev adoption, ---> Cheaper tier on a hunch, ---> Measured dev funnel; pricing\n no funnel data, no dev success = \"more signups\" tied to the real drop-off step;\n interviews (unmeasured) devs self-serve to paid teams\n```\nAs written, the plan moves sideways: it changes price without knowing whether price is the constraint.\n\n## Decision ledger\n\n| ID and owner | Contract and evidence | Current | Proposed | Status | Exact approval and scope |\n|---|---|---|---|---|---|\n| MODE (user) | CLAUDE.md:13 | HOLD SCOPE | - | approved | CLAUDE.md:13 \"choose HOLD SCOPE (CEO)\" |\n| P1 premise/approach (plan owner) | PLAN.md:28-29 no dev evidence | Launch cheaper tier on team intuition | A) as written; B) validate premise first, tier gated on evidence; C) reframe to adoption-friction program | unresolved | pending D1 |\n\n## currentDecision (P1)\nCommitment comparison:\n\nCommitment | Source/approval or pending | Current | A | B | C\n---|---|---|---|---|---\nShip a dev-friendly pricing tier | PLAN.md:19, scope preserved | yes | yes, now | yes, shaped by evidence | maybe, one lever among several\nTalk to developers before pricing | pending | no | no | yes, 8-12 interviews | yes\nFunnel baseline before launch | pending | no | no | yes | yes\nSuccess metric | pending | \"more signups\" | unchanged | activation-qualified signups w/ baseline + target | activation + paid conversion\nEffort | pending | - | S | M | L\n\nQuestion: D1 — P1: The plan's premise (price is the barrier) has zero developer evidence. How should the plan treat it?\nHeader: Premise\nA) Ship tier as written\nKeep the plan: launch a cheaper developer tier now on team intuition. Effort S, risk high, no verification of the premise.\nB) Validate, then ship tier (recommended)\nKeep the tier in scope but gate its price/shape on 8-12 dev interviews plus a funnel baseline, and replace \"more signups\" with a measured target. Effort M, risk low.\nC) Reframe to adoption friction\nTreat pricing as one lever in a broader developer-activation program (onboarding, docs, free usage). Effort L, risk medium; widens scope beyond the plan.\n", + "edits": [ + { + "toolUseId": "toolu_t07_unpublished_edit", + "timestamp": null, + "published": false, + "reconstructed": true, + "input": { + "old_string": "Header: Premise\nA) Ship tier as written\nKeep the plan: launch a cheaper developer tier now on team intuition. Effort S, risk high, no verification of the premise.\nB) Validate, then ship tier (recommended)\nKeep the tier in scope but gate its price/shape on 8-12 dev interviews plus a funnel baseline, and replace \"more signups\" with a measured target. Effort M, risk low.\nC) Reframe to adoption friction\nTreat pricing as one lever in a broader developer-activation program (onboarding, docs, free usage). Effort L, risk medium; widens scope beyond the plan.", + "new_string": "Project/branch/task: CEO review of PLAN.md on main, a cheaper \"developer-friendly\" pricing tier.\nELI10: The team wants to make a cheaper plan for developers because it feels right, but nobody asked developers if price is what stops them. If price isn't the real blocker, we give away revenue and signups barely move, and price cuts are hard to take back publicly.\nStakes if we pick wrong: we cut price, signups stay flat, revenue per user drops, and we still don't know why devs leave.\nRecommendation: B because it keeps the tier in scope but spends ~2 weeks of interviews and funnel data before an irreversible public price change.\nNote: options differ in kind, not coverage — no completeness score.\nNet: speed of launching on a hunch vs. two weeks to know whether price is the lever at all.\nHeader: Premise\nA) Ship tier as written\nLaunch a cheaper dev tier now on team intuition. Effort S, risk high, zero premise verification. ✅ Fastest path, the team can ship it within days of deciding a price. ✅ No research overhead or extra people needed before launch. ❌ Public price cuts are hard to reverse if signups do not move at all.\nB) Validate, then ship tier (recommended)\nKeep the tier, but set its price and shape from 8-12 dev interviews plus a funnel baseline; replace \"more signups\" with a measured target. Effort M, risk low. ✅ Tier still ships, now aimed at the step where devs actually drop off. ✅ Gives a real baseline so the launch can be judged a win or loss. ❌ Delays launch roughly two weeks (human: ~2 wks / CC: ~1 day of synthesis).\nC) Reframe to adoption friction\nTreat price as one lever in a broader dev-activation program (onboarding, docs, free usage). Effort L, risk medium. ✅ Attacks adoption directly instead of a single assumed cause. ✅ Likely the larger long-term win if friction, not price, is the blocker. ❌ Widens scope beyond the supplied plan, which the user asked to preserve.", + "replace_all": false + } + } + ] + }, + { + "trial": "t08", + "source": "ship-measure plan-ceo-finding-floor baseline t08: t08/eval/t1/shards/skill-e2e-plan-ceo-finding-floor/pty-count/local-measure-plan-ceo-finding-floor-baseline-t8-39512/plan-ceo-review-1792361461901-BWluIb", + "sessionId": "77d807e3-5777-405c-9b05-143b4169bae6", + "basename": "gstack-test-plan-ceo-floor.md", + "recorded": { + "outcome": "permission_unanswered", + "transcriptStatus": "ready", + "transcriptCalls": 0, + "transcriptAssistantMessages": 1, + "pendingWriteInputs": [], + "publishedEdits": 0 + }, + "screen": " 67 -Talk to developers first | pending | no | no | yes, 10-15\n - interviews + funnel read | no (data from experiment instead) \n 68 -Funnel instrumentation | pending | unknown | not required | read exist\n -ing funnel, add gaps | required (A/B split) \n 69 -Price-point decision basis | pending | team intuition | team intuition | interview \n -+ funnel evidence | experiment result \n 70 -Reversibility | pending | low | low | high (noth\n -ing shipped until gate) | medium (exposed cohort keeps price) \n 71 -Time to tier in market | pending | fastest | fastest | +1-2 weeks\n - | +1 week setup, 2-4 week run \n 72 -Success metric (D2) | pending, separate row | \"More signups\" | unchanged, pending | unchanged,\n - pending | unchanged, pending \n 65 +Commitment | Source/approval or pending | Current | A (discovery gate) \n + | B (experiment) | C (as written) \n 66 +Developer-friendly tier exists | PLAN.md scope (preserved) | planned | yes, gated on discovery evidenc\n +e | yes, as experiment arm | yes \n 67 +Talk to developers first | pending | no | yes, 10-15 interviews + funnel \n +read | no (data from experiment instead) | no \n 68 +Funnel instrumentation | pending | unknown | read existing funnel, add gaps \n + | required (randomized split) | not required \n 69 +Price-point decision basis | pending | team intuition | interview + funnel evidence \n + | experiment result | team intuition \n 70 +Reversibility | pending | low | high (nothing shipped until gat\n +e) | medium (exposed cohort keeps price) | low \n 71 +Time to tier in market | pending | fastest | +1-2 weeks \n + | +1 week setup, 2-4 week run | fastest \n 72 +Success metric (D2) | pending, separate row | \"More signups\" | unchanged, pending \n + | unchanged, pending | unchanged, pending \n 73 ```\n 74 \n 75 Question: D1 — D1-PREMISE: Should the pricing tier ship on team intuition, or after testing whether price is actual\n ly the barrier? \n╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌\n Do you want to make this edit to gstack-test-plan-ceo-floor.md?\n ❯ 1. Yes\n 2. Yes, and switch to accept edits (auto-approve file edits and common file commands) for this session (shift+tab)\n 3. No\n\n Esc to cancel · Tab to amend\n", + "screenSha256": "13dd1c4e9c135a5e66b511ee9eb3421e7c4a1797e883f88af3eaa0876d72ef4f", + "writeContent": "# CEO Review Working Plan: \"Developer-friendly\" pricing tier\n\nSource under review: `PLAN.md` (commit 47a67a6). Skill: /plan-ceo-review. Branch: main.\nReview depth: strategy-only (plan contains no implementation design; user asked for a plan review, not implementation).\nMode: HOLD SCOPE (pre-answered in CLAUDE.md: \"For review-mode questions choose HOLD SCOPE (CEO)\"). Mode handoff runs after premise/approach decisions.\n\n## Pre-review system audit\n- `git log`: one commit, `47a67a6 Seed review plan`. No stashes, no remote, no base diff.\n- No TODOS.md, no architecture docs, no code. Plan is pure product/pricing strategy.\n- No /office-hours design doc; offer skipped per CLAUDE.md instruction.\n- Prior learnings: 0. Cross-project recall: false (pre-declined). Brain digests: none. Decision log: empty.\n- DESIGN_SCOPE: no UI in plan as written (pricing page changes would be UI, but not specified). Section 11 likely skipped.\n- Landscape (WebSearch; Aside not installed):\n - [L1] Tried-and-true: generous free tier for individual adoption, paid tiers gate team/collab/enterprise features.\n - [L2] 2026 data: 88% of tracked dev tools already have free tiers; Q1 2026 saw net tightening (173 negative vs 32 positive changes). Upgrade drivers skew to features (39%) and usage (28%); barriers are budget (46%) and ROI uncertainty (35%).\n - [L3] First principles: if 88% of competitors already have a free tier, \"cheaper\" is table stakes, not differentiation. If the real barrier is activation friction or unclear value, a price cut lowers revenue and moves no signups.\n\n## Step 0A: Premise challenge\n- **Stated problem:** low developer adoption. **Proposed fix:** a cheaper tier.\n- **Evidence for the causal link:** none. PLAN.md:\"Premise\" says verbatim: \"We haven't talked to any developers about whether the current pricing is actually a barrier. The team agreed it 'feels like' it should be cheaper.\"\n- **Proxy vs. direct:** price is a guessed proxy for the real pain. Unknown whether devs drop at the pricing page, at signup, at first-run/activation, or never arrive (awareness).\n- **Success metric \"More signups\":** no baseline, no target, no time window, no segment (\"developer\" undefined), no guardrail (revenue/ARPU cannibalization, paid-to-cheap downgrades). Signups is also a vanity metric if activation is the bottleneck.\n- **Do-nothing cost:** unknown, because current adoption numbers are not cited.\n- **Cost of being wrong:** pricing is a semi-one-way door. Cutting price is easy; raising it back later angers users and costs trust. Existing paying customers may downgrade.\n\n## Step 0B: Existing leverage\nNo code in repo. Reusable assets to confirm (owner must prove): current pricing page analytics, signup funnel events, existing customer list for interviews, billing system's ability to add a plan/coupon without migration.\n\n### Reality rows\n| premise | claim | receipt | verdict |\n|---|---|---|---|\n| P1 | Current pricing is a barrier to developers | PLAN.md:13-14 (admits no evidence) | unverified |\n| P2 | A cheaper tier increases adoption | PLAN.md:4-5 | unverified |\n| P3 | Signups measure adoption | PLAN.md:7-8 | unverified |\nunverified=3\n\nREALITY: premise-table finding 3 of 3 premises unverified; plan admits no developer conversations PLAN.md:13\nREALITY: already-done pass single-commit repo, no prior pricing work, no remote to fetch PLAN.md:1\nREALITY: surface-check n/a-equivalent pass no CLI commands, flags or routes named in plan PLAN.md:1\nREALITY: binding-decisions pass decision log empty; no `## Binding decisions` section PLAN.md:1\nREALITY: numbers finding plan contains zero numbers: no baseline, target, price point or window PLAN.md:7-8\nREALITY: deferred-asks pass no linked issue PLAN.md:1\nREALITY: pr-count pass no .gstack/ship-policy.json; default one PR per wave; no code planned PLAN.md:1\n\n## Step 0C: Dream state\n```\n CURRENT STATE THIS PLAN 12-MONTH IDEAL\n Unknown adoption baseline. --> Cheaper tier shipped on --> Devs adopt because time-to-first-value\n No dev interviews. team intuition. Signups is minutes; pricing matches how they\n Price assumed to be the may tick up or not; no grow (free solo -> paid team); funnel\n barrier. way to tell why. instrumented; price set from evidence.\n```\nAs written, the plan moves toward the ideal only if price is in fact the barrier. Nothing in the plan can confirm or refute that.\n\n## Decision ledger\n| ID and owner | Contract and evidence | Current | Proposed | Status | Exact approval and scope |\n|---|---|---|---|---|---|\n| MODE (admin) | CLAUDE.md \"choose HOLD SCOPE (CEO)\" | HOLD SCOPE | none | approved | CLAUDE.md instruction; review mode only, approves no plan changes |\n| D1-PREMISE (owner: plan author / PM) | PLAN.md:13-14 admits pricing-as-barrier is untested | A) launch tier on team intuition | B) discovery gate before tier ships; C) ship tier as instrumented reversible experiment | unresolved | pending D1 |\n| D2-METRIC (owner: plan author / PM) | PLAN.md:7-8 \"More signups\" has no baseline/target/guardrail | \"More signups\" | define baseline, target, window, activation metric, revenue guardrail | unresolved | pending; asked after D1 |\n\n## currentDecision (D1-PREMISE)\nCommitment comparison:\n```text\nCommitment | Source/approval or pending | Current | A | B | C\nDeveloper-friendly tier exists | PLAN.md scope (preserved) | yes | yes | yes, gated on discovery evidence | yes, as experiment arm\nTalk to developers first | pending | no | no | yes, 10-15 interviews + funnel read | no (data from experiment instead)\nFunnel instrumentation | pending | unknown | not required | read existing funnel, add gaps | required (A/B split)\nPrice-point decision basis | pending | team intuition | team intuition | interview + funnel evidence | experiment result\nReversibility | pending | low | low | high (nothing shipped until gate) | medium (exposed cohort keeps price)\nTime to tier in market | pending | fastest | fastest | +1-2 weeks | +1 week setup, 2-4 week run\nSuccess metric (D2) | pending, separate row | \"More signups\" | unchanged, pending | unchanged, pending | unchanged, pending\n```\n\nQuestion: D1 — D1-PREMISE: Should the pricing tier ship on team intuition, or after testing whether price is actually the barrier?\nProject/branch/task: Reviewing PLAN.md on main, a plan to launch a cheaper developer pricing tier to grow adoption.\nELI10: The plan says \"devs aren't adopting, so make it cheaper,\" but admits nobody asked a developer if price is the problem. If devs actually leave because setup is confusing or the value is unclear, a cheaper price loses money and fixes nothing. Price cuts are also hard to undo without angering users.\nStakes if we pick wrong: we cut revenue and lock in a price we can't easily raise, while the real adoption blocker stays untouched and signups don't move.\nRecommendation: B because it keeps the tier in scope but spends 1-2 weeks learning whether price is the barrier before making a hard-to-reverse pricing move; 88% of dev tools already have free tiers, so cheaper alone is unlikely to differentiate.\nNote: options differ in kind, not coverage — no completeness score.\nNet: trading 1-2 weeks of speed for confidence that the pricing change targets the real problem.\nHeader: Premise\nA) Discovery gate first (recommended)\nEffort S (human: ~1-2 weeks / CC: ~1 hr to draft script + funnel query), low risk. Before building the tier, interview 10-15 target developers and read the existing signup/activation funnel; ship the tier only if price shows up as a top barrier. ✅ Tests the one unverified premise before a hard-to-reverse pricing move is made. ✅ Findings also reveal non-price blockers (setup, docs, value) the team can fix directly. ❌ Delays the tier by 1-2 weeks and may conclude the tier isn't the right fix.\nB) Ship as instrumented experiment\nEffort M (human: ~1 week setup + 2-4 week run / CC: ~2 hrs), medium risk. Launch the cheaper tier to a random share of new visitors with a holdout, and decide on full rollout from measured signup and activation lift. ✅ Produces causal evidence on price sensitivity from real behavior, not stated opinions. ✅ Gets a version of the tier in market faster than full discovery. ❌ Needs funnel instrumentation and traffic volume; tells you whether price matters, not why devs leave.\nC) Launch on intuition as written\nEffort S (zero extra validation work), high risk. Keep the plan as is: launch the tier because the team believes cheaper is better. ✅ Fastest path to having a cheaper tier publicly available for developers. ✅ No research or experiment overhead for the team to absorb. ❌ No way to know if price was the barrier; revenue loss and a sticky price point if it wasn't.\n", + "edits": [ + { + "toolUseId": "toolu_t08_unpublished_edit", + "timestamp": null, + "published": false, + "reconstructed": true, + "input": { + "old_string": "Commitment | Source/approval or pending | Current | A | B | C\nDeveloper-friendly tier exists | PLAN.md scope (preserved) | yes | yes | yes, gated on discovery evidence | yes, as experiment arm\nTalk to developers first | pending | no | no | yes, 10-15 interviews + funnel read | no (data from experiment instead)\nFunnel instrumentation | pending | unknown | not required | read existing funnel, add gaps | required (A/B split)\nPrice-point decision basis | pending | team intuition | team intuition | interview + funnel evidence | experiment result\nReversibility | pending | low | low | high (nothing shipped until gate) | medium (exposed cohort keeps price)\nTime to tier in market | pending | fastest | fastest | +1-2 weeks | +1 week setup, 2-4 week run\nSuccess metric (D2) | pending, separate row | \"More signups\" | unchanged, pending | unchanged, pending | unchanged, pending", + "new_string": "Commitment | Source/approval or pending | Current | A (discovery gate) | B (experiment) | C (as written)\nDeveloper-friendly tier exists | PLAN.md scope (preserved) | planned | yes, gated on discovery evidence | yes, as experiment arm | yes\nTalk to developers first | pending | no | yes, 10-15 interviews + funnel read | no (data from experiment instead) | no\nFunnel instrumentation | pending | unknown | read existing funnel, add gaps | required (randomized split) | not required\nPrice-point decision basis | pending | team intuition | interview + funnel evidence | experiment result | team intuition\nReversibility | pending | low | high (nothing shipped until gate) | medium (exposed cohort keeps price) | low\nTime to tier in market | pending | fastest | +1-2 weeks | +1 week setup, 2-4 week run | fastest\nSuccess metric (D2) | pending, separate row | \"More signups\" | unchanged, pending | unchanged, pending | unchanged, pending", + "replace_all": false + } + } + ] + } + ] +} diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 805ac3642..2d0bb0844 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -145,6 +145,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/parity-baseline-v1.44.1.json': ['test/helpers/capture-parity-baseline.test.ts', 'test/parity-baseline-integrity.test.ts'], 'test/fixtures/parity-baseline-v1.64.1.0.json': ['test/parity-suite.test.ts', 'test/qa-lazy-sections.test.ts', 'test/skill-size-budget.test.ts'], 'test/fixtures/parity-baseline-v1.81.0.0.json': ['test/skill-size-budget.test.ts'], + 'test/fixtures/plan-ceo-floor-edit-prepublication.json': ['test/plan-edit-cropped-permission.test.ts'], 'test/fixtures/plan-ceo-floor-toolonly-create.json': ['test/plan-create-prepublication.test.ts', 'test/plan-floor-permission.test.ts'], 'test/fixtures/plan-count-design-questionless-report.md': ['test/plan-count-empty-review.test.ts'], 'test/fixtures/plan-create-cropped-title-batching.json': ['test/plan-create-combined-permission.test.ts'], diff --git a/test/helpers/plan-count-file-permission.ts b/test/helpers/plan-count-file-permission.ts index d464d6739..b68850b50 100644 --- a/test/helpers/plan-count-file-permission.ts +++ b/test/helpers/plan-count-file-permission.ts @@ -81,8 +81,28 @@ export function readPendingWriteInput(file: string, expected: string, cwd: strin }catch{return undefined;} } +const digest = (bytes: Buffer) => createHash('sha256').update(bytes).digest('hex'); + +/** Private inputs of open owned Edits, keyed by native ID; the record holds their digest. */ +function readPendingEditInputs(file: string, r: any): Record { + if (typeof r.editInputsSha256 !== 'string') return {}; + const bytes = boundedRegular(file + '.edit.json', MAX_WRITE_INPUT_BYTES); + if (digest(bytes) !== r.editInputsSha256) throw Error('changed Edit input witness'); + return JSON.parse(bytes.toString('utf8')); +} + +function writePendingEditInputs(file: string, state: any, inputs: Record) { + delete state.editInputsSha256; + if (!Object.keys(inputs).length) { fs.rmSync(file + '.edit.json', {force:true}); return; } + const bytes = Buffer.from(JSON.stringify(inputs) + '\n'); + if (bytes.length > MAX_WRITE_INPUT_BYTES) throw Error('oversized Edit input witness'); + fs.writeFileSync(file + '.edit.json.tmp', bytes, {mode:0o600}); + fs.renameSync(file + '.edit.json.tmp', file + '.edit.json'); + state.editInputsSha256 = digest(bytes); +} + /** No stdout, permission decision, input rewrite or model context. Metadata stays - * content-free; only an owned Write gets a bounded private input sibling. */ + * content-free; owned Write and Edit inputs get bounded private siblings. */ export function recordFilePermission(input: string, file: string, cwd: string, config: string, expected: string) { try { if (Buffer.byteLength(input) > MAX_WRITE_INPUT_BYTES) throw Error('oversized hook'); @@ -100,14 +120,17 @@ export function recordFilePermission(input: string, file: string, cwd: string, c const id = `${e.session_id}:${e.tool_use_id}`; const timestamp = new Date().toISOString(); const same = old.cwd === cwd && old.expected === expected && old.sessionId === e.session_id; - const state = { cwd, expected, sessionId: e.session_id, transcriptPath: e.transcript_path, + const edits = same ? readPendingEditInputs(file, old) : {}; + const state: any = { cwd, expected, sessionId: e.session_id, transcriptPath: e.transcript_path, seenIds: same && Array.isArray(old.seenIds) ? old.seenIds : [], + openIds: same && Array.isArray(old.openIds) ? old.openIds : [], pendingId: same ? old.pendingId ?? null : null, completedId: same ? old.completedId ?? null : null, completedIds: same && Array.isArray(old.completedIds) ? old.completedIds : [], timestamp, ...(same && old.overlappingWriteInput === true ? {overlappingWriteInput:true} : {}), - ...(same && typeof old.writeInputSha256 === 'string' ? {writeInputSha256:old.writeInputSha256} : {}) }; + ...(same && typeof old.writeInputSha256 === 'string' ? {writeInputSha256:old.writeInputSha256} : {}), + ...(same && typeof old.editInputsSha256 === 'string' ? {editInputsSha256:old.editInputsSha256} : {}) }; let clearWriteInput = false; if (e.hook_event_name === 'PreToolUse') { // Replayed requests, including failed and older completed IDs, never reopen. @@ -121,12 +144,23 @@ export function recordFilePermission(input: string, file: string, cwd: string, c fs.writeFileSync(file+'.tmp',JSON.stringify(old)+'\n',{mode:0o600}); fs.renameSync(file+'.tmp',file); } + // A repeated Edit ID with different arguments loses its witness for good. + if (edits[id] && !isDeepStrictEqual(edits[id].input, e.tool_input)) { + delete edits[id]; + writePendingEditInputs(file, old, edits); + fs.writeFileSync(file+'.tmp',JSON.stringify(old)+'\n',{mode:0o600}); + fs.renameSync(file+'.tmp',file); + } return; } if (state.seenIds.length >= 128) throw Error('too many file requests'); if(state.pendingId) state.overlappingWriteInput=true; state.seenIds.push(id); + state.openIds.push(id); state.pendingId = id; + if (e.tool_name === 'Edit' && typeof e.tool_input.old_string === 'string' && typeof e.tool_input.new_string === 'string') + edits[id] = {source:'PreToolUse',toolName:'Edit',cwd,expected,sessionId:e.session_id, + transcriptPath:e.transcript_path,pendingId:id,timestamp,input:e.tool_input}; delete state.writeInputSha256; clearWriteInput = true; if(e.tool_name==='Write' && typeof e.tool_input.content==='string') { @@ -143,10 +177,15 @@ export function recordFilePermission(input: string, file: string, cwd: string, c } } else { // An unrelated/late result cannot overwrite the current request epoch. - if (state.pendingId !== id) return; - state.pendingId = null; - delete state.writeInputSha256; - clearWriteInput = true; + // An earlier open request (parallel Edits) completes without moving it. + if (state.pendingId !== id && !state.openIds.includes(id)) return; + state.openIds = state.openIds.filter((open: string) => open !== id); + delete edits[id]; + if (state.pendingId === id) { + state.pendingId = null; + delete state.writeInputSha256; + clearWriteInput = true; + } if (e.hook_event_name === 'PostToolUse') { state.completedId = id; // Polling may miss automatically permitted edits between two menus. @@ -154,6 +193,7 @@ export function recordFilePermission(input: string, file: string, cwd: string, c if (!state.completedIds.includes(id)) state.completedIds.push(id); } } + writePendingEditInputs(file, state, edits); fs.writeFileSync(file+'.tmp',JSON.stringify(state)+'\n',{mode:0o600}); fs.renameSync(file+'.tmp',file); if(clearWriteInput) fs.rmSync(file+'.write.json',{force:true}); @@ -310,11 +350,64 @@ function currentCreatePreview(preview: string, r: any, config: string, cwd: stri (!i||row.line===numbered[i-1]!.line+1)&&compact(row.text)===compact(source[row.line-1]!)); } -/** A cropped in-fixture Edit needs its sole current native request and exact visible diff. */ -function currentEditPreview(preview: string, r: any, config: string, cwd: string, startedAt: number): boolean { +/** One Edit's exact visible diff against the current owned file bytes. */ +function editPreviewMatches(preview: string, input: any, before: string): boolean { + if (typeof input?.old_string !== 'string' || !input.old_string || typeof input.new_string !== 'string' || + (input.replace_all !== undefined && input.replace_all !== false)) return false; + const at = before.indexOf(input.old_string); + if (at < 0 || before.indexOf(input.old_string, at + input.old_string.length) !== -1) return false; + const after = before.slice(0, at) + input.new_string + before.slice(at + input.old_string.length); + const oldLines = before.split(/\r?\n/), newLines = after.split(/\r?\n/); + const firstLine = before.slice(0, at).split(/\r?\n/).length; + const oldLast = firstLine + input.old_string.split(/\r?\n/).length - 1; + const newLast = firstLine + input.new_string.split(/\r?\n/).length - 1; + const rows: Array<{line: number; kind: string; text: string; clipped?: boolean; gutter?: number}> = []; + let leading: {kind: string; text: string} | undefined; + for (const line of preview.split('\n')) { + if (!line.trim() || /^[╌─━]{3,}[ \t]*$/.test(line)) continue; + // A trimmed empty unchanged row keeps only its number. + const numbered = /^( {0,3}([1-9]\d*) )([ +\-])(.*)$/.exec(line) ?? /^( {0,3}([1-9]\d*))()[ \t]*$/.exec(line); + if (numbered) { + const kind = numbered[3] || ' '; + if (leading) { + // A wrapped first row has no coordinate. The next same-kind numbered + // row anchors its complete visible suffix to the preceding source line. + if (leading.kind !== kind || Number(numbered[2]) < 2) return false; + rows.push({line:Number(numbered[2])-1,...leading,clipped:true}); leading=undefined; + } + rows.push({line:Number(numbered[2]),kind,text:numbered[4] ?? '',gutter:numbered[1]!.length}); continue; + } + const last = rows.at(-1); + // An unchanged row wraps under its own gutter with a blank marker column. + if (last?.kind === ' ' && last.gutter && line.startsWith(' '.repeat(last.gutter + 1))) { + last.text += line.slice(last.gutter + 1); continue; + } + const wrapped = /^ {4,5}([+\-])(.*)$/.exec(line); + if (!wrapped) return false; + if (!last) { + if (leading && leading.kind !== wrapped[1]) return false; + leading={kind:wrapped[1]!,text:(leading?.text??'')+wrapped[2]!}; + } else { + if (wrapped[1] !== last.kind) return false; + last.text += wrapped[2]!; + } + } + const compact = (value: string) => value.replace(/\s/g, ''); + return !leading && rows.length >= 2 && rows.some(row => row.kind === '+' || row.kind === '-') && + rows.every((row, i) => Number.isSafeInteger(row.line) && row.line > 0 && + (!i || row.kind === '-' || rows[i-1]!.kind === '-' || row.line === rows[i-1]!.line + 1) && + (row.kind === ' ' || (row.line >= firstLine && row.line <= (row.kind === '-' ? oldLast : newLast))) && + row.line <= (row.kind === '-' ? oldLines : newLines).length && + (row.clipped ? Boolean(compact(row.text)) && compact((row.kind === '-' ? oldLines : newLines)[row.line - 1]!).endsWith(compact(row.text)) : + compact(row.text) === compact((row.kind === '-' ? oldLines : newLines)[row.line - 1]!))); +} + +/** A cropped in-fixture Edit binds its exact visible diff to exactly one pending + * owned request, published or witnessed by its own PreToolUse hook. Returns that ID. */ +function currentEditPreview(preview: string, r: any, config: string, cwd: string, startedAt: number, file: string): string | null { try { if (path.dirname(r.expected) !== cwd || fs.realpathSync(cwd) !== cwd || fs.realpathSync(r.expected) !== r.expected || - [config, path.dirname(r.transcriptPath), r.transcriptPath].some(p => fs.realpathSync(p) !== p)) return false; + [config, path.dirname(r.transcriptPath), r.transcriptPath].some(p => fs.realpathSync(p) !== p)) return null; const pending = new Map(), seen = new Map(); const completed = new Set(); let conflict = false; @@ -331,61 +424,31 @@ function currentEditPreview(preview: string, r: any, config: string, cwd: string completed.add(event.toolUseId); pending.delete(event.toolUseId); } }, r.transcriptPath); - const event = [...pending.values()][0], input = event?.input; - if (conflict || transcript.status !== 'ready' || pending.size !== 1 || event?.name !== 'Edit' || - `${event.sessionId}:${event.toolUseId}` !== r.pendingId || !Number.isFinite(Date.parse(event.timestamp)) || - Date.parse(event.timestamp) < startedAt || Date.parse(event.timestamp) > Date.now() || - input?.file_path !== r.expected || typeof input.old_string !== 'string' || !input.old_string || - typeof input.new_string !== 'string' || (input.replace_all !== undefined && input.replace_all !== false)) return false; - const bytes = boundedRegular(r.expected, MAX_WRITE_INPUT_BYTES), before = bytes.toString('utf8'); - if (!Buffer.from(before).equals(bytes)) return false; - const at = before.indexOf(input.old_string); - if (at < 0 || before.indexOf(input.old_string, at + input.old_string.length) !== -1) return false; - const after = before.slice(0, at) + input.new_string + before.slice(at + input.old_string.length); - const oldLines = before.split(/\r?\n/), newLines = after.split(/\r?\n/); - const firstLine = before.slice(0, at).split(/\r?\n/).length; - const oldLast = firstLine + input.old_string.split(/\r?\n/).length - 1; - const newLast = firstLine + input.new_string.split(/\r?\n/).length - 1; - const rows: Array<{line: number; kind: string; text: string; clipped?: boolean; gutter?: number}> = []; - let leading: {kind: string; text: string} | undefined; - for (const line of preview.split('\n')) { - if (!line.trim() || /^[╌─━]{3,}[ \t]*$/.test(line)) continue; - // A trimmed empty unchanged row keeps only its number. - const numbered = /^( {0,3}([1-9]\d*) )([ +\-])(.*)$/.exec(line) ?? /^( {0,3}([1-9]\d*))()[ \t]*$/.exec(line); - if (numbered) { - const kind = numbered[3] || ' '; - if (leading) { - // A wrapped first row has no coordinate. The next same-kind numbered - // row anchors its complete visible suffix to the preceding source line. - if (leading.kind !== kind || Number(numbered[2]) < 2) return false; - rows.push({line:Number(numbered[2])-1,...leading,clipped:true}); leading=undefined; - } - rows.push({line:Number(numbered[2]),kind,text:numbered[4] ?? '',gutter:numbered[1]!.length}); continue; - } - const last = rows.at(-1); - // An unchanged row wraps under its own gutter with a blank marker column. - if (last?.kind === ' ' && last.gutter && line.startsWith(' '.repeat(last.gutter + 1))) { - last.text += line.slice(last.gutter + 1); continue; - } - const wrapped = /^ {4,5}([+\-])(.*)$/.exec(line); - if (!wrapped) return false; - if (!last) { - if (leading && leading.kind !== wrapped[1]) return false; - leading={kind:wrapped[1]!,text:(leading?.text??'')+wrapped[2]!}; - } else { - if (wrapped[1] !== last.kind) return false; - last.text += wrapped[2]!; - } + if (conflict || transcript.status !== 'ready') return null; + // Any other pending file request (a Write, or an Edit elsewhere) keeps the menu waiting. + const candidates = new Map(); + for (const event of pending.values()) { + if (event.name !== 'Edit' || event.input?.file_path !== r.expected) return null; + candidates.set(`${event.sessionId}:${event.toolUseId}`, event); } - const compact = (value: string) => value.replace(/\s/g, ''); - return !leading && rows.length >= 2 && rows.some(row => row.kind === '+' || row.kind === '-') && - rows.every((row, i) => Number.isSafeInteger(row.line) && row.line > 0 && - (!i || row.kind === '-' || rows[i-1]!.kind === '-' || row.line === rows[i-1]!.line + 1) && - (row.kind === ' ' || (row.line >= firstLine && row.line <= (row.kind === '-' ? oldLast : newLast))) && - row.line <= (row.kind === '-' ? oldLines : newLines).length && - (row.clipped ? Boolean(compact(row.text)) && compact((row.kind === '-' ? oldLines : newLines)[row.line - 1]!).endsWith(compact(row.text)) : - compact(row.text) === compact((row.kind === '-' ? oldLines : newLines)[row.line - 1]!))); - } catch { return false; } + // Claude Code can show the menu before it publishes the Edit; the owned hook input covers that delay. + const open: unknown[] = Array.isArray(r.openIds) ? r.openIds : [r.pendingId]; + for (const [id, witness] of Object.entries(readPendingEditInputs(file, r))) { + if (witness?.pendingId !== id || witness.toolName !== 'Edit' || witness.sessionId !== r.sessionId || witness.cwd !== cwd || + witness.expected !== r.expected || witness.transcriptPath !== r.transcriptPath || witness.input?.file_path !== r.expected || + !open.includes(id) || completed.has(id.slice(r.sessionId.length + 1))) return null; + const published = candidates.get(id); + if (published && !isDeepStrictEqual(published.input, witness.input)) return null; + if (!published) candidates.set(id, witness); + } + const bytes = boundedRegular(r.expected, MAX_WRITE_INPUT_BYTES), before = bytes.toString('utf8'); + if (!Buffer.from(before).equals(bytes)) return null; + const matches = [...candidates].filter(([, {timestamp, input}]) => { + const time = Date.parse(timestamp); + return Number.isFinite(time) && time >= startedAt && time <= Date.now() && editPreviewMatches(preview, input, before); + }); + return matches.length === 1 && open.includes(matches[0]![0]) ? matches[0]![0] : null; + } catch { return null; } } /** Undefined leaves other permissions alone; null keeps this report pane waiting. */ @@ -424,8 +487,9 @@ export function currentFilePermissionEpoch(file: string | undefined, expected: s r.completedIds.some((id: string) => id === r.pendingId || !r.seenIds.includes(id)) || (r.completedId === null ? r.completedIds.length !== 0 : r.completedIds.at(-1) !== r.completedId)) return null; if(create && !currentCreatePreview(create.preview,r,config,cwd,startedAt,file)) return null; - if(edit && edit.directory === undefined && !currentEditPreview(edit.preview,r,config,cwd,startedAt)) return null; - return {pendingId:r.pendingId,completedId:r.completedId,completedIds:r.completedIds}; + const editId = edit && edit.directory === undefined ? currentEditPreview(edit.preview,r,config,cwd,startedAt,file) : undefined; + if(editId === null) return null; + return {pendingId:editId ?? r.pendingId,completedId:r.completedId,completedIds:r.completedIds}; } catch { return null; } } diff --git a/test/plan-edit-cropped-permission.test.ts b/test/plan-edit-cropped-permission.test.ts index eff849cbc..47605bf89 100644 --- a/test/plan-edit-cropped-permission.test.ts +++ b/test/plan-edit-cropped-permission.test.ts @@ -3,10 +3,11 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import {createHash} from 'node:crypto'; -import {createFilePermissionRecorder,currentFilePermissionEpoch,currentFilePermissionBinding} from './helpers/plan-count-file-permission'; +import {createFilePermissionRecorder,currentFilePermissionEpoch,currentFilePermissionBinding,recordFilePermission,isCroppedEditPermissionVisible} from './helpers/plan-count-file-permission'; import {readPlanCountTranscript} from './helpers/plan-count-transcript'; import {createPlanCountPermissionGuard,isPermissionDialogVisible,classifyPlanCountFrame} from './helpers/claude-pty-runner'; import captured from './fixtures/plan-edit-cropped-permission-1579.json'; +import editPrepublication from './fixtures/plan-ceo-floor-edit-prepublication.json'; function fixture(){ const dir=fs.mkdtempSync(path.join(os.tmpdir(),'cropped-edit-')),cwd=path.join(dir,'owned'),config=path.join(dir,'config');fs.mkdirSync(cwd); @@ -129,3 +130,93 @@ test.each(['foreign-basename','foreign-header','quoted','fenced','auq','old-menu expect(createPlanCountPermissionGuard()(screen,'',f.read(screen))).not.toBe('grant'); }finally{f.close();} }); + +// Ship-measure plan-ceo-finding-floor baseline at 6f32737: after the owned Write was +// granted, t04 (three published Edits pending at once) and t07/t08 (Edit menu shown +// before Claude Code published the Edit) stalled on the owned working-plan Edit menu. +function edits(trial:string){ + const c=editPrepublication.cases.find(c=>c.trial===trial)!; + const dir=fs.mkdtempSync(path.join(os.tmpdir(),'edit-prepub-')),cwd=path.join(dir,'owned'),config=path.join(dir,'config');fs.mkdirSync(cwd); + const expected=path.join(cwd,c.basename),sid=c.sessionId,journal=path.join(config,'projects','owned',sid+'.jsonl'); + fs.mkdirSync(path.dirname(journal),{recursive:true}); + const startedAt=Date.now()-1000,recorder=createFilePermissionRecorder(cwd,config,expected)!; + const native=(role:string,content:any[],session=sid)=>({cwd,sessionId:session,isSidechain:false,timestamp:new Date().toISOString(),message:{role,content}}); + const hook=(kind:string,tool:string,id:string,input:any,extra={})=>recordFilePermission(JSON.stringify({hook_event_name:kind,tool_name:tool,session_id:sid, + tool_use_id:id,cwd,transcript_path:journal,tool_input:input,...extra}),recorder.file,cwd,config,expected); + const writeInput={file_path:expected,content:c.writeContent}; + const base=[native('assistant',[{type:'tool_use',id:'owned-write',name:'Write',input:writeInput}]), + native('user',[{type:'tool_result',tool_use_id:'owned-write',content:'ok'}])]; + hook('PreToolUse','Write','owned-write',writeInput);fs.writeFileSync(expected,c.writeContent);hook('PostToolUse','Write','owned-write',writeInput); + const write=(rows:any[]=base)=>fs.writeFileSync(journal,rows.map(r=>JSON.stringify(r)).join('\n')+'\n');write(); + const pending=c.edits.map(e=>({id:e.toolUseId,input:{file_path:expected,...e.input}})); + const block=(e:{id:string;input:any})=>({type:'tool_use',id:e.id,name:'Edit',input:e.input}); + const read=(screen=c.screen)=>currentFilePermissionEpoch(recorder.file,expected,cwd,config,startedAt,readPlanCountTranscript(config,cwd),screen); + return {c,dir,cwd,config,expected,sid,journal,recorder,native,hook,base,write,pending,block,read,startedAt, + close(){recorder.dispose();fs.rmSync(dir,{recursive:true,force:true});}}; +} + +test.each(['t07','t08'])('%s: an unpublished owned Edit is granted once from its own PreToolUse input',trial=>{ + const f=edits(trial);try{ + expect(f.c.recorded).toMatchObject({outcome:'permission_unanswered',transcriptStatus:'ready',publishedEdits:0}); + expect(createHash('sha256').update(f.c.screen).digest('hex')).toBe(f.c.screenSha256); + expect(isCroppedEditPermissionVisible(f.c.screen)).toBe(true); + const [edit]=f.pending;expect(f.read()).toBeNull(); + f.hook('PreToolUse','Edit',edit!.id,edit!.input); + expect(fs.readFileSync(f.recorder.file,'utf8')).not.toContain(edit!.input.new_string); + expect(fs.statSync(f.recorder.file+'.edit.json').mode&0o777).toBe(0o600); + const owned=currentFilePermissionBinding([{file:f.recorder.file,expected:f.expected}],f.cwd,f.config,f.startedAt,readPlanCountTranscript(f.config,f.cwd),f.c.screen); + expect(owned?.epoch.pendingId).toBe(`${f.sid}:${edit!.id}`); + const guard=createPlanCountPermissionGuard();expect(guard(f.c.screen,'',f.read())).toBe('grant');expect(guard(f.c.screen,'',f.read())).toBe('handled'); + // Publication later must agree with the witness; completion retires it. + f.write([...f.base,f.native('assistant',[f.block(edit!)])]);expect(f.read()?.pendingId).toBe(`${f.sid}:${edit!.id}`); + f.write([...f.base,f.native('assistant',[f.block({...edit!,input:{...edit!.input,new_string:edit!.input.new_string+'\nhidden'}})])]);expect(f.read()).toBeNull(); + f.write();f.hook('PostToolUse','Edit',edit!.id,edit!.input); + expect(fs.existsSync(f.recorder.file+'.edit.json')).toBe(false);expect(f.read()).toBeNull(); + }finally{f.close();} +}); + +test.each(['before','after'])('t04: the menu binds to the one of three pending owned Edits its diff shows (later hooks %s the menu)',order=>{ + const f=edits('t04');try{ + expect(f.c.recorded).toMatchObject({outcome:'permission_unanswered',publishedEdits:3}); + f.write([...f.base,f.native('assistant',f.pending.map(f.block))]); + for(const e of order==='before'?f.pending:f.pending.slice(0,1))f.hook('PreToolUse','Edit',e.id,e.input); + const [shown,...rest]=f.pending; + expect(f.read()?.pendingId).toBe(`${f.sid}:${shown!.id}`); + const guard=createPlanCountPermissionGuard();expect(guard(f.c.screen,'',f.read())).toBe('grant');expect(guard(f.c.screen,'',f.read())).toBe('handled'); + // After the shown Edit lands, its stale render never binds to the other two. + const before=fs.readFileSync(f.expected,'utf8'); + fs.writeFileSync(f.expected,before.replace(shown!.input.old_string,shown!.input.new_string)); + f.hook('PostToolUse','Edit',shown!.id,shown!.input); + f.write([...f.base,f.native('assistant',f.pending.map(f.block)),f.native('user',[{type:'tool_result',tool_use_id:shown!.id,content:'ok'}])]); + if(order==='after')for(const e of rest)f.hook('PreToolUse','Edit',e.id,e.input); + expect(f.read()).toBeNull(); + }finally{f.close();} +}); + +test.each(['foreign-file-published','foreign-file-hook','no-match','two-matches-witnessed','two-matches-published','unwitnessed','foreign-session','foreign-session-journal','changed-witness','sidechain']) +('owned Edit menu negative control: %s is refused',mode=>{ + const f=edits(mode.includes('published')||mode==='unwitnessed'?'t04':'t07');try{ + const [edit]=f.pending;let screen=f.c.screen; + if(mode==='foreign-file-published'){ + f.write([...f.base,f.native('assistant',[...f.pending.map(f.block),f.block({id:'foreign-edit',input:{...edit!.input,file_path:path.join(f.dir,'foreign.md')}})])]); + for(const e of f.pending)f.hook('PreToolUse','Edit',e.id,e.input); + } + if(mode==='foreign-file-hook')f.hook('PreToolUse','Edit',edit!.id,{...edit!.input,file_path:path.join(f.dir,f.c.basename)}); + if(mode==='no-match'){f.hook('PreToolUse','Edit',edit!.id,edit!.input);screen=screen.replace('Recommendation: B','Recommendation: A');} + if(mode==='two-matches-witnessed'){f.hook('PreToolUse','Edit',edit!.id,edit!.input);f.hook('PreToolUse','Edit','twin-edit',edit!.input);} + if(mode==='two-matches-published'){ + f.write([...f.base,f.native('assistant',[...f.pending.map(f.block),f.block({id:'twin-edit',input:edit!.input})])]); + for(const e of f.pending)f.hook('PreToolUse','Edit',e.id,e.input); + } + if(mode==='unwitnessed'){f.write([...f.base,f.native('assistant',f.pending.map(f.block))]);for(const e of f.pending.slice(1))f.hook('PreToolUse','Edit',e.id,e.input);} + if(mode==='foreign-session'||mode==='foreign-session-journal'){ + const foreign='11111111-2222-4333-8444-555555555555',journal=path.join(f.config,'projects','owned',foreign+'.jsonl'); + if(mode==='foreign-session-journal')fs.writeFileSync(journal,JSON.stringify(f.native('assistant',[f.block(edit!)],foreign))+'\n'); + f.hook('PreToolUse','Edit',edit!.id,edit!.input,{session_id:foreign,transcript_path:journal}); + } + if(mode==='changed-witness'){f.hook('PreToolUse','Edit',edit!.id,edit!.input);fs.appendFileSync(f.recorder.file+'.edit.json',' ');} + if(mode==='sidechain')f.hook('PreToolUse','Edit',edit!.id,edit!.input,{agent_id:'helper'}); + expect(f.read(screen)).toBeFalsy(); + expect(createPlanCountPermissionGuard()(screen,'',f.read(screen))).not.toBe('grant'); + }finally{f.close();} +});