diff --git a/test/fixtures/judge-calibration/voice/runs/comparison-1.result.json b/test/fixtures/judge-calibration/voice/runs/comparison-1.result.json new file mode 100644 index 000000000..cabd79eae --- /dev/null +++ b/test/fixtures/judge-calibration/voice/runs/comparison-1.result.json @@ -0,0 +1,89 @@ +{ + "config": "voice", + "model": "claude-fable-5-1", + "request_hash": { + "old": "c00326001bb0e5511ead24b7568cfeafc3a9ae4fe4a4f94a1ed5e556fae12497", + "new": "0844de9d541b7a4d20ceb3af7324fe2744141c34f021de7d94609d7b4babd7c2" + }, + "complete": true, + "stopped": null, + "calls": 216, + "cost_usd": 8.0866, + "errors": { + "old": { + "refusal": 0, + "truncation": 0, + "parse": 0, + "schema": 0, + "transport": 0, + "samples": 144, + "rate": 0 + }, + "new": { + "refusal": 0, + "truncation": 0, + "parse": 0, + "schema": 0, + "transport": 0, + "samples": 72, + "rate": 0 + } + }, + "latency_p95_ms": { + "old": 13383, + "new": 13713 + }, + "all": { + "items": 24, + "old_flips": { + "flips": 0, + "compared": 24 + }, + "new_flips": { + "flips": 0, + "compared": 24 + }, + "agreement": { + "old-a": 22, + "old-b": 22, + "new": 22 + }, + "false_passes": { + "old-a": [], + "old-b": [], + "new": [] + }, + "new_false_passes": [] + }, + "heldout": { + "items": 8, + "old_flips": { + "flips": 0, + "compared": 8 + }, + "new_flips": { + "flips": 0, + "compared": 8 + }, + "agreement": { + "old-a": 8, + "old-b": 8, + "new": 8 + }, + "false_passes": { + "old-a": [], + "old-b": [], + "new": [] + }, + "new_false_passes": [] + }, + "landing": { + "lands": true, + "reasons": [], + "strict_no_flip_increase": true + }, + "prose_step_triggered": { + "triggered": false, + "reasons": [] + } +} diff --git a/test/fixtures/judge-calibration/voice/runs/comparison-1.samples.jsonl b/test/fixtures/judge-calibration/voice/runs/comparison-1.samples.jsonl new file mode 100644 index 000000000..6597bbfcf --- /dev/null +++ b/test/fixtures/judge-calibration/voice/runs/comparison-1.samples.jsonl @@ -0,0 +1,216 @@ +{"config":"voice","item":"f01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it mandates opening with acknowledgment/restatement, a 'comprehensive overview' before any recommendation, and explicitly forbids 'strong opinions' — burying the lede and banning positions. Concreteness: no instruction to cite files, commands, line numbers, or real numbers; the example ('several opportunities for enhancement... broader modernization initiative') is maximally vague. Avoids corporate: it actively requires a 'courteous, formal tone at all times' and prescribes corporate filler phrases ('We are pleased to report', 'Moving forward, we recommend') plus an 'executive summary' closer. Avoids AI vocabulary: it recommends AI-tell phrases ('It is important to note', 'robust, scalable value', 'best practices', 'thorough review') rather than banning them, with no banned list. Connects user outcomes: it talks about 'stakeholders' and 'value' abstractly but never instructs tying technical work to what a real user experiences."},"latency_ms":5180,"stop_reason":"end_turn","usage":{"input_tokens":673,"output_tokens":410},"cost_usd":0.02723} +{"config":"voice","item":"f03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":1,"reasoning":"Directness: explicitly demands a verdict first (Approve/Request changes/Block) then findings most-severe-first; strong. Concreteness: requires path:line citations with the offending expression quoted, exact fixes or commands with a real example, and real numbers (allocation/query counts, benchmarks); the good/bad example reinforces it. Avoids corporate: bans corporate and academic tone and gives a concrete alternative persona (terse senior engineer leaving PR comments vs consultant report), though it doesn't elaborate beyond one line. Avoids AI vocabulary: provides a specific banned-word list (delve, crucial, robust, comprehensive, nuanced, etc.) plus a ban on em dashes. Connects user outcomes: the directive does the opposite, explicitly forbidding any discussion of end users, customers, product impact, or business outcomes and instructing the agent to stay strictly at the code level, so it scores at the floor on this dimension."},"latency_ms":5287,"stop_reason":"end_turn","usage":{"input_tokens":734,"output_tokens":385},"cost_usd":0.02659} +{"config":"voice","item":"f05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of every dimension. Directness: it explicitly forbids taking positions ('avoid declaring one'), tells the agent to withhold its own view until after multiple perspectives, and to frame bugs as gentle questions rather than stating them. Concreteness: it instructs the agent to 'avoid overly technical detail in the opening' and 'build up to specifics gradually' — the opposite of leading with files, commands, line numbers, or numbers; nothing asks for specifics at all. Avoids corporate: it actively prescribes the formal, committee-style tone ('balanced, thoughtful', even tradeoffs, headings and bullets) rather than banning it. Avoids AI vocabulary: it mandates classic AI hedge phrases ('it may be worth considering', 'one possible approach', 'this could potentially') as required language, with no banned list. Connects user outcomes: no mention of real users, product impact, or consequences of the technical work."},"latency_ms":6631,"stop_reason":"end_turn","usage":{"input_tokens":551,"output_tokens":383},"cost_usd":0.02466} +{"config":"voice","item":"a03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":4,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, and the good/bad example pair showing a hedged bad version vs. a committed fix. Concreteness: explicit list (files, functions, line numbers, commands, outputs, evals, real numbers) plus examples that model it (auth.ts:47, /login, 'Two lines', '3 files', 'v1.2'). Avoids corporate: bans corporate/academic/PR/hype, filler, throat-clearing, generic optimism, founder cosplay, and offers the 'builder talking to a builder' alternative; slightly short of 5 because the alternative is a stance rather than concrete rewrite patterns beyond the one example. AI vocabulary: explicit named list of ~19 words plus em dash ban. User outcomes: 'what changes for the builder' and the example ('Users hit a white screen') tie technical issues to user impact, but it's one bullet and one example rather than a sustained instruction to always trace work to user experience."},"latency_ms":7828,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":405},"cost_usd":0.03097} +{"config":"voice","item":"f03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness is excellent: it mandates leading with a verdict (Approve/Request changes/Block), ordering findings by severity, and the good/bad example contrasts a precise position against a hedge. Concreteness is excellent: path:line citations, quoted offending expressions, exact fix or command with a real example (`bun test test/session.test.ts`), and real numbers (allocation counts, query counts, benchmarks). Avoids corporate: it explicitly bans corporate/academic tone and gives a concrete alternative persona (terse senior engineer leaving PR comments vs. consultant writing a report), though it offers no banned-phrase list for corporate register itself, so 4. Avoids AI vocabulary: a specific banned-word list is present and covers the most common tells plus em dashes, but it omits many frequent offenders (leverage, utilize, seamless, streamline, 'it's worth noting', 'in conclusion', hedging phrases), so 4. Connects user outcomes: the directive does the opposite of what this dimension asks. It explicitly forbids speculating about end users, customers, product impact, or business outcomes and instructs the agent to describe what code does, not how anyone will experience it. Whether or not that scoping is defensible for a pure code-review persona, on this rubric it is the inverse of the criterion, so 1."},"latency_ms":7885,"stop_reason":"end_turn","usage":{"input_tokens":734,"output_tokens":504},"cost_usd":0.03254} +{"config":"voice","item":"p05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Open with status and action, in that order', 'Commit to a call', explicit example of 'roll back' vs 'a rollback could be considered' — leads with the point and demands positions. Concreteness: requires dashboard names, timestamped log lines, exact commands (`fly releases rollback v118`), and real numbers ('5xx at 7.2%, baseline 0.3%'); the good example is dense with specifics (api-7, 01:52, deploy 4812, 12%). Avoids corporate: explicitly bans 'status-page voice' with quoted examples ('we are aware of an issue impacting some users', 'out of an abundance of caution') and gives the alternative ('say what broke, who it hits, and what you are doing'), plus a good/bad contrast. Avoids AI vocabulary: a specific banned-word list (delve, crucial, robust, comprehensive, nuanced, pivotal, landscape, seamless, leverage, furthermore, moreover, additionally) plus banned phrases ('it's worth noting', 'let's dive in'). Connects user outcomes: 'Translate every metric into what customers are living through' with three concrete translations (checkout failures to orders/minute, blank dashboard, 9-second logins), reinforced in the good example ('Users get an empty results page'). Tight, domain-grounded, and every dimension is explicitly instructed with examples."},"latency_ms":7971,"stop_reason":"end_turn","usage":{"input_tokens":851,"output_tokens":527},"cost_usd":0.03486} +{"config":"voice","item":"a01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: opens with 'Direct, concrete, builder-to-builder,' bans filler, and demands ending with an action, but never explicitly says to lead with the point or take positions (4). Concreteness: explicitly says name the file, function, command, and impact; omits line numbers and real numbers (4). Corporate/academic: banned in a single sentence with no examples of offending phrasing and no suggested alternatives beyond 'short paragraphs' (3). AI vocabulary: provides a concrete banned-word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus em dashes; list is short and omits common tells like 'leverage,' 'seamless,' 'it's worth noting' (4). User outcomes: 'user-visible impact' is named as a required element, which is a real instruction, but it is one phrase with no elaboration on how to connect technical changes to user experience (3). Overall a tight, usable directive that trades depth for brevity."},"latency_ms":8147,"stop_reason":"end_turn","usage":{"input_tokens":465,"output_tokens":388},"cost_usd":0.02405} +{"config":"voice","item":"f01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension measures. Directness: it explicitly instructs the agent to open with acknowledgment and summary, bury recommendations behind a 'comprehensive overview', and avoid 'strong opinions' — the opposite of leading with the point and taking positions. Concreteness: no mention of files, commands, line numbers, or numbers; the example is pure abstraction ('several opportunities for enhancement'). Avoids_corporate: it mandates a formal, courteous register and even supplies corporate boilerplate ('We are pleased to report', 'It is important to note', 'Moving forward, we recommend', 'executive summary') as recommended phrasing. Avoids_ai_vocabulary: rather than banning AI-tell words, it promotes them ('robust, scalable value', 'aligns with best practices', 'broader modernization initiative', 'thorough review'). Connects_user_outcomes: it speaks only of 'stakeholders' and 'value' in the abstract, never of what a real user experiences; the example never says what the auth problems actually do to anyone."},"latency_ms":8474,"stop_reason":"end_turn","usage":{"input_tokens":1186,"output_tokens":426},"cost_usd":0.03316} +{"config":"voice","item":"p04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: opens with 'Lead with the point,' sets 'direct, concrete, sharp' tone, and frames the voice as a builder talking to a builder. Concreteness: dedicated section with explicit before/after examples (file:line, exact bun test command, real latency numbers). Avoids corporate: explicitly bans corporate, academic, PR, and hype tone and offers the consultant-vs-builder contrast plus context-matched energy levels, but gives few concrete rewrites of corporate phrasing compared to the concreteness section. AI vocabulary: specific banned word list (delve, robust, tapestry, etc.), banned-phrase list, and em dash rule. User outcomes: dedicated section with concrete example sentences and the instruction to 'make the user's user real.'"},"latency_ms":6655,"stop_reason":"end_turn","usage":{"input_tokens":1046,"output_tokens":327},"cost_usd":0.02681} +{"config":"voice","item":"p04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":4,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' is explicit, and tone is 'direct, sharp'. It lacks explicit instruction to take positions, commit to a recommendation, or avoid hedging language, so not a full 5. Concreteness: excellent, with before/after examples for commands (`bun test test/billing.test.ts`), real numbers (N+1 ~200ms), and file:line pointers (auth.ts:47). Avoids corporate: explicitly bans corporate, academic, PR, and hype, and gives an alternative framing (builder to builder, YC partner, senior eng, technical blog post). It does not list specific corporate phrases or words to avoid, so a 4. AI vocabulary: a specific list of 20 banned words plus a list of banned filler phrases and an em dash ban, clearly a 5. User outcomes: a dedicated section with concrete example sentences and 'make the user's user real', a 5."},"latency_ms":6989,"stop_reason":"end_turn","usage":{"input_tokens":1559,"output_tokens":372},"cost_usd":0.03419} +{"config":"voice","item":"f04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is two generic sentences with no instruction on any of the five dimensions. It never tells the agent to lead with the point or take positions; 'explain thoroughly' and 'keep a positive tone' actually push toward padding and hedging, the opposite of directness. There is no mention of citing files, commands, line numbers, or real numbers. It does not ban corporate, formal, or academic tone, nor offer alternatives. It contains no list of AI-tell words or phrases to avoid. It says nothing about tying technical work to what users actually experience. 'Helpful, friendly, clear' is itself boilerplate that would produce default assistant prose."},"latency_ms":5342,"stop_reason":"end_turn","usage":{"input_tokens":881,"output_tokens":278},"cost_usd":0.02271} +{"config":"voice","item":"a01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: 'Direct', 'No filler', 'End with what to do' cover leading with the point, but it never explicitly says to take positions or lead with the conclusion first; the user-decides clause slightly hedges. Concreteness: 'Name the file, function, command' is specific and actionable, though line numbers and real numbers are not mentioned. Avoids corporate: explicitly bans corporate and academic tone and offers 'builder-to-builder' and 'short paragraphs' as the alternative, but no before/after examples or description of what corporate tone looks like. AI vocabulary: concrete banned list of six words plus em dashes, which is good, but the list is short and omits common phrase-level tells (e.g. 'it's worth noting', 'leverage', 'seamless', 'in today's landscape'). User outcomes: 'user-visible impact' is named as a required element, which is a real instruction, but it is a single phrase with no guidance on how to connect code changes to what users actually experience."},"latency_ms":5985,"stop_reason":"end_turn","usage":{"input_tokens":465,"output_tokens":391},"cost_usd":0.0242} +{"config":"voice","item":"a06","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Direct, concrete, builder-to-builder', 'No filler', 'End with what to do' are strong, but it does not explicitly say to lead with the point or take positions; the closing section even softens stance by deferring to the user. Concreteness: 'Name the file, function, command' is explicit, though line numbers and real numbers/measurements are not mentioned. Avoids corporate: 'Never corporate or academic' is a flat ban, but no alternatives or examples of what to say instead beyond 'builder-to-builder' and 'short paragraphs'. AI vocabulary: a concrete banned-word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus em dashes, but the list is short and omits common phrase-level tells (e.g., 'it's worth noting', 'leverage', 'seamless'). User outcomes: excellent, 'Tie every finding to what the user sees, waits for, or loses' and 'user-visible impact' directly require connecting technical work to real experience."},"latency_ms":6996,"stop_reason":"end_turn","usage":{"input_tokens":999,"output_tokens":407},"cost_usd":0.03034} +{"config":"voice","item":"p02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'say so plainly', 'Don't dance around judgments', 'End with what to do'... explicit and repeated with examples. Concreteness: a dedicated section with before/after examples (file:line, exact bun test command, N+1 with ~200ms numbers), plus reinforced in writing rules. Avoids corporate: explicitly bans corporate, academic, PR, hype, consultant tone, filler, throat-clearing, generic optimism, and gives the alternative ('builder talking to a builder', energy-matching by context, 'What's interesting here is...' vs 'It is important to understand...'). Slightly short of a 5 because the alternatives are mostly posture descriptions rather than a bank of concrete swaps for corporate phrasing. AI vocabulary: a specific 20-word banned list plus a banned-phrases list plus the em dash rule and 'never self-referential about being AI'. User outcomes: dedicated section with concrete example sentences ('3-second spinner on every page load', 'edge case that loses the customer's data'), plus 'Make the user's user real' and the core framing around jobs to be done and shipping for real people."},"latency_ms":8743,"stop_reason":"end_turn","usage":{"input_tokens":2051,"output_tokens":465},"cost_usd":0.04376} +{"config":"voice","item":"p05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Open with status and action, in that order', 'Commit to a call', explicit example of saying 'roll back' instead of hedged phrasing. Concreteness: demands dashboard names, timestamped log lines, exact commands (`fly releases rollback v118`), error rates as numbers with baselines; the Good example is dense with specifics (api-7, 01:52, deploy 4812, 12%, 3 min). Avoids corporate: explicitly bans 'status-page voice' with named phrases ('we are aware of an issue impacting some users', 'out of an abundance of caution') and provides the alternative structure (what broke, who it hits, what you're doing), plus a Good/Bad pair. Avoids AI vocabulary: a specific list of banned words (delve, crucial, robust, comprehensive, nuanced, pivotal, landscape, seamless, leverage, furthermore, moreover, additionally) plus banned phrases ('it's worth noting', 'let's dive in'). Connects user outcomes: 'Translate every metric into what customers are living through' with three concrete translations (checkout failures to orders/minute, blank dashboard for EU users, 9-second logins), reinforced in the Good example ('Users get an empty results page'). Tightly scoped to incident/deploy work, but every dimension is explicitly instructed with examples."},"latency_ms":8485,"stop_reason":"end_turn","usage":{"input_tokens":851,"output_tokens":514},"cost_usd":0.03421} +{"config":"voice","item":"f05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is essentially the inverse of what each dimension asks for. Directness: it explicitly instructs the agent to avoid taking positions, present perspectives before any view, let the user conclude, and frame bugs as gentle questions — the opposite of leading with the point. Concreteness: no mention of files, commands, line numbers, or numbers; it actually tells the agent to avoid technical detail in the opening and build up gradually. Avoids corporate: it mandates hedged, formal phrasing and heading/bullet structure rather than banning it. Avoids AI vocabulary: it prescribes classic AI-tell hedges ('it may be worth considering', 'one possible approach', 'this could potentially') instead of banning them, with no banned-word list. Connects user outcomes: there is nothing about tying technical work to real user experience."},"latency_ms":4699,"stop_reason":"end_turn","usage":{"input_tokens":1064,"output_tokens":339},"cost_usd":0.02759} +{"config":"voice","item":"f04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is generic and vague. 'Be helpful, friendly, and clear' and 'explain things thoroughly' actually push toward verbosity and hedging rather than leading with the point or taking positions — no instruction to be direct. There is zero mention of naming files, commands, line numbers, or real numbers. It does not ban corporate, formal, or academic tone (if anything, 'positive tone' and 'feels supported' encourage soft corporate-support phrasing) and offers no alternatives. There is no list or mention of AI-tell vocabulary to avoid. Nothing connects technical work to real user experience or outcomes. It fails every dimension."},"latency_ms":3787,"stop_reason":"end_turn","usage":{"input_tokens":368,"output_tokens":281},"cost_usd":0.01773} +{"config":"voice","item":"f04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is a generic 'be helpful, friendly, clear' instruction with no guidance on any scored dimension. Directness: it actually pushes the opposite ('explain thoroughly', 'positive tone') rather than leading with the point or taking positions. Concreteness: no mention of files, commands, line numbers, or real numbers. Avoids corporate: no ban on corporate/formal/academic tone and no alternatives offered. Avoids AI vocabulary: no word or phrase lists at all; 'feels supported' and 'positive tone' invite the exact AI-tell patterns a strong directive would ban. Connects user outcomes: nothing about tying technical work to real user experience."},"latency_ms":5034,"stop_reason":"end_turn","usage":{"input_tokens":881,"output_tokens":294},"cost_usd":0.02351} +{"config":"voice","item":"p03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'first sentence = the answer', 'take a position' with concrete sample verdicts, bans warm-up and recap, and tells the agent how to handle uncertainty. Concreteness: demands file:line, function names, exact commands, and real numbers, with examples of each and a counterexample of vague phrasing. Avoids corporate: names corporate, academic, PR, and hype tones, lists specific banned phrases, and gives a clear alternative ('explain it to a teammate at their desk: short sentences, plain verbs, your actual opinion'). Avoids AI vocabulary: a specific list of 22 banned words, a list of banned phrases, and an em dash ban. Connects user outcomes: dedicated section requiring every bug/tradeoff/change to state what the user sees, loses, waits for, or can do, with a side-by-side example, plus a fallback rule when no user-visible effect exists. The good/bad example at the end reinforces all five dimensions at once. Only nitpick: the banned word list could include a few more common tells (e.g. 'ensure', 'streamline', 'dive into'), but it is already specific and actionable."},"latency_ms":7975,"stop_reason":"end_turn","usage":{"input_tokens":1616,"output_tokens":447},"cost_usd":0.03851} +{"config":"voice","item":"a06","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: says 'Direct', 'No filler', 'End with what to do' but does not explicitly instruct the agent to lead with the point or take positions; strong but not exhaustive. Concreteness: 'Name the file, function, command' is specific, but omits line numbers and real numbers/measurements. Avoids corporate: explicitly bans corporate and academic tone, but offers no alternatives or examples of what to write instead; 'builder-to-builder' is the only hint. Avoids AI vocabulary: provides a concrete list (delve, crucial, robust, comprehensive, nuanced, multifaceted) and bans em dashes, though the list is short and skips common phrase-level tells (e.g. 'it's worth noting', 'in today's landscape'). Connects user outcomes: excellent, 'Tie every finding to what the user sees, waits for, or loses' and 'user-visible impact' directly require linking technical work to real user experience."},"latency_ms":6830,"stop_reason":"end_turn","usage":{"input_tokens":999,"output_tokens":379},"cost_usd":0.02894} +{"config":"voice","item":"p01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point,' 'be direct about quality,' and the auth.ts example that states the bug, impact, and fix in four sentences. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the good example models it (auth.ts:47, /login, 'two lines'). Avoids corporate: bans corporate/academic/PR/hype/filler/throat-clearing/founder cosplay and supplies the alternative frame ('builder talking to a builder, not a consultant presenting to a client') plus a paired good/bad example. AI vocabulary: a specific 19-word ban list plus no em dashes, which is strong, but the directive itself contains an em dash in the bounded-closer section ('format — the report IS the work'), and the list is nearly all single words with little coverage of AI-tell phrases ('it's worth noting,' 'I've identified a potential issue' appears only in the bad example). User outcomes: 'what the real user sees, loses, waits for, or can now do' is a precise, actionable instruction and the example ('Users hit a white screen') demonstrates it."},"latency_ms":14072,"stop_reason":"end_turn","usage":{"input_tokens":1107,"output_tokens":1099},"cost_usd":0.06602} +{"config":"voice","item":"f01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what is wanted on every axis. Directness: it mandates opening with acknowledgment and a summary of understanding, a 'comprehensive overview' before any recommendation, and explicitly forbids strong opinions — the point is buried and positions are discouraged. Concreteness: nothing about naming files, commands, line numbers, or real numbers; the example ('several opportunities for enhancement... broader modernization initiative') is maximally vague. Avoids_corporate: it actively requires a formal corporate register and bans casual language and fragments. Avoids_ai_vocabulary: rather than banning AI-tell phrases, it prescribes them ('It is important to note', 'Moving forward, we recommend', 'robust, scalable value', 'best practices'). Connects_user_outcomes: it frames work in terms of stakeholders and 'value' abstractions, never real user experience."},"latency_ms":7139,"stop_reason":"end_turn","usage":{"input_tokens":1186,"output_tokens":364},"cost_usd":0.03006} +{"config":"voice","item":"p03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' section demands first-sentence verdicts, explicit positions ('Ship it', 'Don't merge this'), bans warm-ups and recaps, and tells the agent what to do when unsure. Concreteness: explicitly requires file:line, function name, command, and real numbers with concrete examples, and bans vague hedging like 'there may be an issue in the auth flow'. Avoids corporate: names corporate, academic, PR, and hype tone as banned, lists specific offending phrases, and supplies the alternative ('explain it to a teammate at their desk: short sentences, plain verbs, your actual opinion'). Avoids AI vocabulary: dedicated section with a specific word list (delve, crucial, robust, etc.), a phrase list ('here's the thing', 'it's worth noting'), and a no-em-dash rule. Connects user outcomes: 'Tie it to the user' requires naming what the real user sees, loses, waits for, or can do for every bug/tradeoff/change, with a before/after example, and a fallback instruction when no user-visible effect exists. The Good/Bad example at the end reinforces all five dimensions simultaneously."},"latency_ms":8277,"stop_reason":"end_turn","usage":{"input_tokens":1103,"output_tokens":460},"cost_usd":0.03403} +{"config":"voice","item":"p05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Open with status and action', 'Commit to a call', explicit rollback vs 'could be considered' contrast — strong. Concreteness: demands dashboard names, timestamped log lines, exact commands (`fly releases rollback v118`), numeric error rates with baselines — excellent. Avoids_corporate: bans 'status-page voice' with two concrete phrases and the Good/Bad examples contrast corporate hedging with plain speech; however it targets status-page/PR tone specifically rather than corporate/formal/academic tone broadly, and offers alternatives mainly via example rather than an explicit substitution list. Avoids_ai_vocabulary: explicit list of 12 banned words plus two banned phrases. Connects_user_outcomes: 'Translate every metric into what customers are living through' with three concrete examples and the Good example ('Users get an empty results page') — excellent."},"latency_ms":5442,"stop_reason":"end_turn","usage":{"input_tokens":1364,"output_tokens":366},"cost_usd":0.03194} +{"config":"voice","item":"a01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":4,"concreteness":3,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: 'Direct, concrete, builder-to-builder', 'No filler', 'End with what to do' are strong, but it doesn't explicitly say to lead with the point or take positions. Concreteness: 'Name the file, function, command' is good but omits line numbers and real numbers/measurements. Avoids corporate: 'Never corporate or academic' is a flat ban with 'builder-to-builder' as an implied alternative, but no concrete replacements or examples. Avoids AI vocabulary: bans em dashes and gives a specific six-word list (delve, crucial, robust, comprehensive, nuanced, multifaceted); solid but short, missing common tells like 'leverage', 'seamless', 'it's worth noting'. Connects user outcomes: 'user-visible impact' is named directly but only as a one-phrase mention without instruction on how to tie technical changes to real user experience."},"latency_ms":7224,"stop_reason":"end_turn","usage":{"input_tokens":978,"output_tokens":365},"cost_usd":0.02803} +{"config":"voice","item":"f03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness is excellent: it mandates leading with a verdict (Approve/Request changes/Block) and ordering findings by severity. Concreteness is excellent: path:line citations, quoted offending expressions, exact fixes or commands, real numbers (allocation/query counts, benchmarks), reinforced by a strong good/bad example. Corporate tone is explicitly banned with a clear alternative persona ('terse senior engineer leaving PR comments, not a consultant writing a report'). AI vocabulary gets a specific banned word list plus the em dash ban, though it covers single words only and omits common AI phrases ('it's worth noting', 'leverage', 'seamless'), so a 4. On user outcomes the directive does the opposite of what the dimension asks: it explicitly forbids speculating about end users, customers, product impact, or business outcomes and instructs the agent to 'describe what the code does, not how anyone will experience it.' That is a deliberate scoping choice, but it scores 1 on this dimension."},"latency_ms":10300,"stop_reason":"end_turn","usage":{"input_tokens":734,"output_tokens":859},"cost_usd":0.05029} +{"config":"voice","item":"f02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness is explicit ('Lead with the point', 'Be direct about quality') and modeled in the good/bad examples, including taking a position ('Fix: add a null check... Two lines'). Concreteness is spelled out verbatim ('Name files, functions, line numbers, commands, outputs, evals, and real numbers') with a concrete auth.ts:47 example. Corporate/academic/PR/hype tone is explicitly banned, filler and throat-clearing named, and a clear alternative register is given ('builder talking to a builder, not a consultant presenting to a client') plus contrasting examples. User outcomes are directly required ('what the real user sees, loses, waits for, or can now do') and demonstrated ('Users hit a white screen'). The weak spot is AI vocabulary: there is no list of AI-tell words or phrases (no 'delve', 'leverage', 'robust', 'it's worth noting', 'certainly', etc.); the bad example only implicitly shows hedge-speak ('potential issue... may cause problems under certain conditions') and 'filler/throat-clearing' is too general to count as a specific ban list."},"latency_ms":10610,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":886},"cost_usd":0.05438} +{"config":"voice","item":"p02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'say so plainly', 'Don't dance around judgments', 'End with what to do', explicit examples like 'this is a mess'. Concreteness: dedicated section with before/after examples (file:line, exact commands like `bun test test/billing.test.ts`, real numbers like '~200ms per page load with 50 items'). Avoids corporate: explicitly bans corporate, academic, PR, hype, consultant tone and gives the alternative ('builder talking to a builder', 'typing fast', contextual energy levels). Avoids AI vocabulary: specific banned word list (delve, crucial, robust, etc.), banned phrase list, em dash ban, plus bans on filler and throat-clearing. Connects user outcomes: dedicated section with concrete examples ('3-second spinner', 'loses the customer's data'), 'make the user's user real', and the core philosophy ties all work back to shipping for real people. Every dimension is covered explicitly with examples rather than vague guidance."},"latency_ms":10555,"stop_reason":"end_turn","usage":{"input_tokens":2051,"output_tokens":407},"cost_usd":0.04086} +{"config":"voice","item":"p05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Open with status and action, in that order' and 'Commit to a call... say roll back, not a rollback could be considered' are explicit instructions to lead with the point and take positions, reinforced by the good/bad example. Concreteness: demands evidence for every claim with named dashboards, timestamped log lines, exact commands (`fly releases rollback v118`), and real numbers ('5xx at 7.2%, baseline 0.3%'); the good example models deploy IDs, pod names, percentages, and ETAs. Avoids corporate: explicitly bans 'status-page voice' with specific phrases ('we are aware of an issue impacting some users', 'out of an abundance of caution') and gives the alternative ('say what broke, who it hits, and what you are doing'), plus a contrasting bad example. Avoids AI vocabulary: provides a specific banned word list (delve, crucial, robust, comprehensive, nuanced, pivotal, landscape, seamless, leverage, furthermore, moreover, additionally) and banned phrases ('it's worth noting', 'let's dive in'). Connects user outcomes: 'Translate every metric into what customers are living through' with three concrete translations (failed checkouts/orders per minute, blank dashboard, slow logins), and the good example includes 'Users get an empty results page'. Every dimension is directly instructed with specifics and examples."},"latency_ms":8466,"stop_reason":"end_turn","usage":{"input_tokens":851,"output_tokens":527},"cost_usd":0.03486} +{"config":"voice","item":"a04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point', 'Be direct about quality', with a sharp good/bad contrast. Concreteness is explicit and exemplified (auth.ts:47, '/login', 'Two lines'). User outcomes are tied in directly ('what the real user sees, loses, waits for, or can now do') and the example shows it ('Users hit a white screen'). AI vocabulary gets a specific banned-word list plus a no-em-dash rule, though the list covers only single words (no banned phrases like 'it's worth noting' or 'leverage'), and the directive itself contains an em dash, undercutting the rule. The corporate dimension is the weak spot: rather than banning corporate/formal/academic tone, it actually prescribes a 'friendly, professional tone appropriate for stakeholders,' which pulls toward the register it should be rejecting. 'Avoid filler' and the bad example implicitly discourage hedging, but there is no explicit ban on corporate-speak and no stated alternative register beyond the examples."},"latency_ms":12729,"stop_reason":"end_turn","usage":{"input_tokens":1065,"output_tokens":1043},"cost_usd":0.0628} +{"config":"voice","item":"a05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing' all explicitly push for leading and taking positions. Concreteness: it names exactly what to cite (files, functions, line numbers, commands, outputs, evals, real numbers). Avoids corporate: it bans corporate, academic, PR, hype, filler, and throat-clearing and offers a positive alternative ('builder talking to a builder, not a consultant'), though it gives no concrete before/after examples of what the replacement sounds like. AI vocabulary: a specific, explicit banned-word list plus the em dash ban. User outcomes: directly instructs tying technical choices to 'what the real user sees, loses, waits for, or can now do' and closes with a strong note that the user holds the context and makes the decision."},"latency_ms":8110,"stop_reason":"end_turn","usage":{"input_tokens":723,"output_tokens":335},"cost_usd":0.02398} +{"config":"voice","item":"p04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: opens with 'Lead with the point,' demands positions on quality (no normalizing sloppy software), builder-to-builder framing. Concreteness: explicit standard with paired bad/good examples for commands, line numbers, and real numbers (bun test path, auth.ts:47, ~200ms N+1). Avoids corporate: bans corporate/academic/PR/hype and gives positive alternatives (YC partner, senior eng, technical blog post energy), but doesn't give concrete before/after rewrites of corporate phrasing the way it does for concreteness, so a 4. AI vocabulary: specific banned word list plus banned phrase list plus em dash rule. User outcomes: dedicated section with example sentences tying code to spinner time and data loss, and 'make the user's user real.'"},"latency_ms":8736,"stop_reason":"end_turn","usage":{"input_tokens":1046,"output_tokens":333},"cost_usd":0.02711} +{"config":"voice","item":"f04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is generic and does nothing the rubric asks for. It never tells the agent to lead with the point or take positions — 'explain thoroughly' actually pushes toward padding, not directness. There is no mention of naming files, commands, line numbers, or real numbers. It does not ban corporate/formal/academic tone or offer alternatives; 'friendly and positive' is itself a soft corporate register. No AI-tell words or phrases are listed or prohibited. Nothing connects technical work to real user experience or outcomes. Every dimension is at the floor."},"latency_ms":3688,"stop_reason":"end_turn","usage":{"input_tokens":368,"output_tokens":250},"cost_usd":0.01618} +{"config":"voice","item":"f03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness is excellent: the directive demands a verdict first (Approve/Request changes/Block), then findings ordered by severity, with a good/bad example contrasting a hedged statement against a specific claim. Concreteness is excellent: it requires path:line citations, quoted offending expressions, exact fixes or runnable commands, and real numbers (allocation counts, query counts, benchmarks). Corporate tone is explicitly banned and a concrete alternative persona is given (terse senior engineer leaving PR comments, not a consultant writing a report). AI vocabulary gets a specific banned word list plus an em dash ban, though the list covers only single words and omits common AI phrase patterns, so it scores 4 rather than 5. Connecting to user outcomes scores 1 because the directive does the opposite of what the dimension asks: it explicitly forbids discussing end users, customers, product impact, or how anyone will experience the code, telling the agent to stay strictly at the code level."},"latency_ms":10105,"stop_reason":"end_turn","usage":{"input_tokens":1247,"output_tokens":719},"cost_usd":0.04842} +{"config":"voice","item":"p03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' section demands first-sentence verdicts, explicit positions ('Ship it', 'Don't merge this'), bans warm-up and recap, and tells the agent what to do when unsure. Concreteness: 'Be concrete' section requires file:line, function name, exact command, real numbers, with examples for each and a 'never say X, say which check/input/output' rule. Avoids corporate: explicitly bans corporate, academic, PR, and hype tone, lists specific banned phrases, and gives the alternative (teammate at their desk, short sentences, plain verbs, actual opinion). Avoids AI vocabulary: dedicated section with a specific word list (22 words), a phrase list, and an em dash ban, plus a good/bad example contrasting the two voices. Connects user outcomes: 'Tie it to the user' requires naming what the real user sees/loses/waits for on every bug or change, gives a concrete contrast example, and handles the fallback case where no user-visible effect exists. All five dimensions are covered with explicit rules, specific lists, and worked examples."},"latency_ms":8197,"stop_reason":"end_turn","usage":{"input_tokens":1103,"output_tokens":431},"cost_usd":0.03258} +{"config":"voice","item":"a03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":4,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Bugs matter. Edge cases matter', plus a sharp good/bad pair that contrasts a hedged consultant sentence with a flat statement of fact. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example (auth.ts:47, /login, 'Two lines') models it. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives a positive alternative ('builder talking to a builder, not a consultant presenting to a client') plus the bounded closer rules against feature tours and self-justification. AI vocabulary: a specific 20-word ban list and a no-em-dash rule, but the list is words only (no banned phrases like 'it's worth noting' or 'I've identified a potential issue' are called out as patterns beyond the single example), and the directive itself contains an em dash in the exemption clause, undercutting the rule. User outcomes: 'why it matters, and what changes for the builder' and the example's 'Users hit a white screen' push toward real impact, but there is no explicit standing instruction to translate every technical finding into what the end user sees or feels; the bounded closer's 'what to watch' is operational rather than user-facing."},"latency_ms":13173,"stop_reason":"end_turn","usage":{"input_tokens":1585,"output_tokens":954},"cost_usd":0.06355} +{"config":"voice","item":"f05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is essentially the inverse of what a strong voice directive should do. Directness: it explicitly instructs hedging, avoiding verdicts, presenting multiple perspectives before any position, and framing bugs as gentle questions — the opposite of leading with the point and taking positions. Concreteness: no mention of naming files, commands, line numbers, or real numbers; it actively says to avoid technical detail in the opening and build up gradually, which suppresses specificity. Avoids_corporate: it doesn't ban corporate/formal tone — it mandates it, prescribing phrases like 'it may be worth considering' and 'this could potentially' that are hallmarks of corporate hedging, plus headings/bullets by default. Avoids_ai_vocabulary: no banned-word list at all; instead it recommends classic AI-tell hedge phrases as the house style. Connects_user_outcomes: nothing about tying technical work to real user experience; the focus is on tone-smoothing toward the user, not on what shipped code does for end users."},"latency_ms":7845,"stop_reason":"end_turn","usage":{"input_tokens":551,"output_tokens":407},"cost_usd":0.02586} +{"config":"voice","item":"f05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of every dimension. Directness: it explicitly tells the agent to avoid declaring answers, present perspectives before any view, let the user conclude, and frame bugs as gentle questions — the opposite of leading with the point and taking positions. Concreteness: no mention of files, commands, line numbers, or real numbers; it actually instructs the agent to avoid technical detail up front and 'build up to specifics gradually.' Avoids corporate: it mandates hedged, formal phrasing ('it may be worth considering', 'this could potentially') and headings/bullets, which is corporate/academic tone, with no ban or alternatives. Avoids AI vocabulary: it prescribes classic AI-tell hedges rather than banning them, and provides no list of forbidden words. Connects user outcomes: nothing links technical work to real user experience; the focus is on tone management toward the person asking, not on what breaks for end users."},"latency_ms":5491,"stop_reason":"end_turn","usage":{"input_tokens":1064,"output_tokens":374},"cost_usd":0.02934} +{"config":"voice","item":"a04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', plus a bounded closer rule and a good/bad pair showing position-taking vs hedging. Concreteness is explicit and well-exemplified: files, functions, line numbers, commands, outputs, evals, real numbers, and the auth.ts:47 example models it. User outcomes are directly addressed ('what the real user sees, loses, waits for, or can now do') and reinforced in the example ('Users hit a white screen'). The corporate-tone dimension is the weak spot: rather than banning corporate/formal/academic register, the directive instructs a 'friendly, professional tone appropriate for stakeholders', which actively pulls toward boardroom voice; the only counterweight is 'Avoid filler' and the bad example, with no explicit prohibition or named alternatives like casual/plain-spoken/builder voice. AI vocabulary gets a concrete banned list and an em dash ban, which is good, but it omits common AI-tell phrases (e.g. 'it's worth noting', 'in today's fast-paced', 'ensure', 'leverage', 'seamless', 'navigate') and the directive itself violates its own em dash rule in the bounded closer paragraph, which undercuts credibility."},"latency_ms":13383,"stop_reason":"end_turn","usage":{"input_tokens":1065,"output_tokens":1113},"cost_usd":0.0663} +{"config":"voice","item":"p04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: opens with 'Lead with the point' and frames the voice as a builder taking positions (YC partner energy, senior eng energy), explicitly serious about zero defects rather than hedging. Concreteness: strongest section, with explicit before/after examples for commands (`bun test test/billing.test.ts`), numbers (~200ms, N+1, 50 items), and file:line references (auth.ts:47). Avoids corporate: explicitly bans corporate, academic, PR, and hype tone and gives the 'builder talking to a builder, not a consultant presenting to a client' contrast plus positive alternatives by context; loses a point because it doesn't give concrete rewrite examples of corporate phrasing the way it does for concreteness. Avoids AI vocabulary: specific list of 20 banned words, a banned phrase list, and the em dash ban. Connects user outcomes: dedicated section with two concrete example sentences and the instruction to 'make the user's user real,' plus the zero-defects paragraph tying quality to real user impact."},"latency_ms":7659,"stop_reason":"end_turn","usage":{"input_tokens":1046,"output_tokens":412},"cost_usd":0.03106} +{"config":"voice","item":"a04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', plus a good/bad pair that contrasts a hedged non-statement with a flat diagnosis and fix, and a bounded closer rule that cuts unrequested prose. Concreteness is explicit and enumerated (files, functions, line numbers, commands, outputs, evals, real numbers) and modeled in the example with 'auth.ts:47' and 'Two lines'. User outcomes are directly required ('what the real user sees, loses, waits for, or can now do') and demonstrated ('Users hit a white screen'). The weak spot is corporate tone: the directive never bans corporate, formal, or academic register; it actually instructs 'Keep a friendly, professional tone appropriate for stakeholders', which pulls toward exactly the voice this dimension wants excluded, and offers no replacement register beyond 'Avoid filler'. The only alternative is implied by the example. AI vocabulary gets a solid specific word list and an em dash ban, but no banned phrases (hedging openers, 'It's worth noting', 'I hope this helps', etc.), and the directive itself contains an em dash in the closer exemption clause, which undercuts the rule."},"latency_ms":14666,"stop_reason":"end_turn","usage":{"input_tokens":1578,"output_tokens":1070},"cost_usd":0.06928} +{"config":"voice","item":"p02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Don't dance around judgments', 'If something is broken... say so plainly', 'End with what to do'. Explicit and repeated. Concreteness: a dedicated section with before/after examples (bun test command, N+1 ~200ms, auth.ts:47) plus a writing rule reinforcing real file/function names and numbers. Avoids corporate: explicitly bans corporate, academic, PR, hype, consultant-to-client framing, filler, throat-clearing, generic optimism, and gives the alternative ('builder talking to a builder', 'typing fast', short paragraphs, curious not lecturing, 'What's interesting here is...' beats 'It is important to understand...'). Avoids AI vocabulary: specific 20-word banned list, banned phrase list, and em dash ban. Connects user outcomes: dedicated section with example phrasings ('3-second spinner on every page load', 'loses the customer's data'), plus the core belief section pushing toward the user and job to be done. Every dimension is covered explicitly with examples, not just asserted."},"latency_ms":7389,"stop_reason":"end_turn","usage":{"input_tokens":2051,"output_tokens":440},"cost_usd":0.04251} +{"config":"voice","item":"p02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Don't dance around judgments', 'End with what to do', 'say so plainly'. Explicit and repeated. Concreteness: a dedicated 'Concreteness is the standard' section with contrasting examples (bun test test/billing.test.ts, auth.ts:47, ~200ms N+1), plus a writing rule reinforcing real files/functions/numbers. Avoids corporate: 'never corporate, never academic, never PR, never hype', 'builder talking to a builder, not a consultant presenting to a client', plus alternatives (YC partner energy, senior eng energy, technical blog post energy) and the 'What's interesting here is...' vs 'It is important to understand...' swap. Avoids AI vocabulary: explicit banned word list (delve, crucial, robust, etc.), banned phrase list ('here's the kicker', 'plot twist'), no em dashes, no self-referential AI humor. Connects user outcomes: dedicated 'Connect to user outcomes' section with example lines ('3-second spinner on every page load', 'loses the customer's data'), plus 'Make the user's user real' and the start-from-lived-experience rule. Minor gaps: no explicit 'take positions / commit to a recommendation' phrasing beyond quality judgments, and the AI vocabulary list could include a few more common tells, but all five dimensions are covered with specifics and examples."},"latency_ms":7796,"stop_reason":"end_turn","usage":{"input_tokens":2564,"output_tokens":546},"cost_usd":0.05294} +{"config":"voice","item":"f03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness is excellent: it mandates a verdict first (Approve/Request changes/Block), findings ordered by severity, and the good/bad example shows a committed claim versus a hedge. Concreteness is excellent: path:line citations, quoted offending expressions, exact fix or command, and real numbers (allocation/query counts, benchmarks). Avoids-corporate is strong but brief: it bans corporate/academic tone and supplies a clear alternative persona (terse senior engineer leaving PR comments vs consultant writing a report), though it does not illustrate what corporate phrasing looks like beyond the one bad example. AI-vocabulary is a solid specific word list plus the em dash ban, but it covers only single words and omits common AI phrases (e.g. 'it's worth noting', 'I hope this helps', 'in conclusion'). Connects-user-outcomes scores 1 because the directive explicitly forbids it: 'Do not speculate about end users, customers, product impact, or business outcomes... Describe what the code does, not how anyone will experience it.' This is a deliberate scoping decision for a code-reviewer role, but on this dimension it is the direct opposite of what is asked."},"latency_ms":11429,"stop_reason":"end_turn","usage":{"input_tokens":734,"output_tokens":914},"cost_usd":0.05304} +{"config":"voice","item":"f02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point,' 'Be direct about quality,' explicit good/bad examples showing a position-taking fix vs. hedged language — excellent. Concreteness: explicitly names 'files, functions, line numbers, commands, outputs, evals, and real numbers' and the example uses auth.ts:47, /login, 'Two lines' — excellent. Avoids corporate: explicitly bans corporate, academic, PR, hype, filler, throat-clearing, and gives the 'builder talking to a builder, not a consultant' alternative plus a bad-example sentence; loses a point because it doesn't offer a concrete list of corporate phrases to avoid or rewrites beyond one example. Avoids AI vocabulary: no specific list of AI-tell words (e.g., 'delve,' 'leverage,' 'robust,' 'potential issue,' 'it's worth noting'); the bad example implicitly shows hedging ('potential issue,' 'may cause problems under certain conditions') and 'filler/throat-clearing' is banned, but there is no explicit banned-word list — weak on this dimension. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' and the example 'Users hit a white screen' — excellent."},"latency_ms":6173,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":504},"cost_usd":0.03528} +{"config":"voice","item":"a03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":4,"reasoning":"Directness is explicit and reinforced: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', plus a good/bad pair that shows the contrast and a bounded-closer rule that cuts padding. Concreteness is named outright (files, functions, line numbers, commands, outputs, evals, real numbers) and demonstrated in the example with auth.ts:47 and 'Two lines.' Corporate tone is banned by name (corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay) and an alternative register is supplied: builder talking to a builder, not consultant to client. AI vocabulary gets a specific 19-word ban list plus the em dash rule; the list is solid though it omits a few common tells (leverage, seamless, 'it's worth noting'), and the directive itself slips an em dash into the bounded-closer paragraph. User outcomes are covered but mostly by implication: 'why it matters, and what changes for the builder' and the example line 'Users hit a white screen' model it well, but there is no standalone instruction telling the agent to tie every technical change back to what the end user experiences, so a 4 rather than 5."},"latency_ms":13297,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":963},"cost_usd":0.05887} +{"config":"voice","item":"p03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' section mandates first sentence = verdict, requires taking positions with concrete examples ('Ship it', 'Don't merge this'), bans warm-up and recap, and tells the agent how to handle uncertainty. Concreteness: explicitly names file:line format, function names, exact commands, and real numbers with examples, and bans vague phrasing with a specific anti-example. Avoids corporate: explicitly bans corporate, academic, PR, and hype tone, lists specific banned phrases, and gives a positive alternative (teammate at their desk, short sentences, plain verbs). Avoids AI vocabulary: dedicated section with a substantial word list and phrase list, plus no em dashes. Connects user outcomes: dedicated 'Tie it to the user' section requiring a user-visible effect for every bug/tradeoff/change, with a before/after example, and a fallback rule when no user effect can be named. The good/bad example at the end reinforces all five dimensions at once."},"latency_ms":7433,"stop_reason":"end_turn","usage":{"input_tokens":1103,"output_tokens":401},"cost_usd":0.03108} +{"config":"voice","item":"a05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', and 'builder talking to a builder' all push toward leading with the point and taking positions; the closing line on user authority doesn't undercut this. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, and real numbers. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, and offers 'builder talking to a builder, not a consultant presenting to a client' as the alternative; one point short because alternatives beyond that single contrast are thin. AI vocabulary: a specific, lengthy banned-word list plus a no-em-dash rule. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' is explicit and operational."},"latency_ms":6303,"stop_reason":"end_turn","usage":{"input_tokens":723,"output_tokens":348},"cost_usd":0.02463} +{"config":"voice","item":"a05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' plus 'Be direct about quality' and 'Fix the whole thing, not the demo path' explicitly push for leading with the conclusion and taking positions. Concreteness: it names exactly the categories asked for (files, functions, line numbers, commands, outputs, evals, real numbers). Avoids corporate: it bans corporate, academic, PR, hype, filler, throat-clearing, and founder cosplay, and offers an alternative framing ('builder talking to a builder, not a consultant presenting to a client'), though the alternative is a posture rather than concrete replacement phrasings, so a 4. AI vocabulary: an explicit, specific banned-word list plus a no-em-dash rule. User outcomes: a dedicated bullet instructing the agent to tie technical choices to what the real user sees, loses, waits for, or can now do, reinforced by 'what changes for the builder' in the opening bullet."},"latency_ms":6336,"stop_reason":"end_turn","usage":{"input_tokens":723,"output_tokens":381},"cost_usd":0.02628} +{"config":"voice","item":"a04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', plus a bounded-closer rule and a good/bad pair that model blunt, positioned statements. Concreteness is explicit and exhaustive: files, functions, line numbers, commands, outputs, evals, real numbers, reinforced by the auth.ts:47 example. User outcomes are directly required ('what the real user sees, loses, waits for, or can now do') and demonstrated ('Users hit a white screen'). AI vocabulary gets a real banned-word list and a no-em-dash rule, though it covers only single words (no banned phrases like 'it's worth noting', 'I hope this helps', hedges like 'potential issue... under certain conditions' appear only in the bad example rather than as named bans), and the directive itself violates its own em-dash rule in the exemption clause. The weak spot is corporate tone: rather than banning it, the directive actively prescribes 'a friendly, professional tone appropriate for stakeholders', which is corporate-adjacent language; the only anti-formal guidance is 'Avoid filler' and the bad example's hedging, with no explicit prohibition on academic or formal register and no stated alternative beyond the examples."},"latency_ms":15183,"stop_reason":"end_turn","usage":{"input_tokens":1578,"output_tokens":1063},"cost_usd":0.06893} +{"config":"voice","item":"a01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: 'Direct', 'No filler', 'End with what to do' are strong, but it never explicitly says to lead with the point or take positions; the closing 'the user decides' slightly softens the mandate to commit. Concreteness: explicitly names file, function, command, which is good, but omits line numbers and real numbers. Avoids corporate: 'Never corporate or academic' is an explicit ban, but it provides no alternatives or examples of what replaces that tone beyond 'builder-to-builder' and 'short paragraphs'. Avoids AI vocabulary: gives a specific word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus the em dash ban, though the list is short and no phrases are covered. Connects user outcomes: 'user-visible impact' is named as a required element, which is a direct instruction, but it is a single phrase with no elaboration on how to tie technical changes to real user experience."},"latency_ms":4875,"stop_reason":"end_turn","usage":{"input_tokens":465,"output_tokens":378},"cost_usd":0.02355} +{"config":"voice","item":"a05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Bugs matter. Edge cases matter' all push the agent to take positions and open with the conclusion. Concreteness: it explicitly lists files, functions, line numbers, commands, outputs, evals, and real numbers, which is exactly the target list. Avoids corporate: it bans corporate, academic, PR, hype, filler, throat-clearing, and generic optimism, and supplies a positive alternative ('builder talking to a builder, not a consultant presenting to a client'), though it does not give concrete before/after phrasing examples, so 4 rather than 5. Avoids AI vocabulary: it bans em dashes and gives a specific list of nineteen AI-tell words including delve, crucial, robust, comprehensive, furthermore, moreover, tapestry, landscape, and underscore. Connects user outcomes: the directive explicitly says to tie technical choices to what the real user sees, loses, waits for, or can now do, plus 'what changes for the builder', which is a strong, specific framing."},"latency_ms":9817,"stop_reason":"end_turn","usage":{"input_tokens":1236,"output_tokens":423},"cost_usd":0.03351} +{"config":"voice","item":"a05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' and 'Be direct about quality' with a stance on fixing the whole thing, not the demo path. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, and offers the builder-to-builder alternative, but gives no before/after examples of what to say instead, so a 4. AI vocabulary: an explicit no-em-dash rule plus a specific 19-word banned list. User outcomes: directly instructs tying technical choices to what the real user sees, loses, waits for, or can now do."},"latency_ms":5711,"stop_reason":"end_turn","usage":{"input_tokens":1236,"output_tokens":296},"cost_usd":0.02716} +{"config":"voice","item":"p05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: explicitly mandates status-then-action openings and committing to a call ('say roll back, not a rollback could be considered') with a concrete good/bad pair. Concreteness: demands dashboard names, timestamped log lines, exact commands, and numeric error rates with baselines; the example is dense with specifics (api-7, 01:52, deploy 4812, 12%, ETA 3 min). Avoids corporate: bans 'status-page voice' with named phrases ('aware of an issue impacting some users', 'abundance of caution') and gives the replacement rule (what broke, who it hits, what you're doing). Avoids AI vocabulary: provides an explicit banned-word list plus two filler phrases; solid but shorter than it could be (missing common tells like 'utilize', 'ensure', 'streamline', 'in today's', 'it is important to', em-dash overuse, 'I'd be happy to'), so a 4. Connects user outcomes: directly requires translating every metric into customer experience with multiple vivid examples (checkouts failing, blank dashboard, 9-second logins) and the good example models it."},"latency_ms":7978,"stop_reason":"end_turn","usage":{"input_tokens":851,"output_tokens":436},"cost_usd":0.03031} +{"config":"voice","item":"a02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'The user decides', plus a sharp good/bad pair that shows leading with the finding and the fix. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the auth.ts:47 example models it. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives a clear alternative framing (builder to builder, not consultant to client) with examples. Avoids AI vocabulary: it does ban em dashes and gives a list, but the list is only four words (delve, crucial, robust, comprehensive) and covers no AI-tell phrases ('it's worth noting', 'I hope this helps', 'leverage', 'seamless', etc.); worse, the directive itself contains an em dash in the Bounded closer paragraph, undercutting its own rule. Connects user outcomes: explicit instruction to tie choices to what the real user sees, loses, waits for, or can do, and the example ('Users hit a white screen') demonstrates it."},"latency_ms":12617,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":976},"cost_usd":0.05921} +{"config":"voice","item":"a04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', with a good/bad pair that shows hedging as the failure mode. Concreteness is explicit and exhaustive (files, functions, line numbers, commands, outputs, evals, real numbers) and the example demonstrates it with auth.ts:47 and 'Two lines.' User outcomes get a dedicated bullet with a specific framing (what the real user sees, loses, waits for, or can now do) and the example lands it with 'Users hit a white screen.' AI-vocabulary gets a concrete banned-word list plus the em dash ban, though it only lists words, not AI-tell phrases or structural tells (triplets, 'it's worth noting', sycophantic openers), and the directive itself slips an em dash into the closer section. Avoids_corporate is the weak spot: rather than banning corporate/formal/academic register, it actively prescribes 'a friendly, professional tone appropriate for stakeholders', which is itself corporate framing, and offers no named alternative register beyond 'avoid filler'. The bad example implies the target but the instruction text pulls the other way."},"latency_ms":12743,"stop_reason":"end_turn","usage":{"input_tokens":1065,"output_tokens":979},"cost_usd":0.0596} +{"config":"voice","item":"a03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness is strong: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', plus a bounded closer that cuts unrequested prose. Concreteness is explicit: name files, functions, line numbers, commands, outputs, evals, real numbers, backed by the auth.ts:47 example. Corporate tone is banned by name (corporate, academic, PR, hype, filler, throat-clearing, founder cosplay) with a clear alternative (builder talking to a builder vs consultant to client) and a good/bad pair. AI vocabulary gets a specific word list and an em dash ban, but it covers only single words, no AI-tell phrases (e.g. 'it's worth noting', 'I've identified a potential issue' appears only in the bad example, not the ban list), and the directive itself uses an em dash in the closer section. User outcomes are only implicit: 'why it matters, and what changes for the builder' and the example's 'Users hit a white screen' show the intent, but there is no explicit instruction to tie technical changes to end-user experience."},"latency_ms":13202,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":936},"cost_usd":0.05752} +{"config":"voice","item":"p02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Don't dance around judgments', 'say so plainly', 'End with what to do'. Explicit and repeated. Concreteness: dedicated section with before/after examples (bun test command, N+1 ~200ms, auth.ts:47), plus writing rule 'Real file names, real function names, real numbers.' Avoids corporate: tone line bans corporate, academic, PR, hype, with the alternative 'builder talking to a builder, not a consultant presenting to a client', plus context-matched energy modes and bans on filler, throat-clearing, generic optimism, founder cosplay. AI vocabulary: explicit list of 20 banned words, 7 banned phrases, em dash ban, and 'never self-referential about being AI'. User outcomes: dedicated 'Connect to user outcomes' section with two example sentences and 'Make the user's user real', reinforced by the core belief section on shipping for real people. Each dimension is addressed explicitly with examples, not just asserted."},"latency_ms":5687,"stop_reason":"end_turn","usage":{"input_tokens":2564,"output_tokens":416},"cost_usd":0.04644} +{"config":"voice","item":"f02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, plus a good/bad example that leads with the finding and a fix — strong. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers, and the example demonstrates 'auth.ts:47 ... Two lines' — excellent. Avoids corporate: explicitly bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and offers the 'builder talking to a builder, not a consultant' alternative plus a bad example of hedged consultant-speak; loses a point for not giving concrete replacement phrasings beyond the examples. Avoids AI vocabulary: no specific list of AI-tell words (e.g., 'delve', 'leverage', 'robust', 'I hope this helps', 'certainly'); the bad example implicitly discourages hedging ('potential issue', 'may cause problems under certain conditions') and the bounded closer cuts preamble/restatement, but there is no explicit vocabulary ban — weak on this dimension. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' plus 'Users hit a white screen' in the example — excellent."},"latency_ms":7174,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":509},"cost_usd":0.03553} +{"config":"voice","item":"p04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: opens with 'Lead with the point', tone list includes direct/sharp, and the quality section takes firm positions (zero defects, fix the whole thing). Concreteness: explicitly demands file, function, line number, exact commands (bun test example), real numbers (N+1 ~200ms), and before/after examples for each. Avoids corporate: bans corporate, academic, PR, hype and gives the builder-to-builder alternative plus context-matched energy models (YC partner, senior eng, tech blog), though it offers fewer concrete phrase-level substitutions than it does for other dimensions. Avoids AI vocabulary: enumerates a specific list of ~20 banned words, a list of banned phrases, and bans em dashes with alternatives. Connects user outcomes: dedicated section with two concrete example sentences and 'make the user's user real', reinforced by the opening line about caring whether it works for users."},"latency_ms":6975,"stop_reason":"end_turn","usage":{"input_tokens":1559,"output_tokens":369},"cost_usd":0.03404} +{"config":"voice","item":"a05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', take positions on bugs and edge cases, builder-to-builder framing. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, and offers the alternative 'builder talking to a builder, not a consultant presenting to a client', though it gives no concrete before/after rewrites. AI vocabulary: explicit list of 19 AI-tell words plus em dash ban. User outcomes: 'what the real user sees, loses, waits for, or can now do' is a precise instruction to connect technical work to lived user experience."},"latency_ms":6180,"stop_reason":"end_turn","usage":{"input_tokens":723,"output_tokens":314},"cost_usd":0.02293} +{"config":"voice","item":"a06","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong ('Direct, concrete, builder-to-builder', 'No filler', 'End with what to do') but it never explicitly says to lead with the point or take positions. Concreteness names file, function, command, and user-visible impact, but omits line numbers and real numbers/measurements. Corporate/academic tone is explicitly banned, but alternatives are thin beyond 'builder-to-builder' and 'short paragraphs'. AI vocabulary ban is specific (six words plus em dashes) but the list is short and covers no phrases. User-outcome connection is excellent: 'Tie every finding to what the user sees, waits for, or loses' is precise and actionable."},"latency_ms":7498,"stop_reason":"end_turn","usage":{"input_tokens":486,"output_tokens":642},"cost_usd":0.03696} +{"config":"voice","item":"p01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', and the good/bad example pair model leading with the finding rather than hedging. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example (auth.ts:47, /login, 'Two lines') demonstrates it. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and offers a clear alternative register ('builder talking to a builder, not a consultant presenting to a client') plus a bad-example sentence. AI vocabulary: a specific banned-word list of 19 terms plus no em dashes, which is strong, but it lists only single words (no banned phrases like 'it's worth noting' or 'I've identified a potential issue' as a pattern), and the directive itself breaks its own em dash rule in the bounded closer paragraph, so 4. User outcomes: explicit instruction to tie technical choices to 'what the real user sees, loses, waits for, or can now do', reinforced by the example ('Users hit a white screen')."},"latency_ms":12986,"stop_reason":"end_turn","usage":{"input_tokens":1107,"output_tokens":955},"cost_usd":0.05882} +{"config":"voice","item":"a01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: 'Direct, concrete, builder-to-builder... No filler... End with what to do' is strong and explicit. Concreteness: names file, function, command explicitly, but omits line numbers and real numbers/measurements. Avoids corporate: bans corporate and academic tone in one line but gives no alternatives or examples of what to write instead. Avoids AI vocabulary: provides a specific banned word list plus em dashes, though the list is short and skips common phrase-level tells (e.g., 'it's worth noting', 'leverage'). Connects user outcomes: 'user-visible impact' is named as a required element, which is a real instruction, but it is a single phrase with no guidance on how to translate technical changes into what users actually experience."},"latency_ms":4391,"stop_reason":"end_turn","usage":{"input_tokens":465,"output_tokens":311},"cost_usd":0.0202} +{"config":"voice","item":"f02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, bounded closer rule, plus a good/bad example — strong. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example (auth.ts:47, /login, 'Two lines') models it — strong. Avoids corporate: explicitly bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the builder-not-consultant alternative plus a bad example; slightly short of 5 because alternatives are illustrated rather than enumerated (e.g., no 'say X instead of Y' list). Avoids AI vocabulary: the bad example implicitly flags hedging ('potential issue', 'may cause problems under certain conditions') and 'filler/throat-clearing' is named, but there is no explicit list of AI-tell words or phrases (delve, leverage, robust, 'it's worth noting', 'I hope this helps', etc.) — weak on this dimension. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' and the example ('Users hit a white screen') — strong."},"latency_ms":7784,"stop_reason":"end_turn","usage":{"input_tokens":1521,"output_tokens":488},"cost_usd":0.03961} +{"config":"voice","item":"f04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is two generic sentences with no actionable guidance on any dimension. It never tells the agent to lead with the point or take positions; 'explain things thoroughly' and 'keep a positive tone' actually push toward padding and hedging rather than directness. There is no mention of naming files, commands, line numbers, or real numbers. It does not ban corporate/formal/academic tone or offer alternatives — 'helpful, friendly, clear' is itself boilerplate. No AI-tell words or phrases are listed or prohibited. Nothing connects technical work to user experience or product outcomes. This is essentially a default-assistant persona, not a voice directive."},"latency_ms":4312,"stop_reason":"end_turn","usage":{"input_tokens":368,"output_tokens":282},"cost_usd":0.01778} +{"config":"voice","item":"p02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality... Don't dance around judgments', 'If something is broken... say so plainly', 'End with what to do.' Explicit and repeated. Concreteness: has a dedicated section with paired bad/good examples (vague 'you should test this' vs `bun test test/billing.test.ts`, 'might be slow' vs '~200ms per page load with 50 items', 'issue in auth flow' vs 'auth.ts:47...'). Names files, functions, line numbers, real numbers explicitly. Avoids corporate: 'never corporate, never academic, never PR, never hype', plus alternatives ('builder talking to a builder, not consultant to client', YC partner / senior eng / technical blog energy), bans filler and throat-clearing, and gives a lecturing-vs-curious phrasing swap. Avoids AI vocabulary: explicit list of ~20 AI-tell words (delve, crucial, robust, tapestry, etc.) plus a separate banned-phrases list and an em-dash ban. Connects user outcomes: dedicated section with concrete example sentences ('3-second spinner on every page load', 'the edge case you're skipping is the one that loses the customer's data'), plus 'Make the user's user real' and the overall framing around the user and job to be done. Minor nitpick: the YC-application flattery clause is slightly off-mission, but it doesn't weaken any scored dimension."},"latency_ms":9341,"stop_reason":"end_turn","usage":{"input_tokens":2051,"output_tokens":550},"cost_usd":0.04801} +{"config":"voice","item":"f05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what the rubric rewards on every axis. Directness: it explicitly tells the agent to avoid taking positions, present perspectives before any view, let the user reach their own conclusion, and frame bugs as gentle questions—the opposite of leading with the point. Concreteness: no mention of files, commands, line numbers, or real numbers; it actively says to avoid technical detail in the opening and 'build up gradually.' Avoids corporate: it mandates hedged, formal register ('it may be worth considering', 'this could potentially') and heavy headings/bullets rather than banning that tone. Avoids AI vocabulary: no banned-word list; the exemplar phrases it prescribes ('one possible approach', 'could potentially') are classic AI-tell hedges. Connects user outcomes: nothing about tying technical work to real user experience."},"latency_ms":5308,"stop_reason":"end_turn","usage":{"input_tokens":1064,"output_tokens":351},"cost_usd":0.02819} +{"config":"voice","item":"f03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness is excellent: it mandates a verdict first (Approve/Request changes/Block) and severity-ordered findings. Concreteness is excellent: path:line citations, quoted offending expressions, exact fixes or commands, and real numbers (allocation/query counts, benchmarks), reinforced by a good/bad example. Corporate tone is explicitly banned with a concrete alternative persona (terse senior engineer leaving PR comments vs. consultant report). AI vocabulary has a specific banned-word list plus an em dash ban, though it covers only single words and omits common AI phrases (e.g. 'it's worth noting', 'in today's world'). Connecting to user outcomes scores 1 because the directive does the opposite: it explicitly forbids speculating about end users, customers, product impact, or business outcomes and instructs the agent to describe what the code does rather than how anyone will experience it."},"latency_ms":9154,"stop_reason":"end_turn","usage":{"input_tokens":734,"output_tokens":700},"cost_usd":0.04234} +{"config":"voice","item":"p05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Open with status and action, in that order' and 'Commit to a call... say roll back, not a rollback could be considered' directly mandate leading with the point and taking positions; the Good/Bad examples reinforce it. Concreteness: demands dashboard names, timestamped log lines, exact commands (`fly releases rollback v118`), and numeric error rates with baselines; the Good example models it with api-7, deploy 4812, 01:52, 12%. Avoids corporate: explicitly bans 'status-page voice' with named phrases ('we are aware of an issue impacting some users', 'out of an abundance of caution') and gives the alternative (say what broke, who it hits, what you're doing); the Bad example is a clear corporate anti-pattern. Avoids AI vocabulary: provides a specific word list (delve, crucial, robust, comprehensive, nuanced, pivotal, landscape, seamless, leverage, furthermore, moreover, additionally) plus phrases ('it's worth noting', 'let's dive in'); solid but shorter than a comprehensive ban (no mention of e.g. 'in today's fast-paced', 'game-changer', em-dash overuse, hedging qualifiers), so a 4. Connects user outcomes: 'Translate every metric into what customers are living through' with three concrete translations, and the Good example ends with 'Users get an empty results page' — explicit and well-modeled."},"latency_ms":8350,"stop_reason":"end_turn","usage":{"input_tokens":851,"output_tokens":540},"cost_usd":0.03551} +{"config":"voice","item":"p03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'first sentence = the answer', demands positions with examples ('Ship it', 'Don't merge this'), bans warm-up and recap, and tells the agent what to do when unsure. Concreteness: names file:line, function, command, and real-number formats with examples, and explicitly forbids vague phrasing with a before/after. Avoids corporate: bans corporate/academic/PR/hype tone with a specific banned phrase list and gives a clear alternative (teammate at their desk, short sentences, plain verbs, actual opinion). Avoids AI vocabulary: dedicated section with a long explicit word list, a phrase list, and the em dash ban, plus a good/bad example pair. User outcomes: 'Tie it to the user' section requires a user-visible effect for every bug/tradeoff/change, gives a concrete contrast example, and handles the case where no user effect exists. Closing line keeps the user as decision-maker."},"latency_ms":7173,"stop_reason":"end_turn","usage":{"input_tokens":1103,"output_tokens":380},"cost_usd":0.03003} +{"config":"voice","item":"f04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is generic and does the opposite of what a strong GStack voice directive should do. It asks for 'helpful, friendly, clear' with a 'positive tone' and thorough explanation — which actively encourages hedging, padding, and cheerful filler rather than leading with the point or taking positions (directness: 1). It says nothing about citing files, commands, line numbers, or real numbers (concreteness: 1). It does not ban corporate, formal, or academic tone, nor offer alternatives; 'feels supported' nudges toward customer-service register (avoids_corporate: 1). There is no list of AI-tell words or phrases to avoid (avoids_ai_vocabulary: 1). It never connects technical work to what users actually experience (connects_user_outcomes: 1). Essentially a placeholder with no operational guidance on any scored dimension."},"latency_ms":4856,"stop_reason":"end_turn","usage":{"input_tokens":368,"output_tokens":347},"cost_usd":0.02103} +{"config":"voice","item":"a05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'Lead with the point', 'Be direct about quality', and a stance-taking posture (bugs matter, fix the whole thing) earn a 5. Concreteness: it literally enumerates files, functions, line numbers, commands, outputs, evals, and real numbers, so 5. Avoids corporate: it bans corporate, academic, PR, and hype tone and names filler, throat-clearing, generic optimism, and founder cosplay as things to avoid, and offers a positive alternative ('builder talking to a builder, not a consultant presenting to a client'). It's a 4 rather than 5 because the alternative is a single framing rather than concrete substitutions or examples of what the corporate version looks like versus the preferred version. Avoids AI vocabulary: a specific, well-chosen banned word list plus the em dash rule, 5. Connects user outcomes: explicit instruction to tie technical choices to 'what the real user sees, loses, waits for, or can now do', which is unusually specific and actionable, 5."},"latency_ms":7469,"stop_reason":"end_turn","usage":{"input_tokens":723,"output_tokens":409},"cost_usd":0.02768} +{"config":"voice","item":"a03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":4,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', and the bounded closer rule all push the agent to state positions up front; the good/bad example makes the contrast explicit. Concreteness: it names exactly what to cite (files, functions, line numbers, commands, outputs, evals, real numbers) and the example shows auth.ts:47, /login, 'Two lines'. Avoids corporate: explicitly bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and supplies the alternative register ('builder talking to a builder, not a consultant presenting to a client') with before/after samples. Avoids AI vocabulary: a specific banned-word list plus a no-em-dash rule, but the list is words only (no banned phrases like 'it's worth noting' or 'I hope this helps'), and the directive itself violates its own em dash rule in the bounded closer paragraph, which weakens it. Connects user outcomes: 'why it matters, and what changes for the builder' plus the example's 'Users hit a white screen' model the behavior, but there is no explicit standalone instruction to tie technical findings to real user experience; it is shown more than stated."},"latency_ms":12866,"stop_reason":"end_turn","usage":{"input_tokens":1585,"output_tokens":928},"cost_usd":0.06225} +{"config":"voice","item":"f01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension measures. Directness: it mandates opening with acknowledgment and a summary of understanding, requires a 'comprehensive overview of relevant considerations' before any recommendation, and explicitly bans strong opinions and sentence fragments — burying the lede and forbidding positions. Concreteness: no instruction to cite files, commands, line numbers, or numbers; the example ('several opportunities for enhancement') is vague by design. Avoids corporate: it actively requires a formal stakeholder register and prescribes corporate filler ('We are pleased to report', 'It is important to note', 'Moving forward, we recommend'). Avoids AI vocabulary: it endorses AI-tell phrases ('robust, scalable value', 'aligns with best practices', 'may be worth considering as part of a broader modernization initiative') rather than banning any. Connects user outcomes: it frames everything around stakeholders and executive summaries, never around what a real user experiences when the code runs."},"latency_ms":7599,"stop_reason":"end_turn","usage":{"input_tokens":1186,"output_tokens":400},"cost_usd":0.03186} +{"config":"voice","item":"a03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":4,"reasoning":"Directness is excellent: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, and a bounded closer rule that cuts unrequested prose, all reinforced by paired good/bad examples. Concreteness is explicit and specific: files, functions, line numbers, commands, outputs, evals, real numbers, with the auth.ts:47 example modeling exactly that. Corporate tone is banned by name (corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay) and the alternative is stated clearly (builder, not consultant). AI vocabulary gets a concrete 20-word blocklist plus the em dash ban, which is strong, but it stops at single words and omits common AI phrase patterns (hedges like 'potential issue... under certain conditions' appear only implicitly in the bad example), and the directive itself uses an em dash in the exemption clause, slightly undercutting the rule. User-outcome connection is present ('why it matters, and what changes for the builder'; 'Users hit a white screen' in the example) but is modeled more than mandated; there is no explicit standing instruction to tie each technical finding to end-user impact, so the example does most of the work."},"latency_ms":13992,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":992},"cost_usd":0.06032} +{"config":"voice","item":"p01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is explicit ('Lead with the point', 'Be direct about quality', builder-to-builder framing) and reinforced by the bounded closer rule and good/bad examples. Concreteness is spelled out with a full list (files, functions, line numbers, commands, outputs, evals, real numbers) and demonstrated in the auth.ts:47 example. Corporate/academic/PR/hype tone is explicitly banned, filler and throat-clearing are named, and the alternative is given both as a stance ('builder talking to a builder, not a consultant') and as a concrete good/bad pair. AI vocabulary gets a specific 19-word banned list plus a no-em-dash rule, but it only lists single words and omits common AI-tell phrases ('it's worth noting', 'in today's fast-paced', 'at the end of the day'), and the directive itself slips an em dash into the bounded-closer exemption clause, which slightly undercuts the rule. User outcomes are directly required ('what the real user sees, loses, waits for, or can now do') and modeled in the example ('Users hit a white screen')."},"latency_ms":12317,"stop_reason":"end_turn","usage":{"input_tokens":1107,"output_tokens":877},"cost_usd":0.05492} +{"config":"voice","item":"p04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: opens with 'Lead with the point,' demands saying what it does and why it matters, and the tone section stresses 'direct, sharp' and taking craft seriously rather than hedging; the zero-defects stance is itself a position. Concreteness: explicitly requires file, function, and line numbers, gives a literal command example (`bun test test/billing.test.ts`), demands real numbers (~200ms, N+1), and contrasts vague vs. exact phrasing with auth.ts:47. Avoids corporate: bans corporate, academic, PR, and hype tone and offers the 'builder talking to a builder, not consultant to client' alternative plus context-matched energy models; slightly short of 5 because it doesn't show concrete before/after rewrites of corporate phrasing the way it does for concreteness. Avoids AI vocabulary: a specific 20-word banned list plus a banned-phrase list and an em dash rule, which is exactly what the dimension asks for. Connects user outcomes: dedicated section instructing regular connection to real user experience with two example sentences and the 'make the user's user real' framing."},"latency_ms":8495,"stop_reason":"end_turn","usage":{"input_tokens":1046,"output_tokens":435},"cost_usd":0.03221} +{"config":"voice","item":"p04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: opens with 'Lead with the point', demands saying what it does and why it matters, and the tone section names 'direct' first. Concreteness: explicit paragraph requiring file, function, line number, exact commands (bun test example), real numbers (N+1 ~200ms), and a before/after example for pointing at auth.ts:47. Avoids corporate: bans corporate, academic, PR, and hype explicitly, and offers concrete alternatives (builder-to-builder, YC partner energy, senior eng energy, technical blog post energy). AI vocabulary: a specific list of 20 banned words plus a list of banned filler phrases and the em dash ban. User outcomes: dedicated section with two concrete examples (3-second spinner, lost customer data) and the instruction to 'make the user's user real'. Minor gaps: the directness dimension could more explicitly tell the agent to take positions and avoid hedging, and the corporate section could list specific corporate phrases to avoid, but every dimension is covered strongly."},"latency_ms":7277,"stop_reason":"end_turn","usage":{"input_tokens":1559,"output_tokens":404},"cost_usd":0.03579} +{"config":"voice","item":"f03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness is excellent: it mandates leading with a verdict (Approve/Request changes/Block) and ordering findings by severity, which forces positions up front. Concreteness is excellent: it requires path:line citations, quoted offending expressions, exact fixes or runnable commands, and real numbers (allocation counts, query counts, benchmarks), reinforced by a good/bad example. Corporate tone is explicitly banned with a clear alternative ('terse senior engineer leaving PR comments, not a consultant writing a report'). AI vocabulary gets a specific banned list of 14 words plus em dashes, which is solid though not exhaustive (no phrase-level bans like 'it's worth noting' or words like 'leverage', 'seamless'). On user outcomes the directive does the opposite of what is asked: it explicitly forbids speculating about end users, customers, product impact, or business outcomes and tells the agent to describe what code does rather than how anyone experiences it. That is a deliberate scope choice, but it scores minimally on this dimension."},"latency_ms":9942,"stop_reason":"end_turn","usage":{"input_tokens":1247,"output_tokens":721},"cost_usd":0.04852} +{"config":"voice","item":"a02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness is explicit and modeled: 'Lead with the point', 'Be direct about quality', 'builder talking to a builder', plus a bounded closer rule that cuts unrequested prose, with sharp good/bad examples. Concreteness is a full list (files, functions, line numbers, commands, outputs, evals, real numbers) and the auth.ts:47 example demonstrates it. Corporate tone is banned by name (corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay) and an alternative register is given (builder-to-builder vs consultant-to-client), reinforced by the bad example. User outcomes are tied in explicitly ('what the real user sees, loses, waits for, or can now do') and shown ('Users hit a white screen'). The weak spot is AI vocabulary: only four words are listed (delve, crucial, robust, comprehensive), no banned phrases or hedging patterns beyond the one bad example, and the directive itself uses an em dash in the bounded-closer paragraph right after banning them, which undercuts the rule."},"latency_ms":10800,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":881},"cost_usd":0.05446} +{"config":"voice","item":"f02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'Lead with the point', 'Be direct about quality', 'Say what it does, why it matters', plus a good/bad example contrasting hedged vs. flat statements — 5. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers, and the example shows auth.ts:47 and a two-line fix — 5. Avoids corporate: bans corporate/academic/PR/hype, filler, throat-clearing, generic optimism, 'founder cosplay', and gives an alternative framing ('builder talking to a builder, not a consultant presenting to a client') with examples; loses a point for not giving a concrete list of banned corporate phrasings or explicit tone replacements beyond the one example — 4. Avoids AI vocabulary: there is no explicit list of AI-tell words (e.g., 'delve', 'leverage', 'robust', 'it's worth noting', 'certainly', 'great question'). The bad example implicitly discourages hedging ('potential issue... may cause problems under certain conditions') and 'filler/throat-clearing' gestures at it, but no specific banned vocabulary is provided — 2. Connects user outcomes: explicit 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', reinforced by the example ('Users hit a white screen') — 5."},"latency_ms":6732,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":528},"cost_usd":0.03648} +{"config":"voice","item":"a05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'builder talking to a builder' all instruct leading with the point and taking positions; the closing line on user deciding tempers it slightly but still frames agent output as recommendations with a stance. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, and gives the builder-to-builder alternative, but the alternative is a single framing rather than concrete replacements, so 4. AI vocabulary: specific banned list of 20 words plus em dash ban. User outcomes: 'what the real user sees, loses, waits for, or can now do' is a precise instruction to connect technical work to user experience."},"latency_ms":6694,"stop_reason":"end_turn","usage":{"input_tokens":723,"output_tokens":337},"cost_usd":0.02408} +{"config":"voice","item":"f03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness is excellent: lead with a verdict (Approve/Request changes/Block), findings ordered by severity, and the good/bad example shows a committed claim versus a hedge. Concreteness is excellent: path:line citations, quoted offending expression, exact fix or command, and real numbers (allocation counts, query counts, benchmark output). Corporate tone is explicitly banned with a clear alternative persona (terse senior engineer leaving PR comments, not a consultant writing a report). AI vocabulary gets a specific banned-word list plus an em dash ban, though the list is moderate in length and omits common phrase-level tells (e.g., 'it's worth noting', 'I hope this helps', 'in today's world'). Connecting to user outcomes scores the minimum because the directive does the opposite: it explicitly forbids speculating about end users, customers, product impact, or business outcomes and tells the agent to describe what the code does rather than how anyone experiences it."},"latency_ms":9304,"stop_reason":"end_turn","usage":{"input_tokens":1247,"output_tokens":602},"cost_usd":0.04257} +{"config":"voice","item":"f01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of every dimension. Directness: it mandates opening with acknowledgment and summary, front-loads a 'comprehensive overview' before any recommendation, and explicitly bans 'strong opinions' — the opposite of leading with the point and taking positions. Concreteness: zero mention of files, commands, line numbers, or numbers; the example is pure abstraction ('several opportunities for enhancement'). Avoids corporate: it actively prescribes a formal corporate register and bans casual language and fragments, rather than banning corporate tone. Avoids AI vocabulary: it recommends AI-tell phrases ('It is important to note', 'robust, scalable value', 'best practices', 'Moving forward') instead of banning them. Connects user outcomes: no instruction to tie work to real user experience — only to stakeholder perception and 'modernization initiatives'. Every bullet pushes toward hedged, padded, generic output."},"latency_ms":4759,"stop_reason":"end_turn","usage":{"input_tokens":673,"output_tokens":380},"cost_usd":0.02573} +{"config":"voice","item":"a04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":4,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' and 'Be direct about quality' plus the auth.ts example are strong, though it never explicitly tells the agent to take a position or state what it would do; the user-decides clause is fine but there's no 'commit to a recommendation' instruction. Concreteness: explicit list (files, functions, line numbers, commands, outputs, evals, real numbers) and a worked example with auth.ts:47 and 'two lines'. Avoids corporate: this is the weak spot. The directive says 'Keep a friendly, professional tone appropriate for stakeholders', which actively nudges toward corporate register rather than banning it; there is no explicit prohibition on formal/academic/corporate tone and no alternative offered beyond 'avoid filler' and one bad example. Avoids AI vocabulary: a concrete banned-word list and a no-em-dash rule, but only single words, no banned phrases (hedges like 'potential issue', 'it's worth noting', 'under certain conditions' are only implied by the bad example), and the directive itself uses an em dash. Connects user outcomes: explicit and well-phrased ('what the real user sees, loses, waits for, or can now do') with the white-screen example reinforcing it."},"latency_ms":14910,"stop_reason":"end_turn","usage":{"input_tokens":1065,"output_tokens":1219},"cost_usd":0.0716} +{"config":"voice","item":"f01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it mandates opening with acknowledgment and a summary of understanding, requires a 'comprehensive overview' before any recommendation, and explicitly bans 'strong opinions' — burying the point and forbidding positions. Concreteness: nothing about files, commands, line numbers, or numbers; the example is pure abstraction ('several opportunities for enhancement... broader modernization initiative'). Avoids corporate: it actively prescribes corporate/formal register ('polished, professional register for stakeholders', 'courteous, formal tone at all times') and bans the casual alternatives. Avoids AI vocabulary: it recommends AI-tell phrases outright ('It is important to note', 'Moving forward, we recommend', 'robust, scalable value', 'best practices', 'enhancement') rather than banning them. Connects user outcomes: it talks about aligning with best practices and delivering 'value' to stakeholders, never about what actual users experience. Lowest possible score on every axis."},"latency_ms":5298,"stop_reason":"end_turn","usage":{"input_tokens":673,"output_tokens":414},"cost_usd":0.02743} +{"config":"voice","item":"p01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', and the good/bad pair shows a position-taking fix vs hedged mush; the user-decides clause bounds authority without softening tone. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example demonstrates auth.ts:47 with a two-line fix. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the alternative register ('builder talking to a builder, not a consultant presenting to a client') plus a bounded closer rule with good/bad examples. Avoids AI vocabulary: concrete 20-word banlist plus em dash ban, though the directive itself slips an em dash into the closer paragraph, a minor self-inconsistency. Connects user outcomes: 'what the real user sees, loses, waits for, or can now do' plus the example's 'Users hit a white screen' ties the bug to user experience directly."},"latency_ms":12681,"stop_reason":"end_turn","usage":{"input_tokens":1107,"output_tokens":937},"cost_usd":0.05792} +{"config":"voice","item":"p03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' section explicitly puts the answer first, demands positions with examples (Ship it / Don't merge), bans warm-up and recap, and tells the agent how to handle uncertainty. Concreteness: names files with line numbers, functions, exact commands, and real numbers, plus a before/after example and a ban on vague phrasing. Avoids corporate: explicitly bans corporate, academic, PR, and hype tone, lists sample phrases, and gives the alternative (teammate at their desk, short sentences, plain verbs). Avoids AI vocabulary: dedicated section with a specific word list, a phrase list, and an em dash ban. Connects user outcomes: 'Tie it to the user' requires naming what the user sees, loses, waits for, or can do for every bug/tradeoff/change, with a concrete contrast and a fallback when no user-visible effect exists. The good/bad example reinforces all five dimensions at once."},"latency_ms":7152,"stop_reason":"end_turn","usage":{"input_tokens":1616,"output_tokens":385},"cost_usd":0.03541} +{"config":"voice","item":"p04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'builder talking to a builder', and strong position-taking on quality (zero defects, no hand-waving) make this a 5. Concreteness: explicit instructions to name file, function, line number, exact commands (`bun test test/billing.test.ts`), real numbers (~200ms per page load), and a worked example (auth.ts:47) — exemplary. Avoids corporate: it bans corporate, academic, PR, and hype tone and offers the alternative framing (builder-to-builder, YC partner / senior eng / technical blog energy), but doesn't give concrete before/after rewrites of corporate phrasing, so a 4 rather than 5. Avoids AI vocabulary: a specific 20-word banned list plus a banned-phrase list and the em dash rule — 5. Connects user outcomes: a dedicated section with concrete example sentences ('3-second spinner', 'loses the customer's data') and 'make the user's user real' — 5."},"latency_ms":7618,"stop_reason":"end_turn","usage":{"input_tokens":1046,"output_tokens":393},"cost_usd":0.03011} +{"config":"voice","item":"f02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, plus a concrete good/bad example — excellent. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers, and the example shows 'auth.ts:47' — excellent. Avoids corporate: explicitly bans corporate, academic, PR, hype, filler, throat-clearing, founder cosplay, and offers the 'builder talking to a builder' alternative and a bad/good contrast; loses a point for not giving specific replacement phrasings beyond the single example. Avoids AI vocabulary: no specific list of AI-tell words ('delve', 'leverage', 'robust', 'it's worth noting', etc.); the bad example hints at hedging language ('potential issue', 'may cause problems under certain conditions') but there is no enumerated ban list. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' plus the white-screen example — excellent."},"latency_ms":4959,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":424},"cost_usd":0.03128} +{"config":"voice","item":"a01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":4,"concreteness":3,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: 'Direct, concrete, builder-to-builder... No filler... End with what to do' pushes leading with the point and action, though it does not explicitly say to take positions or lead with the conclusion. Concreteness: it says 'Name the file, function, command' but omits line numbers and real numbers/measurements. Avoids corporate: 'Never corporate or academic' is a flat ban with no examples of what that looks like or what to say instead beyond 'short paragraphs'. AI vocabulary: it gives a specific list (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus bans em dashes, but the list is short and skips common phrase-level tells (e.g., 'it's worth noting', 'let's dive in', 'leverage', 'seamless'). User outcomes: 'user-visible impact' is named once as something to state, but there is no instruction to explain how a change feels to real users or why it matters to them beyond that phrase. The final paragraph on user authority is good framing but outside these five dimensions."},"latency_ms":7813,"stop_reason":"end_turn","usage":{"input_tokens":978,"output_tokens":409},"cost_usd":0.03023} +{"config":"voice","item":"p01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', the good/bad example contrasts a hedged consultant sentence with a flat statement of fact and fix. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example demonstrates it (auth.ts:47, /login, 'Two lines'). Avoids corporate: names corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay as banned, and gives the alternative framing (builder to builder, not consultant to client) plus a worked bad example. AI vocabulary: bans em dashes and gives a specific 19-word list. User outcomes: 'what the real user sees, loses, waits for, or can now do' and the example ties the bug to 'Users hit a white screen.' The bounded closer section with exemptions is well-scoped and keeps the directive from becoming self-undermining. One minor irony: the exemption clause itself uses an em dash, but it does not affect the quality of the instruction."},"latency_ms":8027,"stop_reason":"end_turn","usage":{"input_tokens":1620,"output_tokens":417},"cost_usd":0.03705} +{"config":"voice","item":"a03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":3,"reasoning":"Directness is explicit and reinforced: 'Lead with the point', 'Be direct about quality', 'builder talking to a builder', plus a good/bad pair contrasting a blunt diagnosis with hedged consultant-speak, and a bounded closer rule that cuts unrequested prose. Concreteness is spelled out as a checklist (files, functions, line numbers, commands, outputs, evals, real numbers) and demonstrated with 'auth.ts:47' and 'Two lines'. Corporate/academic/PR/hype tone is banned by name, with filler, throat-clearing, generic optimism, and founder cosplay called out, and a clear alternative given (builder-to-builder). AI vocabulary gets a specific 19-word blocklist plus a no-em-dash rule, and the bad example illustrates hedging phrases ('potential issue', 'under certain conditions'); minor flaw is that the directive itself uses an em dash in the Bounded closer paragraph, and the list is words only with no banned phrases like 'it's worth noting'. The weakest dimension is user outcomes: the example does it well ('Users hit a white screen'), and 'why it matters, and what changes for the builder' gestures at it, but there is no explicit instruction to tie technical findings to end-user experience, and 'builder' refers to the GStack operator rather than the product's end users. It's modeled, not mandated."},"latency_ms":17127,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":1293},"cost_usd":0.07537} +{"config":"voice","item":"a02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness is explicit ('Lead with the point', 'Be direct about quality') and reinforced by a good/bad pair where the bad example is hedged consultant-speak. Concreteness is spelled out as a checklist (files, functions, line numbers, commands, outputs, evals, real numbers) and demonstrated with 'auth.ts:47' and 'Two lines.' Corporate tone is banned by name (corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay) with a clear alternative frame: builder talking to builder, not consultant to client. User outcomes are a standalone bullet with a concrete vocabulary (sees, loses, waits for, can now do) and shown in the example ('Users hit a white screen'). The weak spot is AI vocabulary: it names only four words (delve, crucial, robust, comprehensive) and bans em dashes, but there is no list of AI-tell phrases (e.g. 'it's worth noting', 'I've identified a potential issue', 'leverage', 'seamless', 'ensure'), and the directive itself contains an em dash in the bounded-closer paragraph, undercutting the rule. The bounded closer section is a strong addition for brevity but doesn't lift the vocabulary score."},"latency_ms":28836,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":974},"cost_usd":0.05911} +{"config":"voice","item":"f04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is two sentences of generic pleasantries. It never tells the agent to lead with the point or take a position — 'explain things thoroughly' actually pushes toward padding, the opposite of directness. There is no mention of naming files, commands, line numbers, or real numbers. It does not ban corporate, formal, or academic tone, nor offer alternatives; 'positive tone' and 'feels supported' lean toward exactly the soft, customer-service register a good directive would prohibit. There is no list of AI-tell words or phrases to avoid. And nothing connects technical work to what a real user experiences in the product. Every dimension scores at the floor."},"latency_ms":6505,"stop_reason":"end_turn","usage":{"input_tokens":881,"output_tokens":279},"cost_usd":0.02276} +{"config":"voice","item":"a06","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong ('Direct', 'No filler', 'End with what to do') but it never explicitly says to lead with the point or take a position; the cross-model line even softens stance-taking. Concreteness explicitly demands file, function, command, and user-visible impact but omits line numbers and real numbers. Corporate/academic tone is banned outright and 'builder-to-builder' plus 'short paragraphs' gesture at an alternative, but no real substitutes or examples are given. AI vocabulary gets a specific word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) and an em-dash ban, though the list is short and skips AI-tell phrases. User-outcome linkage is excellent: 'Tie every finding to what the user sees, waits for, or loses' is precise and actionable."},"latency_ms":8601,"stop_reason":"end_turn","usage":{"input_tokens":486,"output_tokens":720},"cost_usd":0.04086} +{"config":"voice","item":"p03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'first sentence = the answer', demands taking a position with concrete examples, bans warm-up and recap. Concreteness: names files with line numbers, functions, exact commands, and real numbers with examples, and explicitly bans vague phrasing. Avoids corporate: bans corporate, academic, PR, and hype tone with a specific list of offending phrases and gives a clear alternative (teammate at their desk, short sentences, plain verbs). AI vocabulary: a long specific banned word list plus banned phrases and a no-em-dash rule. User outcomes: requires stating what the real user sees or loses for every bug or change, with a worked contrast example, and handles the case where no user-visible effect exists. Good/bad example at the end reinforces all five dimensions."},"latency_ms":6568,"stop_reason":"end_turn","usage":{"input_tokens":1616,"output_tokens":326},"cost_usd":0.03246} +{"config":"voice","item":"a04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point,' 'Be direct about quality,' plus a good/bad pair that shows hedged prose as the failure mode. Concreteness is explicit and exhaustive (files, functions, line numbers, commands, outputs, evals, real numbers) and the auth.ts:47 example models it. User outcomes are named directly ('what the real user sees, loses, waits for, or can now do') and the example lands it with 'Users hit a white screen.' AI vocabulary gets a concrete banned-word list and an em dash ban, though it omits common AI phrases (e.g. 'it's worth noting,' 'I hope this helps,' 'in today's fast-paced') and the directive itself slips an em dash into the bounded-closer paragraph. The corporate dimension is the weak spot: rather than banning formal/corporate/academic tone, the directive actually prescribes 'a friendly, professional tone appropriate for stakeholders,' which is itself corporate phrasing. 'Avoid filler' and the bounded closer push against bloat, but there is no explicit prohibition on corporate register or alternatives offered beyond the one bad example."},"latency_ms":13713,"stop_reason":"end_turn","usage":{"input_tokens":1578,"output_tokens":954},"cost_usd":0.06348} +{"config":"voice","item":"f01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it mandates opening with acknowledgment and a 'comprehensive overview' before any recommendation, and explicitly forbids 'strong opinions' — burying the point and banning positions. Concreteness: nothing asks for file names, commands, line numbers, or numbers; the example ('several opportunities for enhancement') is pure vagueness. Avoids corporate: it actively prescribes a formal corporate register and bans the casual, fragment-friendly voice that would counter it. Avoids AI vocabulary: it recommends AI-tell/corporate filler ('It is important to note', 'Moving forward, we recommend', 'robust, scalable value', 'best practices') rather than banning them. Connects user outcomes: it frames value in terms of 'stakeholders' and 'modernization initiatives', never in terms of what a real user experiences. Lowest possible score on every axis."},"latency_ms":5154,"stop_reason":"end_turn","usage":{"input_tokens":673,"output_tokens":378},"cost_usd":0.02563} +{"config":"voice","item":"a06","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: opens with 'Direct' and 'No filler', closes with 'End with what to do', but never says to lead with the conclusion or to commit to a position rather than hedge. Concreteness: 'Name the file, function, command' is good, but line numbers and real numbers (latency, sizes, counts) are absent. Avoids corporate: 'Never corporate or academic' is an explicit ban, and 'builder-to-builder' plus 'Short paragraphs' sketch the alternative, but there are no before/after examples or banned formal constructions. Avoids AI vocabulary: there is a specific list (delve, crucial, robust, comprehensive, nuanced, multifaceted) and an em-dash ban, which is better than most, but the list is six words and omits common phrase-level tells (it's worth noting, in today's landscape, I hope this helps, let's dive in). Connects user outcomes: this is the strongest part, with 'user-visible impact' and 'Tie every finding to what the user sees, waits for, or loses' stated twice in concrete terms. The closing paragraph on user context and cross-model agreement is a nice epistemic note but doesn't affect these dimensions."},"latency_ms":11319,"stop_reason":"end_turn","usage":{"input_tokens":486,"output_tokens":952},"cost_usd":0.05246} +{"config":"voice","item":"p02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality... Don't dance around judgments', 'If something is broken... say so plainly', 'End with what to do', plus the YC-partner/senior-eng framing. Concreteness: a dedicated section demanding file, function, line number, exact commands (`bun test test/billing.test.ts`), real numbers (~200ms, N+1, 50 items), and a worked example of pointing at auth.ts:47. Corporate tone: explicitly 'never corporate, never academic, never PR, never hype', bans filler, throat-clearing, founder cosplay, and offers alternatives (builder-to-builder, 'What's interesting here is...' over 'It is important to understand...', short punchy paragraphs, typing-fast cadence). AI vocabulary: an explicit word list (delve, robust, comprehensive, tapestry, etc.), a banned-phrase list, an em-dash ban, and a rule against self-referential AI humor. User outcomes: a dedicated 'Connect to user outcomes' section with concrete examples (3-second spinner, lost customer data), 'make the user's user real', and a core-belief section pushing toward the job to be done and shipping for real people. Every dimension is covered with explicit instruction and examples."},"latency_ms":8595,"stop_reason":"end_turn","usage":{"input_tokens":2051,"output_tokens":491},"cost_usd":0.04506} +{"config":"voice","item":"p02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'say so plainly', 'Be direct about quality... Don't dance around judgments', 'End with what to do', punchy standalone sentences. Concreteness: dedicated section with before/after examples (`bun test test/billing.test.ts`, 'auth.ts:47', '~200ms per page load with 50 items'), plus writing rule 'Real file names, real function names, real numbers.' Avoids corporate: explicit 'never corporate, never academic, never PR, never hype', bans filler/throat-clearing/generic optimism/founder cosplay, and gives the alternative register ('builder talking to a builder, not a consultant presenting to a client', 'sound like typing fast', 'What's interesting here is...' beats 'It is important to understand...'). AI vocabulary: specific 20-word banned list plus banned-phrase list and no em dashes. User outcomes: dedicated 'Connect to user outcomes' section with concrete examples ('3-second spinner', 'loses the customer's data'), reinforced by the make-something-people-want framing and 'start with the user'. Minor nit: humor examples and YC-application nudge are slightly off-topic for a voice directive, but every scored dimension is covered explicitly with examples."},"latency_ms":7859,"stop_reason":"end_turn","usage":{"input_tokens":2051,"output_tokens":487},"cost_usd":0.04486} +{"config":"voice","item":"a02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, plus a good/bad example that shows hedged vs. direct phrasing. Strong. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example demonstrates auth.ts:47 with a two-line fix. Strong. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the 'builder talking to a builder' alternative plus the bounded closer rule; slightly short of 5 because the alternatives are mostly implied by example rather than spelled out as substitutions. AI vocabulary: bans em dashes and names only four words (delve, crucial, robust, comprehensive); no phrase-level list (e.g. 'it's worth noting', 'in today's landscape', 'leverage', 'seamless'), so the list is thin. Also the directive itself uses an em dash in the closer paragraph, undercutting its own rule. User outcomes: explicit instruction to tie choices to what the real user sees, loses, waits for, or can do, reinforced by the example's 'Users hit a white screen.' Strong."},"latency_ms":7994,"stop_reason":"end_turn","usage":{"input_tokens":1554,"output_tokens":483},"cost_usd":0.03969} +{"config":"voice","item":"p01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', plus a bad example that models exactly the hedging to avoid. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers, and the good example (auth.ts:47, /login, 'Two lines') demonstrates it. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the alternative frame ('builder talking to a builder, not a consultant presenting to a client') with paired good/bad samples. AI vocabulary: a specific 19-word banlist plus no em dashes, but it only lists single words (no phrase-level tells like 'it's worth noting', 'great question', 'I hope this helps'), and the bounded-closer paragraph itself uses an em dash, which undercuts the rule. User outcomes: 'what the real user sees, loses, waits for, or can now do' is explicit and the example ('Users hit a white screen') shows it in practice."},"latency_ms":11789,"stop_reason":"end_turn","usage":{"input_tokens":1107,"output_tokens":903},"cost_usd":0.05622} +{"config":"voice","item":"a04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', plus a bounded-closer rule and good/bad examples that reward position-taking. Concreteness is explicit and exemplified: files, functions, line numbers, commands, outputs, evals, real numbers, and the auth.ts:47 example models it. Connecting to user outcomes is explicit ('what the real user sees, loses, waits for, or can now do') and demonstrated ('Users hit a white screen'). The AI-vocabulary dimension has a specific banned-word list and bans em dashes, but covers only single words (no banned phrases like 'it's worth noting' or 'I hope this helps'), and the directive itself uses an em dash in the Bounded closer paragraph, undercutting the rule. The weakest dimension is avoids_corporate: it never bans corporate, formal, or academic tone, and actually prescribes 'a friendly, professional tone appropriate for stakeholders', which is itself corporate framing. The only counterweight is the implicit bad example of hedged corporate phrasing and 'Avoid filler', with no stated alternative register beyond that."},"latency_ms":11604,"stop_reason":"end_turn","usage":{"input_tokens":1065,"output_tokens":979},"cost_usd":0.0596} +{"config":"voice","item":"a02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, plus a Good/Bad example showing position-taking vs hedging. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the auth.ts:47 example demonstrates it. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the 'builder not consultant' alternative plus contrasting examples. AI vocabulary: bans em dashes and names four words (delve, crucial, robust, comprehensive) but the list is short and misses common tells (leverage, seamless, streamline, 'it's worth noting', 'I hope this helps', etc.); notably the directive itself uses an em dash in the bounded-closer paragraph, undercutting its own rule. User outcomes: explicit bullet tying technical choices to what the user sees, loses, waits for, or can do, reinforced by 'Users hit a white screen' in the example."},"latency_ms":7011,"stop_reason":"end_turn","usage":{"input_tokens":1554,"output_tokens":423},"cost_usd":0.03669} +{"config":"voice","item":"a02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness is explicit and reinforced: 'Lead with the point', 'Be direct about quality', a bounded-closer rule, and good/bad examples that show the difference. Concreteness is spelled out with a specific checklist (files, functions, line numbers, commands, outputs, evals, real numbers) and the good example models it (auth.ts:47, /login, 'Two lines'). Corporate/academic/PR/hype tone is banned by name, with filler, throat-clearing, generic optimism, and founder cosplay called out, and the alternative is given ('builder talking to a builder, not a consultant presenting to a client') plus contrasting examples. User outcomes are directly mandated ('what the real user sees, loses, waits for, or can now do') and demonstrated ('Users hit a white screen'). AI vocabulary is the weak spot: it bans em dashes and names only four words (delve, crucial, robust, comprehensive) with no banned phrases or hedging patterns beyond what the bad example implies, and the directive itself uses an em dash in the bounded-closer paragraph, undercutting the rule."},"latency_ms":13159,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":972},"cost_usd":0.05901} +{"config":"voice","item":"p05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Open with status and action, in that order' and 'Commit to a call... say roll back, not a rollback could be considered' are explicit instructions to lead with the point and take positions, reinforced by the good/bad examples. Concreteness: requires dashboard names, timestamped log lines, exact commands (`fly releases rollback v118`), and real numbers (7.2% vs 0.3% baseline), with the good example modeling all of it. Avoids corporate: explicitly bans status-page voice with concrete banned phrases ('we are aware of an issue impacting some users', 'out of an abundance of caution') and gives the alternative ('say what broke, who it hits, and what you are doing'), plus a Bad example. Avoids AI vocabulary: provides a specific list of banned words (delve, crucial, robust, comprehensive, nuanced, pivotal, landscape, seamless, leverage, furthermore, moreover, additionally) and banned phrases ('it's worth noting', 'let's dive in'). Connects user outcomes: 'Translate every metric into what customers are living through' with three concrete translations and the good example ('Users get an empty results page'). Tight, domain-specific, with a persona (2am reader) that motivates every rule."},"latency_ms":7899,"stop_reason":"end_turn","usage":{"input_tokens":851,"output_tokens":479},"cost_usd":0.03246} +{"config":"voice","item":"p01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality. Bugs matter. Edge cases matter. Fix the whole thing, not the demo path', plus a bounded closer rule and a sharp good/bad pair. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the good example (auth.ts:47, /login, 'Two lines') models it. Corporate avoidance: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and supplies the alternative register ('builder talking to a builder, not a consultant presenting to a client') with a contrasting bad example. AI vocabulary: concrete 19-word ban list plus no em dashes, but it covers only single words, not AI-tell phrases ('it's worth noting', 'I hope this helps', 'in today's fast-paced'), and the directive itself uses an em dash in the closer exemption clause, which undercuts the rule. User outcomes: 'what the real user sees, loses, waits for, or can now do' and the example's 'Users hit a white screen' tie bugs to lived experience directly."},"latency_ms":16359,"stop_reason":"end_turn","usage":{"input_tokens":1620,"output_tokens":963},"cost_usd":0.06435} +{"config":"voice","item":"f05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it explicitly instructs the agent to present multiple perspectives before any view, avoid declaring an answer, let the user reach their own conclusion, and frame bugs as gentle questions — the opposite of leading with the point and taking positions. Concreteness: no mention of files, commands, line numbers, or numbers; it actually tells the agent to avoid technical detail in the opening and 'build up gradually'. Avoids_corporate: it mandates corporate/academic hedging ('it may be worth considering', 'one possible approach', 'this could potentially') and heading/bullet scaffolding rather than banning them. Avoids_ai_vocabulary: no banned-word list; the phrases it prescribes are classic AI-tell hedges. Connects_user_outcomes: nothing about tying technical work to real user experience. Lowest possible score across the board."},"latency_ms":6612,"stop_reason":"end_turn","usage":{"input_tokens":551,"output_tokens":368},"cost_usd":0.02391} +{"config":"voice","item":"a05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', and 'Fix the whole thing' explicitly instruct leading with conclusions and taking positions. Concreteness: it names files, functions, line numbers, commands, outputs, evals, and real numbers verbatim. Avoids corporate: it bans corporate, academic, PR, and hype tone and bans filler, throat-clearing, generic optimism, and founder cosplay; the 'builder talking to a builder' line is the only positive alternative offered, so it is slightly thin on replacement guidance. AI vocabulary: an explicit list of twenty banned words plus the em dash ban. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' is a direct, specific instruction with concrete framing."},"latency_ms":6169,"stop_reason":"end_turn","usage":{"input_tokens":1236,"output_tokens":343},"cost_usd":0.02951} +{"config":"voice","item":"f05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what a strong voice directive should do. Directness: it explicitly tells the agent to withhold its own view, avoid verdicts, lay out tradeoffs evenly rather than recommend, and frame bugs as gentle questions — the opposite of leading with the point and taking positions. Concreteness: it says to 'avoid overly technical detail in the opening' and never mentions files, commands, line numbers, or real numbers; it actively pushes toward vagueness. Avoids corporate tone: rather than banning formal/hedged language, it mandates it, prescribing phrases like 'it may be worth considering' and 'this could potentially' and requiring headings and bullet points. Avoids AI vocabulary: no banned list at all; the recommended hedges are themselves classic AI-tell phrases. Connects user outcomes: nothing about tying technical work to what users experience. Every dimension scores at the floor."},"latency_ms":7198,"stop_reason":"end_turn","usage":{"input_tokens":551,"output_tokens":372},"cost_usd":0.02411} +{"config":"voice","item":"a02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, and a good/bad pair that shows a flat declarative diagnosis vs. hedged mush. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example models it (auth.ts:47, /login, 'Two lines'). Avoids corporate: bans corporate/academic/PR/hype, names filler, throat-clearing, generic optimism, founder cosplay, and gives the alternative register (builder, not consultant) plus the bounded closer rule against tours and justification padding. AI vocabulary: there is a ban with a specific list, but it is only four words (delve, crucial, robust, comprehensive) and no AI-tell phrases ('it's worth noting', 'I hope this helps', 'let's dive in', 'in today's landscape', etc.); it also bans em dashes but then uses one itself in the closer paragraph ('mandated report format — the report IS'), which undercuts the rule. User outcomes: explicit instruction to tie choices to what the real user sees, loses, waits for, or can now do, reinforced by 'Users hit a white screen' in the example."},"latency_ms":12428,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":912},"cost_usd":0.05601} +{"config":"voice","item":"a06","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Direct, concrete, builder-to-builder,' 'No filler,' and 'End with what to do' set a clear action-first register, but it never explicitly says to lead with the conclusion or take a position; 'the user decides' is about authority, not hedging, so it doesn't undercut directness, but no affirmative instruction to commit to a view either. Concreteness: strong on naming file, function, command, and user-visible impact, but omits line numbers and real numbers/measurements. Avoids corporate: bans corporate and academic tone outright and gestures at the alternative ('builder-to-builder,' short paragraphs), but gives no examples or contrast of what to say instead. Avoids AI vocabulary: provides a specific, usable word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus the em-dash ban, but the list is short and covers no phrases ('it's worth noting,' 'in summary,' etc.). Connects user outcomes: excellent and specific: 'Tie every finding to what the user sees, waits for, or loses' is a concrete, enforceable rule."},"latency_ms":11001,"stop_reason":"end_turn","usage":{"input_tokens":486,"output_tokens":902},"cost_usd":0.04996} +{"config":"voice","item":"p03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' section mandates first sentence = answer, requires taking positions with concrete examples ('Ship it', 'Don't merge this'), bans warm-up and recap, and tells the agent what to do when unsure. Concreteness: explicitly names file:line, function, command, and real numbers with realistic examples, plus a direct ban on vague phrasing with a replacement rule. Avoids corporate: bans corporate/academic/PR/hype with a specific list of offending phrases and gives a positive alternative (teammate at their desk, short sentences, plain verbs, actual opinion). Avoids AI vocabulary: a long explicit word list, a phrase list, and an em dash ban. Connects user outcomes: requires a user-visible effect for every bug/tradeoff/change with a good-vs-bad example, and handles the case where no user effect exists. The good/bad example at the end demonstrates all five dimensions at once. Only minor omission is no mention of hedging language beyond the one example, but the position-taking rule covers it."},"latency_ms":8143,"stop_reason":"end_turn","usage":{"input_tokens":1103,"output_tokens":414},"cost_usd":0.03173} +{"config":"voice","item":"a06","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: opens with 'Direct, concrete, builder-to-builder', bans filler, says 'End with what to do', but does not explicitly say to lead with the point or take positions; the final paragraph actually softens positions toward recommendation-only. Concreteness: explicitly names file, function, command, and impact, but omits line numbers and real numbers/measurements. Avoids corporate: 'Never corporate or academic' is a flat ban with no alternatives or examples of what replaces it, though 'builder-to-builder' and 'short paragraphs' gesture at the alternative. Avoids AI vocabulary: provides a specific word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) and bans em dashes, but the list is short and covers no phrases (e.g. 'it's worth noting', 'in today's landscape'). Connects user outcomes: strongest dimension; 'user-visible impact' and 'tie every finding to what the user sees, waits for, or loses' is explicit and well-phrased."},"latency_ms":7078,"stop_reason":"end_turn","usage":{"input_tokens":999,"output_tokens":408},"cost_usd":0.03039} +{"config":"voice","item":"p02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality... Don't dance around judgments', 'say so plainly', 'End with what to do', plus a ban on filler and throat-clearing. Concreteness: a dedicated section demanding file, function, line number, exact commands (bun test test/billing.test.ts), real numbers (~200ms, N+1), and a worked example (auth.ts:47). Avoids corporate: explicitly 'never corporate, never academic, never PR, never hype', with alternatives given ('builder talking to a builder, not a consultant presenting to a client', context-matched energy, 'sound like typing fast', 'What's interesting here is...' over 'It is important to understand...'). AI vocabulary: explicit banned word list (delve, crucial, robust, comprehensive, etc.), banned phrase list, and no em dashes with a replacement instruction. User outcomes: dedicated 'Connect to user outcomes' section with concrete examples (3-second spinner, lost customer data), 'Make the user's user real', and the whole core-belief section pushes toward the user and job to be done. Every dimension is explicitly instructed with examples."},"latency_ms":6426,"stop_reason":"end_turn","usage":{"input_tokens":2564,"output_tokens":453},"cost_usd":0.04829} +{"config":"voice","item":"f05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it explicitly tells the agent to withhold its own view, lay out tradeoffs 'evenly rather than recommending one option', and frame bugs as gentle questions instead of stating them — the opposite of leading with the point and taking positions. Concreteness: it says to 'avoid overly technical detail in the opening' and never mentions files, commands, line numbers, or real numbers; it pushes toward abstraction. Avoids corporate: rather than banning hedged/formal tone, it mandates it, prescribing phrases like 'it may be worth considering' and 'this could potentially', plus heavy headings and bullets. Avoids AI vocabulary: no banned-word list at all, and the phrases it actively recommends are classic AI-tell hedges. Connects user outcomes: no mention of users, product impact, or real-world consequences of technical choices. Every dimension scores at the floor."},"latency_ms":7190,"stop_reason":"end_turn","usage":{"input_tokens":551,"output_tokens":380},"cost_usd":0.02451} +{"config":"voice","item":"a03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":4,"reasoning":"Directness is explicit and reinforced: 'Lead with the point', 'Be direct about quality', 'Fix the whole thing, not the demo path', plus a good/bad pair that shows hedged consultant-speak versus a flat statement of the bug and fix. Concreteness is spelled out as a named list (files, functions, line numbers, commands, outputs, evals, real numbers) and the example demonstrates it with auth.ts:47 and 'Two lines'. Corporate tone is banned by name (corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay) and the alternative is given as a posture ('builder talking to a builder, not a consultant presenting to a client') and modeled in both example pairs and the bounded closer. AI vocabulary gets a concrete 19-word blocklist plus an em dash ban, which is strong, but it only lists single words; it does not ban common AI phrases ('it's worth noting', 'I hope this helps', 'in today's fast-paced world', 'as an AI'), so it loses a point against the 'words and phrases' bar. User outcomes are addressed via 'why it matters, and what changes for the builder' and the example's 'Users hit a white screen', but the instruction is oriented toward the builder reading the output rather than an explicit rule to trace every technical change to the end-user experience; the example carries most of that weight rather than the rule text."},"latency_ms":14678,"stop_reason":"end_turn","usage":{"input_tokens":1072,"output_tokens":1028},"cost_usd":0.06212} +{"config":"voice","item":"a03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness is explicit and strong: 'Lead with the point', 'Be direct about quality', reinforced by a good/bad pair and a bounded closer rule that cuts throat-clearing. Concreteness is explicit with a full list (files, functions, line numbers, commands, outputs, evals, real numbers) and the auth.ts:47 example models it exactly. Corporate tone is banned by name (corporate, academic, PR, hype, filler, founder cosplay) with a clear alternative (builder talking to a builder, not consultant to client) and contrasting examples. AI vocabulary gets a specific 20-word ban list plus no em dashes, but the list is words only, not phrases ('it's worth noting', 'I've identified a potential issue' appears only in the bad example), and the directive itself contains an em dash in the bounded closer paragraph, undercutting the rule. User outcomes are only partially covered: 'what changes for the builder' and the example line 'Users hit a white screen' model the behavior, but there is no explicit instruction to tie technical changes to real user experience, so the agent must infer it from one example."},"latency_ms":13225,"stop_reason":"end_turn","usage":{"input_tokens":1585,"output_tokens":922},"cost_usd":0.06195} +{"config":"voice","item":"a06","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Direct, concrete, builder-to-builder', 'No filler', 'End with what to do' are strong, but it never explicitly says to lead with the conclusion or take a position; the final paragraph about deferring to the user slightly softens the mandate to commit. Concreteness: explicitly names file, function, command, and user-visible impact, but omits line numbers and real numbers/measurements. Avoids corporate: 'Never corporate or academic' is an explicit ban, with 'builder-to-builder' and 'short paragraphs' as implicit alternatives, but no concrete before/after guidance. Avoids AI vocabulary: provides a specific banned-word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus em dashes, but the list is short and omits common AI phrases (e.g., 'it's worth noting', 'I hope this helps', 'leverage', 'seamless'). Connects user outcomes: excellent, with 'Tie every finding to what the user sees, waits for, or loses' and 'user-visible impact' making this an explicit requirement for every finding."},"latency_ms":8898,"stop_reason":"end_turn","usage":{"input_tokens":486,"output_tokens":809},"cost_usd":0.04531} +{"config":"voice","item":"p04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":4,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' is explicit and the tone list includes 'direct' and 'sharp,' but there's no instruction to take positions, commit to recommendations, or avoid hedging language, so it stops short of a 5. Concreteness: excellent, with paired bad/good examples for commands, numbers, and file:line references. Avoids corporate: it bans corporate, academic, PR, and hype tone and offers the 'builder talking to a builder, not a consultant' alternative plus context-specific energies, but doesn't list concrete corporate phrases to avoid or show before/after rewrites. AI vocabulary: strong, with a specific list of ~20 banned words, a separate list of banned phrases, and the em dash rule. User outcomes: explicit section with two concrete example sentences and the instruction to connect work back to what the real user experiences, plus the zero-defects stance reinforcing it."},"latency_ms":7085,"stop_reason":"end_turn","usage":{"input_tokens":1046,"output_tokens":366},"cost_usd":0.02876} +{"config":"voice","item":"p05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Open with status and action, in that order' plus 'Commit to a call' with a hedging-vs-decisive example is about as direct as it gets. Concreteness: explicitly demands dashboard names, timestamped log lines, exact commands (`fly releases rollback v118`), and numeric error rates with baselines; the Good example models all of it. Avoids corporate: strong ban on status-page voice with two concrete banned phrases and a bad example, plus 'say what broke, who it hits, and what you are doing' as the alternative — but it targets PR/status-page tone specifically rather than naming formal or academic register more broadly, so slightly short of 5. Avoids AI vocabulary: a specific list of a dozen tell words plus two banned phrases. Connects user outcomes: 'Translate every metric into what customers are living through' with three concrete translations and the Good example ('Users get an empty results page') nails it."},"latency_ms":5228,"stop_reason":"end_turn","usage":{"input_tokens":1364,"output_tokens":371},"cost_usd":0.03219} +{"config":"voice","item":"f04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is generic and does not address any of the evaluated dimensions. It never asks the agent to lead with the point or take positions; 'explain thoroughly' and 'positive tone' actually push toward hedging and padding. It gives no instruction to cite files, commands, line numbers, or real numbers. It does not ban corporate/formal/academic tone or list AI-tell words. It makes no connection between technical work and real user outcomes. 'Helpful, friendly, clear' is a boilerplate assistant persona, not a voice directive."},"latency_ms":3640,"stop_reason":"end_turn","usage":{"input_tokens":368,"output_tokens":243},"cost_usd":0.01583} +{"config":"voice","item":"a01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: opens with 'Direct, concrete, builder-to-builder' and 'No filler', 'End with what to do', but does not explicitly say lead with the point or take positions; in fact it defers heavily to the user. Concreteness: tells the agent to name file, function, command, and impact, but omits line numbers and real numbers/measurements. Avoids corporate: bans corporate and academic tone and gives the 'builder-to-builder' framing, but offers no concrete alternatives or examples of what to say instead. Avoids AI vocabulary: bans em dashes and lists six specific AI-tell words, a short but real list; no banned phrases (e.g. 'it's worth noting', 'in today's landscape'). Connects user outcomes: 'user-visible impact' is named as something to state, which is a direct nod, but there is no instruction to tie each technical change to what the user actually experiences or why it matters."},"latency_ms":6573,"stop_reason":"end_turn","usage":{"input_tokens":978,"output_tokens":366},"cost_usd":0.02808} +{"config":"voice","item":"a01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: explicitly 'direct', 'no filler', 'end with what to do', but never says lead with the conclusion or take a position; the user-decides clause leans toward deference rather than stating positions. Concreteness: names file, function, command, user-visible impact, but omits line numbers and real numbers/measurements. Avoids corporate: 'never corporate or academic' is explicit, with 'builder-to-builder' and 'short paragraphs' as the positive alternative, though thin. AI vocabulary: a real word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus em dash ban, but short and covers no phrases ('it's worth noting', 'in today's landscape', etc.). User outcomes: 'user-visible impact' is a single phrase; no instruction to translate technical changes into what users actually experience or why it matters to them."},"latency_ms":11123,"stop_reason":"end_turn","usage":{"input_tokens":465,"output_tokens":925},"cost_usd":0.0509} +{"config":"voice","item":"p03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point' section mandates first sentence = answer/verdict, requires taking a position with concrete examples ('Ship it', 'Don't merge this'), and bans warm-up and recaps. Concreteness: explicitly requires file:line, function names, exact commands, and real numbers with worked examples, and bans vague hedging like 'there may be an issue in the auth flow'. Avoids corporate: explicitly bans corporate, academic, PR, and hype tone, lists specific banned phrases, and provides a clear alternative (explain it like a teammate at their desk, short sentences, plain verbs, actual opinion). AI vocabulary: dedicated section with a specific list of banned words, banned phrases, and even bans em dashes. User outcomes: 'Tie it to the user' section requires naming what the real user sees/loses/waits for for every bug or change, with a good/bad contrast, and handles the case where no user-visible effect exists. The good/bad example at the end reinforces all five dimensions in one place. Minor nit: the banned-word list could include a few more common tells (e.g., 'ensure', 'streamline'), but it's substantial and specific."},"latency_ms":8234,"stop_reason":"end_turn","usage":{"input_tokens":1103,"output_tokens":456},"cost_usd":0.03383} +{"config":"voice","item":"a06","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Direct, concrete, builder-to-builder,' 'No filler,' and 'End with what to do' push for a direct, action-oriented voice, but there is no explicit instruction to lead with the conclusion or to commit to a position; the closing 'the user decides' is about deference, not hedging, so it doesn't undercut directness much. Concreteness: 'Name the file, function, command' is specific and good, but line numbers and real measured numbers are not mentioned. Avoids corporate: 'Never corporate or academic' is an explicit ban, and 'builder-to-builder' plus 'Short paragraphs' gesture at an alternative, but the replacement register is thinly described with no examples or contrast. Avoids AI vocabulary: a real list is given (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus the em dash ban, which is concrete and enforceable, but it is short and covers only single words, not stock AI phrases or structural tells like tidy triplets or 'it's worth noting.' Connects user outcomes: 'Tie every finding to what the user sees, waits for, or loses' and 'user-visible impact' are strong, specific, and universal ('every finding'), which is exactly what this dimension asks for."},"latency_ms":12388,"stop_reason":"end_turn","usage":{"input_tokens":486,"output_tokens":1033},"cost_usd":0.05651} +{"config":"voice","item":"p01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality. Bugs matter. Edge cases matter.', builder-to-builder framing, and a paired good/bad example that shows a flat claim versus hedged mush. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example (auth.ts:47, /login, 'Two lines') models it. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and supplies the alternative ('builder talking to a builder, not a consultant presenting to a client') plus the bad example. AI vocabulary: a specific ~20-word banlist plus no em dashes, which is strong, but it lists only single words (no banned phrases like 'it's worth noting' or 'I've identified a potential issue' beyond the one example), and the directive itself uses an em dash in the bounded-closer paragraph, undercutting its own rule. User outcomes: 'what the real user sees, loses, waits for, or can now do' is a crisp instruction, reinforced by 'Users hit a white screen' in the example."},"latency_ms":12974,"stop_reason":"end_turn","usage":{"input_tokens":1107,"output_tokens":1000},"cost_usd":0.06107} +{"config":"voice","item":"p01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', takes positions ('Bugs matter. Edge cases matter.'), plus the bounded closer rule cuts prose. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the good example shows auth.ts:47 with a two-line fix. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and provides the alternative framing ('builder talking to a builder, not a consultant presenting to a client') with a contrasting good/bad pair. AI vocabulary: bans em dashes and gives a specific 19-word list of AI-tell words. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', reinforced in the example ('Users hit a white screen'). Minor nit: the directive itself contains an em dash in the exempt clause, slightly undercutting its own rule, but every dimension is explicitly and concretely covered."},"latency_ms":7423,"stop_reason":"end_turn","usage":{"input_tokens":1620,"output_tokens":429},"cost_usd":0.03765} +{"config":"voice","item":"f02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, plus a good/bad example that contrasts a hedged sentence with a blunt one — excellent. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers, and the example demonstrates it (auth.ts:47, /login, 'Two lines') — excellent. Avoids corporate: explicitly bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and offers the 'builder talking to a builder, not a consultant presenting to a client' alternative plus a bad example; strong but the alternatives are mostly a stance rather than concrete rewrites, so 4. Avoids AI vocabulary: no specific list of AI-tell words or phrases (e.g. 'delve', 'leverage', 'robust', 'it's worth noting', 'I've identified a potential issue'); the bad example implicitly shows hedging language but there is no explicit banned-vocabulary list, so 2. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' plus the example's 'Users hit a white screen' — excellent."},"latency_ms":5622,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":490},"cost_usd":0.03458} +{"config":"voice","item":"a04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":2,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point,' 'Be direct about quality,' a bounded closer that cuts unrequested prose, and a clear good/bad pair. Concreteness is explicit and exemplified (auth.ts:47, commands, evals, real numbers, 'Two lines'). User outcomes are tied directly to technical work ('what the real user sees, loses, waits for, or can now do'; 'Users hit a white screen'). AI vocabulary gets a specific banned word list plus an em dash ban, though it covers single words rather than phrases (e.g. 'it's worth noting', 'I hope this helps'), and the directive itself slips in an em dash. The weak spot is corporate tone: rather than banning formal/corporate/academic register, it actively asks for a 'professional tone appropriate for stakeholders,' which pulls the other way; only the bad example and 'avoid filler' gesture at the problem, with no explicit prohibition or named alternative register."},"latency_ms":10616,"stop_reason":"end_turn","usage":{"input_tokens":1065,"output_tokens":866},"cost_usd":0.05395} +{"config":"voice","item":"p05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: explicitly orders status-then-action ordering, demands committing to a call with a concrete contrast ('roll back' vs 'a rollback could be considered'), and the good/bad examples reinforce it. Concreteness: requires dashboard names, timestamped log lines, exact commands with a real example, and numeric error rates with baseline; the Good example models all of it. Avoids corporate: bans 'status-page voice' with two specific banned phrases and gives a direct alternative (what broke, who it hits, what you're doing), plus a Bad example. Avoids AI vocabulary: provides a specific banned word list and two banned phrases, which is solid, though the list is somewhat short compared to a fuller AI-tell inventory (no mention of em-dash overuse, 'it's important to', 'in today's', hedging stacks, etc.). Connects user outcomes: explicitly requires translating every metric into customer experience with three concrete examples and the Good example demonstrates it ('Users get an empty results page')."},"latency_ms":5475,"stop_reason":"end_turn","usage":{"input_tokens":1364,"output_tokens":400},"cost_usd":0.03364} +{"config":"voice","item":"f04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"The directive is generic and actively works against the evaluated dimensions. It says nothing about leading with the point or taking positions; 'explain thoroughly' and 'keep a positive tone' push toward hedged, padded output rather than directness. There is no instruction to cite files, commands, line numbers, or real numbers. It does not ban corporate/formal/academic tone — 'helpful, friendly, supportive' is the default corporate-assistant register, with no alternatives offered. It provides no list of AI-tell words or phrases to avoid. It never connects technical work to user-facing outcomes; 'the user feels supported' is about emotional tone, not product impact. Minimum scores across the board."},"latency_ms":4250,"stop_reason":"end_turn","usage":{"input_tokens":368,"output_tokens":292},"cost_usd":0.01828} +{"config":"voice","item":"f02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, plus a good/bad example contrasting a hedged sentence with a blunt one — strong. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example uses 'auth.ts:47' and '/login' — excellent. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the consultant-vs-builder alternative plus examples; loses a point for not offering many positive phrasing alternatives beyond the two examples. AI vocabulary: no specific banned-word list (no 'delve', 'leverage', 'robust', 'it's worth noting', etc.); the bad example implicitly flags hedging like 'potential issue' and 'may cause problems under certain conditions', and 'generic optimism'/'filler' gesture at it, but there is no explicit list — weak on this dimension. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' plus the example 'Users hit a white screen' — excellent."},"latency_ms":8261,"stop_reason":"end_turn","usage":{"input_tokens":1521,"output_tokens":469},"cost_usd":0.03866} +{"config":"voice","item":"a02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', bounded closer, and a sharp good/bad example pair showing the difference. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example uses auth.ts:47 and '/login'. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and offers the alternative framing 'builder talking to a builder, not a consultant presenting to a client'; the bad example illustrates the banned register, though alternatives could be slightly more elaborated. AI vocabulary: bans em dashes and names only four words (delve, crucial, robust, comprehensive); the list is short and omits common tells like 'leverage', 'seamless', 'it's worth noting', 'in today's landscape', etc. Also note the directive itself contains an em dash in the closer section, undermining its own rule. User outcomes: explicit instruction to tie choices to what the user 'sees, loses, waits for, or can now do', reinforced by the 'Users hit a white screen' example."},"latency_ms":8396,"stop_reason":"end_turn","usage":{"input_tokens":1554,"output_tokens":446},"cost_usd":0.03784} +{"config":"voice","item":"a02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', and a Good/Bad example that models leading with the finding and fix. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example uses auth.ts:47 and '/login'. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, and gives the 'builder talking to a builder' alternative plus a bad example of consultant-speak; slightly short of 5 because the alternative is stated as a persona rather than a substitution list. AI vocabulary: bans em dashes and names only four words (delve, crucial, robust, comprehensive); the list is real but thin, and the directive itself uses an em dash in the Bounded closer section, undercutting the rule. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do' and the example ('Users hit a white screen') nail this."},"latency_ms":6771,"stop_reason":"end_turn","usage":{"input_tokens":1041,"output_tokens":397},"cost_usd":0.03026} +{"config":"voice","item":"f01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it mandates opening with acknowledgment and a 'comprehensive overview' before any recommendation, and explicitly bans 'strong opinions' — the opposite of leading with the point and taking positions. Concreteness: nothing asks for file names, commands, line numbers, or real numbers; the example ('several opportunities for enhancement') is maximally vague. Avoids_corporate: it actively prescribes a 'formal tone at all times,' corporate stock phrases ('We are pleased to report'), and executive summaries. Avoids_ai_vocabulary: rather than banning AI-tell words, it recommends them ('It is important to note', 'robust, scalable value', 'best practices', 'Moving forward'). Connects_user_outcomes: it frames value in terms of stakeholders and 'modernization initiatives,' never real user experience. Minimum score on every axis."},"latency_ms":4689,"stop_reason":"end_turn","usage":{"input_tokens":673,"output_tokens":377},"cost_usd":0.02558} +{"config":"voice","item":"a01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":4,"connects_user_outcomes":3,"reasoning":"Directness: 'Direct', 'No filler', 'End with what to do' are strong, but it doesn't explicitly say to lead with the point or take positions; the closing lines actually lean toward deference ('The user decides'), which slightly undercuts position-taking. Concreteness: 'Name the file, function, command' is specific and good, but omits line numbers and real numbers/measurements. Avoids corporate: explicitly bans corporate and academic tone, but gives only 'builder-to-builder' as a loose alternative with no before/after examples or banned patterns beyond the label. Avoids AI vocabulary: provides a real banned word list (delve, crucial, robust, comprehensive, nuanced, multifaceted) plus em dashes, but the list is short and misses common phrase-level tells (e.g. 'it's worth noting', 'in conclusion', 'leverage', 'seamless'). Connects user outcomes: 'user-visible impact' is named as something to include, which is a genuine instruction, but it's a single phrase with no elaboration on how to tie technical changes to what the user actually experiences."},"latency_ms":5628,"stop_reason":"end_turn","usage":{"input_tokens":465,"output_tokens":431},"cost_usd":0.0262} +{"config":"voice","item":"f01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it mandates opening with acknowledgment and a 'comprehensive overview' before any recommendation, and explicitly bans 'strong opinions' — the opposite of leading with the point and taking positions. Concreteness: no mention of files, commands, line numbers, or real numbers; the example is pure abstraction ('several opportunities for enhancement', 'broader modernization initiative'). Avoids_corporate: it actively prescribes a formal corporate register and supplies corporate filler phrases ('We are pleased to report', 'It is important to note', 'Moving forward, we recommend') as recommended language rather than banning them. Avoids_ai_vocabulary: no ban list at all, and it encourages classic AI/consultant-speak ('robust, scalable value', 'aligns with best practices', 'thorough review', 'opportunities for enhancement'). Connects_user_outcomes: it frames everything around 'stakeholders' and 'value' in the abstract, never instructs the agent to tie technical work to what real users will see or experience."},"latency_ms":5360,"stop_reason":"end_turn","usage":{"input_tokens":673,"output_tokens":430},"cost_usd":0.02823} +{"config":"voice","item":"f05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what the rubric rewards on nearly every axis. Directness: it explicitly forbids taking positions ('avoid declaring one', 'rather than recommending one option'), mandates hedged phrasing ('it may be worth considering', 'this could potentially'), and tells the agent to soften bugs into gentle questions instead of stating them. Concreteness: no mention of files, commands, line numbers, or real numbers; it actively says to 'avoid overly technical detail in the opening' and 'build up to specifics gradually'. Avoids_corporate: it prescribes the formal/corporate register ('measured language', 'balanced, thoughtful collaborator', headings and bullets) rather than banning it, and offers no plain-speech alternatives. Avoids_ai_vocabulary: no banned-word list at all—the phrases it recommends are textbook AI-tell hedges. Connects_user_outcomes: nothing ties technical work to what users actually experience; the focus is entirely on the user's feelings about the agent's tone, not on product outcomes."},"latency_ms":7972,"stop_reason":"end_turn","usage":{"input_tokens":551,"output_tokens":421},"cost_usd":0.02656} +{"config":"voice","item":"f03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness is excellent: it mandates leading with an explicit verdict (Approve/Request changes/Block) and ordering findings by severity, with a good/bad example reinforcing it. Concreteness is excellent: path:line citations, quoted offending expressions, exact fixes or commands, and real numbers (allocation/query counts, benchmarks). Corporate tone is explicitly banned with a clear alternative framing ('terse senior engineer leaving PR comments, not a consultant writing a report'), though the alternative is a single persona line rather than detailed guidance, so 4. AI vocabulary has a specific banned-word list plus an em dash ban, but it is a modest list focused on single words and omits common AI phrase-level tells (e.g., 'it's worth noting', 'in today's fast-paced world'), so 4. Connecting to user outcomes scores 1 because the directive does the opposite: it explicitly forbids speculating about end users, customers, product impact, or business outcomes and instructs the agent to describe what code does rather than how anyone experiences it. Whether that is appropriate for a narrow code-review persona is a design choice, but on this dimension as defined it actively fails."},"latency_ms":6480,"stop_reason":"end_turn","usage":{"input_tokens":734,"output_tokens":457},"cost_usd":0.03019} +{"config":"voice","item":"f02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', builder-to-builder framing, plus a good/bad example contrasting a hedged 'potential issue... may cause problems' with a blunt fix — strong. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example shows 'auth.ts:47' and 'Two lines' — strong. Avoids corporate: explicitly bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the alternative (builder talking to a builder) plus a bad example — but no explicit list of banned corporate phrases, so 4. Avoids AI vocabulary: no specific list of AI-tell words (e.g., 'delve', 'leverage', 'robust', 'it's worth noting', 'certainly'); the bad example implicitly discourages hedging language and 'filler/throat-clearing' is named, but the dimension asks for specific lists, which are absent — 2. Connects user outcomes: explicit bullet 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', reinforced in the example ('Users hit a white screen') — strong."},"latency_ms":7890,"stop_reason":"end_turn","usage":{"input_tokens":1521,"output_tokens":485},"cost_usd":0.03946} +{"config":"voice","item":"f06","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":3,"connects_user_outcomes":1,"reasoning":"Directness: 'direct, sharp, no filler, no throat-clearing, end with what to do' covers leading with the point and taking action, but doesn't explicitly say to take positions or lead with the conclusion. Concreteness: 'Name the file, the function, the command' is good and specific, but omits line numbers and real numbers. Avoids corporate: bans corporate and academic tone and offers 'builder, not consultant' as a frame, but gives no concrete alternatives or examples of what to write instead. Avoids AI vocabulary: names five words plus 'etc.' and bans em dashes, which is a start but a short list with an open-ended 'etc.' is weak; ironically the directive itself uses an em dash in the last line. Connects user outcomes: nothing about tying technical work to what users actually experience; the closing note about user context is about deference, not outcome connection."},"latency_ms":4711,"stop_reason":"end_turn","usage":{"input_tokens":1013,"output_tokens":351},"cost_usd":0.02768} +{"config":"voice","item":"p07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'first sentence carries the point', 'take a position', hedge only with stated reason, plus table examples showing it. Concreteness: names files, functions, line numbers, commands, measured numbers, with worked examples (auth.ts:47, bun test 412/412, 51 queries/200ms). Avoids corporate: bans corporate, academic, PR, hype tone, throat-clearing, generic optimism, and the table gives alternatives; slightly short on explaining what the replacement register sounds like beyond 'builder to builder', so a 4. AI vocabulary: a specific banned-word list of 19 terms plus em dashes, and the delve/robust/comprehensive table row. User outcomes: 'every finding says what the real user sees, loses, or waits for' plus the white-screen example makes this explicit and testable."},"latency_ms":4899,"stop_reason":"end_turn","usage":{"input_tokens":1311,"output_tokens":343},"cost_usd":0.03026} +{"config":"voice","item":"p08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'End with what to do', plus a good/bad example showing a flat position vs. hedging. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example uses auth.ts:47 and a specific fix. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, and offers the alternative frame 'builder talking to a builder, not a consultant presenting to a client', reinforced by contrasting examples and the bounded closer. AI vocabulary: specific word list (delve, crucial, robust, etc.), em dash ban, and a banned-phrase list. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', demonstrated in the example ('Users hit a white screen'). Minor nit: the directive itself contains an em dash in the Bounded closer paragraph while banning them, but that does not weaken the instruction content."},"latency_ms":6962,"stop_reason":"end_turn","usage":{"input_tokens":1703,"output_tokens":414},"cost_usd":0.03773} +{"config":"voice","item":"a08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":4,"reasoning":"Directness: 'Answer first, then evidence. Pick a side when the evidence supports one' is a clear, tight instruction to lead with the point and take positions. Concreteness: explicitly demands file:line, the command run, and the printed number, with a sharp contrast example ('Probably slow' vs '412ms p95 on /search'). Avoids corporate: bans press-release tone with three concrete banned phrases and offers the alternative frame ('talk like an engineer'), but the list is short and doesn't cover academic hedging or formal register beyond marketing-speak. Avoids AI vocabulary: provides a specific banned word list (delve, leverage, robust, seamless, utilize, crucial), though it is brief and omits common AI-tell phrases like 'it's worth noting', 'in today's landscape', 'tapestry', or structural tells like excessive hedging. Connects user outcomes: directly instructs to state what changes for the person using the product with concrete dimensions (see, wait for, can no longer do); solid but could push harder on making this mandatory for every technical claim."},"latency_ms":7118,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":421},"cost_usd":0.03113} +{"config":"voice","item":"p08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'End with what to do', with a good/bad example contrasting a blunt diagnosis against hedged prose. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, and real numbers, and the example (auth.ts:47, /login, 'Two lines') models it. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, and founder cosplay, and gives the alternative ('builder talking to a builder, not a consultant presenting to a client'). Avoids AI vocabulary: a specific word list of 19 AI-tell words plus a separate list of banned phrases and a no-em-dash rule. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', reinforced in the example ('Users hit a white screen'). The bounded closer section adds a length discipline with its own good/bad examples. Minor irony: the directive itself contains an em dash in the closer exemption clause, but that doesn't weaken the instructions it gives."},"latency_ms":7292,"stop_reason":"end_turn","usage":{"input_tokens":1190,"output_tokens":449},"cost_usd":0.03435} +{"config":"voice","item":"f08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what a good voice directive should be. Directness: it instructs the agent to open with filler hype ('Love this!') and to avoid 'dwelling on the problem' when something is broken, which actively discourages leading with the point or taking honest positions. Concreteness: there is zero mention of files, commands, line numbers, or real numbers; the only guidance is on emotional framing. Avoids corporate: it doesn't ban formal/academic tone and instead mandates a different kind of hollow register (hype marketing speak). Avoids AI vocabulary: it explicitly *requires* the classic AI-tell words—revolutionary, seamless, cutting-edge, next-level, world-class, game-changer—plus emoji spam and motivational sign-offs, the exact opposite of a ban list. Connects user outcomes: nothing ties technical work to what real users experience; 'celebrate the builder' is about flattering the developer, not about end-user impact. Every dimension scores at the floor."},"latency_ms":7344,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":396},"cost_usd":0.03051} +{"config":"voice","item":"p06","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: Rule 1 explicitly demands point-first, position-taking, with a good/bad contrast; rule 7 reinforces with a closing action line. Concreteness: Rule 2 names file, function, line number, command, and real number, with a worked example (`cart.ts:88`, `bun test ...`, 0.01 drift on 3 of 12). Avoids corporate: Rule 4 bans corporate, academic, PR, and hype registers, gives the 'engineer at the next desk' alternative and a good/bad pair. Avoids AI vocabulary: Rule 5 provides an explicit banned-word list (delve, crucial, robust, comprehensive, etc.) plus em dashes. Connects user outcomes: Rule 3 requires every technical point to end in user impact (see, lose, wait for, can now do) with a concrete example. Each dimension is covered explicitly with examples; minor room for a longer AI-tell phrase list (e.g. 'it's worth noting', 'I hope this helps'), but the directive is strong across the board."},"latency_ms":7545,"stop_reason":"end_turn","usage":{"input_tokens":1401,"output_tokens":411},"cost_usd":0.03456} +{"config":"voice","item":"a07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point,' 'Be direct about quality,' 'Don't dance around judgments,' 'End with what to do,' and 'say so plainly' make this a 5. Concreteness: a dedicated section demanding file names, function names, line numbers, exact commands (`bun test test/billing.test.ts`), and real numbers (~200ms N+1 example, auth.ts:47 example) with explicit before/after contrasts. 5. Avoids corporate: explicitly bans corporate, academic, PR, hype, and 'consultant presenting to a client,' and offers alternatives (builder-to-builder, YC partner / senior eng / tech blog post energy, 'What's interesting here is...' over 'It is important to understand...'). Strong but the alternatives are mostly framed as energy/register rather than a fuller phrase-level swap list, so 4. Avoids AI vocabulary: only one em-dash ban plus the vague 'avoid cliches and words that sound machine-written' and 'filler, throat-clearing, generic optimism.' No specific banned word list (delve, leverage, robust, seamless, 'I'd be happy to', etc.), so the agent has little to operate on. 2. Connects user outcomes: a dedicated section with concrete example sentences ('3-second spinner on every page load,' 'loses the customer's data'), plus 'Make the user's user real' and the whole make-something-people-want framing. 5."},"latency_ms":9485,"stop_reason":"end_turn","usage":{"input_tokens":2426,"output_tokens":547},"cost_usd":0.05161} +{"config":"voice","item":"p07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'first sentence carries the point,' 'take a position,' hedge only with a stated reason, plus a 'Here's the fix' example. Concreteness: names files, functions, line numbers, commands, measured numbers, and every table row demonstrates it (auth.ts:47, 51 queries/200ms, 412/412). Avoids corporate: bans corporate/academic/PR/hype tone and throat-clearing, and the table provides before/after alternatives; slightly short of 5 because the ban is a single line and the alternatives are tied to specific examples rather than general patterns (e.g. no explicit guidance on passive voice or 'we recommend' framing beyond one row). Avoids AI vocabulary: a specific, well-chosen banned-word list including delve, robust, comprehensive, tapestry, landscape, furthermore/moreover/additionally, plus an em dash ban. Connects user outcomes: 'every finding says what the real user sees, loses, or waits for' with a concrete example (white screen, log in again)."},"latency_ms":4782,"stop_reason":"end_turn","usage":{"input_tokens":798,"output_tokens":412},"cost_usd":0.02858} +{"config":"voice","item":"p07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit rule 'First sentence carries the point. Take a position; hedge only with a stated reason' plus 'End with what to do next', and the table models leading with the finding. Concreteness: rule names files, functions, line numbers, commands, measured numbers, and every table row demonstrates it (`auth.ts:47`, 51 queries/200ms, 412/412). Avoids corporate: explicitly bans corporate, academic, PR, hype tone, throat-clearing, and generic optimism, with before/after alternatives in the table. Avoids AI vocabulary: a specific banned-word list of 19 common AI tells plus an em-dash ban, and the table shows 'delve/robust/comprehensive' replaced with 'Here's the fix.' Connects user outcomes: 'Every finding says what the real user sees, loses, or waits for' is a direct rule, and the table translates vague impact into 'Users see a white screen and have to log in again.' Compact, operational, and every dimension is covered by both a rule and an example."},"latency_ms":5312,"stop_reason":"end_turn","usage":{"input_tokens":798,"output_tokens":408},"cost_usd":0.02838} +{"config":"voice","item":"a08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Answer first, then evidence. Pick a side' is a crisp, complete instruction to lead with the point and take positions. Concreteness: explicitly demands file:line, the command run, and the printed number, with a sharp before/after example ('Probably slow' vs '412ms p95 on /search'). Avoids corporate: bans press-release tone with three concrete phrases, and the 'talk like an engineer' framing is a positive alternative, but it doesn't name academic/formal register explicitly or give replacement phrasings, so a 4. Avoids AI vocabulary: provides a specific banned list (delve, leverage, robust, seamless, utilize, crucial) but it's short and misses common tells like 'tapestry', 'navigate', 'it's worth noting', 'in today's', hedging openers, and structural tells (em-dash overuse, tricolons, 'Not X, but Y'), so a 4. Connects user outcomes: 'what they see, wait for, or can no longer do' is a direct, concrete instruction to translate technical work into user experience. Overall a tight, well-constructed directive; the only gaps are the brevity of the banned-word and corporate-phrase lists."},"latency_ms":7721,"stop_reason":"end_turn","usage":{"input_tokens":495,"output_tokens":462},"cost_usd":0.02805} +{"config":"voice","item":"p07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit rule 'First sentence carries the point. Take a position; hedge only with a stated reason' plus table rows that convert hedges into flat claims and 'End with what to do next.' Concreteness: rule names files, functions, line numbers, commands, measured numbers, and every table example demonstrates it (auth.ts:47, 51 queries, 200ms, 412/412, bun test). Avoids corporate: bans corporate, academic, PR, hype tone, throat-clearing, and generic optimism, with table rows showing before/after; slightly short of 5 because beyond the table it gives few concrete alternatives for the tone categories themselves (e.g., no guidance on replacement register for PR/academic phrasing). AI vocabulary: specific banned-word list of 19 common AI tells plus an em-dash ban and a table row mocking 'delve into the robust, comprehensive solution.' User outcomes: 'Every finding says what the real user sees, loses, or waits for' plus examples translating 'users may experience difficulties' into 'white screen and log in again' and N+1 into milliseconds per load."},"latency_ms":6048,"stop_reason":"end_turn","usage":{"input_tokens":1311,"output_tokens":436},"cost_usd":0.03491} +{"config":"voice","item":"a08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":4,"reasoning":"Directness: 'Answer first, then evidence. Pick a side when the evidence supports one' is a clear instruction to lead with the point and take positions. Concreteness: explicitly demands file:line, the command run, and the printed number, with a sharp contrasting example ('Probably slow' vs '412ms p95 on /search'). Avoids corporate: bans press-release tone and names three concrete phrases to avoid, and offers the alternative ('talk like an engineer'); it doesn't address academic/formal hedging tone separately, so not a full 5. Avoids AI vocabulary: gives a specific word list (delve, leverage, robust, seamless, utilize, crucial) but the list is short and omits common phrase-level tells (e.g., 'it's worth noting', 'in today's landscape', 'navigate', 'tapestry'). Connects user outcomes: explicitly instructs describing what the user sees, waits for, or can no longer do, which is concrete and good, though brief and lacks an example tying a technical change to that experience."},"latency_ms":7010,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":407},"cost_usd":0.03043} +{"config":"voice","item":"p06","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: Rule 1 'Point first' explicitly demands the first line state the finding/recommendation and 'Take a position', with a concrete good/bad pair, plus rule 7 forces an action-oriented close. Concreteness: Rule 2 names file, function, line number, command, and real number, with an example showing a path:line, a runnable test command, and measured drift counts. Avoids corporate: Rule 4 explicitly bans corporate, academic, PR, and hype registers, supplies the alternative ('engineer at the next desk'), and contrasts a good and bad example. Avoids AI vocabulary: Rule 5 gives a specific banned-word list (delve, crucial, robust, comprehensive, etc.) and also bans em dashes. Connects user outcomes: Rule 3 requires every technical point to end in what the user sees, loses, waits for, or can do, with a concrete money-impact example. All five dimensions are explicitly addressed with examples rather than vague guidance."},"latency_ms":6810,"stop_reason":"end_turn","usage":{"input_tokens":1401,"output_tokens":394},"cost_usd":0.03371} +{"config":"voice","item":"f08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what good voice guidance looks like. Directness: it tells the agent to open with hype filler and to avoid dwelling on problems, which actively buries the point and discourages taking honest positions. Concreteness: zero mention of files, commands, line numbers, or real numbers; instead it mandates vague superlatives. Avoids corporate: it doesn't ban formal tone, it mandates marketing-speak ('world-class', 'cutting-edge', 'seamless'), which is corporate sludge. Avoids AI vocabulary: it explicitly prescribes the exact AI-tell words (revolutionary, seamless, game-changer, next-level) and emoji spam that a good directive would ban. Connects user outcomes: nothing about end users; the only 'outcome' is making the builder feel pumped. Worse, 'focus on the exciting opportunity rather than the problem' instructs the agent to soften or hide broken code, which is actively harmful for a coding assistant."},"latency_ms":7943,"stop_reason":"end_turn","usage":{"input_tokens":558,"output_tokens":394},"cost_usd":0.02528} +{"config":"voice","item":"p07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'First sentence carries the point. Take a position; hedge only with a stated reason' plus table rows converting hedges to assertions. Concreteness: explicit rule to name files, functions, line numbers, commands, measured numbers, and every table example demonstrates it (auth.ts:47, 51 queries, 412/412). Avoids corporate: bans corporate, academic, PR, hype tone, throat-clearing, generic optimism, and the table gives alternatives; loses a point because it doesn't define what corporate tone looks like beyond the handful of examples, and 'PR' tone is left unspecified. Avoids AI vocabulary: a concrete banned-word list of 19 words plus em dashes and 'delve' examples; docked because it covers single words only and misses common AI phrases (e.g. 'it's worth noting', 'I hope this helps', 'in today's fast-paced world', 'certainly'). Connects user outcomes: 'Every finding says what the real user sees, loses, or waits for' is an explicit, strong rule, reinforced by the white-screen table row."},"latency_ms":5808,"stop_reason":"end_turn","usage":{"input_tokens":1311,"output_tokens":427},"cost_usd":0.03446} +{"config":"voice","item":"p06","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: Rule 1 demands point-first, take a position, with a paired good/bad example; rule 7 reinforces ending with the action. Concreteness: Rule 2 explicitly names file, function, line number, command, and real number, and the example includes all of them (cart.ts:88, bun test command, 0.01 drift on 3 of 12). Avoids corporate: Rule 4 bans corporate, academic, PR, and hype registers, gives the 'engineer at the next desk' alternative and a good/bad pair. Avoids AI vocabulary: Rule 5 gives an explicit banned-word list (delve, crucial, robust, comprehensive, etc.) plus an em-dash ban. Connects user outcomes: Rule 3 requires every technical point to end in user impact (see, lose, wait for, can now do) with a concrete example. Every dimension is addressed explicitly with examples; the directive is also compact and checkable."},"latency_ms":6603,"stop_reason":"end_turn","usage":{"input_tokens":888,"output_tokens":377},"cost_usd":0.02773} +{"config":"voice","item":"f07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the conclusion and take a clear position' and 'builder talking to a builder' directly instruct the agent to be direct and opinionated. Concreteness is actively inverted: the directive explicitly tells the agent NOT to name file names, line numbers, exact commands, or benchmark figures, and to describe problems by category rather than location, which is the opposite of what this dimension asks for. Avoids_corporate is excellent: it bans corporate, academic, and PR tone with concrete examples ('we are pleased to', 'leverage', 'going forward') and gives the alternative ('plain words, short sentences'). Avoids_ai_vocabulary is excellent: a specific banned-word list (delve, crucial, robust, comprehensive, etc.) plus a ban on em dashes. Connects_user_outcomes is excellent: 'Always connect the work to the people using the product: what they will notice, what gets faster, what stops breaking for them' is a clear, specific instruction tying technical work to user experience."},"latency_ms":5004,"stop_reason":"end_turn","usage":{"input_tokens":1118,"output_tokens":407},"cost_usd":0.03153} +{"config":"voice","item":"a08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Answer first, then evidence. Pick a side' is a crisp instruction to lead with the point and take positions. Concreteness: explicitly demands file:line, the command run, and the printed number, with a sharp contrast example ('Probably slow' vs '412ms p95 on /search'). Avoids corporate: bans press-release tone with three concrete phrases, but gives only a loose alternative ('talk like an engineer') and doesn't address academic/formal register directly. Avoids AI vocabulary: provides a specific banned list (delve, leverage, robust, seamless, utilize, crucial), though the list is short and omits common tells like 'it's worth noting', 'dive into', 'comprehensive', 'streamline', 'navigate'. Connects user outcomes: directly instructs saying what changes for the person using the product in observable terms (see, wait for, can no longer do)."},"latency_ms":6792,"stop_reason":"end_turn","usage":{"input_tokens":495,"output_tokens":362},"cost_usd":0.02305} +{"config":"voice","item":"a08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Answer first, then evidence. Pick a side when the evidence supports one' is an explicit instruction to lead with the point and take positions. Concreteness: demands file:line, the exact command, and the printed number, with a sharp contrast example ('Probably slow' vs '412ms p95 on /search'). Avoids corporate: explicitly bans press-release tone and names three corporate phrases with the alternative framing 'talk like an engineer'; academic tone isn't addressed and the alternative is a framing rather than concrete substitutions, so 4. Avoids AI vocabulary: gives a specific banned-word list (delve, leverage, robust, seamless, utilize, crucial), but the list is short and omits common AI-tell phrases ('it's worth noting', 'in today's landscape', 'navigate', 'tapestry', hedging/summary boilerplate), so 4. Connects user outcomes: 'Say what changes for the person using the product: what they see, wait for, or can no longer do' is a direct, concrete instruction tying technical work to user experience."},"latency_ms":7048,"stop_reason":"end_turn","usage":{"input_tokens":495,"output_tokens":421},"cost_usd":0.026} +{"config":"voice","item":"a07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness is explicit and repeated: 'Lead with the point,' 'Don't dance around judgments,' 'say so plainly,' 'End with what to do.' Concreteness has its own section with worked contrasts (the exact `bun test` command, 'auth.ts:47', '~200ms per page load with 50 items'). Corporate/academic/PR/hype tone is explicitly banned, with a clear contrast ('builder talking to a builder, not a consultant presenting to a client') and a concrete alternative phrasing ('What's interesting here is...' beats 'It is important to understand...'), plus bans on filler, throat-clearing, and generic optimism. User outcomes get a dedicated section with example sentences and the 'make the user's user real' framing. The weak spot is AI vocabulary: it bans em dashes and says 'avoid cliches and words that sound machine-written,' but provides no actual list of AI-tell words or phrases (no 'delve,' 'leverage,' 'it's worth noting,' 'in today's landscape,' etc.), so an agent has no concrete reference for what to avoid."},"latency_ms":9339,"stop_reason":"end_turn","usage":{"input_tokens":1913,"output_tokens":806},"cost_usd":0.05943} +{"config":"voice","item":"a07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness is strong and explicit: 'Lead with the point,' 'Don't dance around judgments,' 'say so plainly,' 'End with what to do,' plus examples of blunt judgments ('this is a mess'). Concreteness has its own section with file/function/line-number requirements, an exact command example (`bun test test/billing.test.ts`), real-number tradeoff examples, and a before/after for pointing at auth.ts:47. Corporate tone is explicitly banned ('never corporate, never academic, never PR, never hype') with clear alternatives (builder-to-builder, YC partner / senior eng / technical-blog energy), bans on filler, throat-clearing, generic optimism, founder cosplay, and a concrete swap ('What's interesting here is...' over 'It is important to understand...'). User outcomes get a dedicated section with specific example sentences ('3-second spinner on every page load,' 'loses the customer's data') and the 'make the user's user real' framing. The weak dimension is AI vocabulary: it bans em dashes and says 'avoid cliches and words that sound machine-written' and 'never self-referential about being AI,' but provides no actual list of AI-tell words or phrases (no 'delve,' 'leverage,' 'robust,' 'seamless,' 'it's worth noting,' 'in today's fast-paced,' etc.). A single gesture at 'machine-written' words without naming them is far less effective than an explicit blocklist, so this scores low despite the em-dash rule."},"latency_ms":14978,"stop_reason":"end_turn","usage":{"input_tokens":1913,"output_tokens":1150},"cost_usd":0.07663} +{"config":"voice","item":"p07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'First sentence carries the point. Take a position; hedge only with a stated reason', plus table rows showing hedged -> direct translations and 'End with what to do next'. Concreteness: rule to name files, functions, line numbers, commands, measured numbers, and every table row demonstrates it (auth.ts:47, 51 queries/200ms, bun test 412/412). Avoids corporate: explicitly bans corporate, academic, PR, hype tone, throat-clearing, and generic optimism, with table alternatives; slightly short of 5 because the ban is a single line without naming specific corporate constructions beyond the table examples. Avoids AI vocabulary: a specific 20-word banned list (delve, robust, comprehensive, tapestry, etc.) plus a no-em-dash rule and a table row targeting the 'delve into the robust, comprehensive solution' pattern. User outcomes: 'Every finding says what the real user sees, loses, or waits for' and the 'white screen and have to log in again' example directly tie technical findings to user experience."},"latency_ms":5434,"stop_reason":"end_turn","usage":{"input_tokens":798,"output_tokens":422},"cost_usd":0.02908} +{"config":"voice","item":"f06","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"directness":4,"concreteness":3,"avoids_corporate":4,"avoids_ai_vocabulary":3,"connects_user_outcomes":1,"reasoning":"Directness is strong: 'direct, sharp, no filler, no throat-clearing, end with what to do' covers leading with the point, though it never explicitly tells the agent to take a position or state an opinion. Concreteness is partial: it names files, functions, and commands, but says nothing about line numbers, real numbers, or measured results. Corporate/academic tone is explicitly banned with a usable alternative frame ('builder, not consultant'), but no examples of what corporate phrasing looks like. AI vocabulary gets a specific but short word list (five words plus 'etc.') and an em-dash ban; no AI-tell phrases ('it's worth noting', 'in today's fast-paced', etc.) and the directive itself uses an em dash. Connecting technical work to user outcomes is absent entirely; the 'user always has context' line is about deferring to the operator, not about translating code changes into end-user impact."},"latency_ms":10944,"stop_reason":"end_turn","usage":{"input_tokens":500,"output_tokens":823},"cost_usd":0.04615} +{"config":"voice","item":"f08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it instructs the agent to open with filler hype ('Love this!') and to avoid 'dwelling on the problem' — i.e., bury the point and soften bad news rather than state it. Concreteness: no mention of files, commands, line numbers, or real numbers; it pushes vague superlatives ('revolutionary', 'next-level') instead of specifics. Avoids_corporate: it doesn't ban anything; it mandates marketing-brochure tone, which is a different flavor of empty register. Avoids_ai_vocabulary: it actively requires AI-tell/hype words (seamless, cutting-edge, game-changer, world-class) and emoji padding, plus formulaic motivational closers — the opposite of a ban list. Connects_user_outcomes: nothing ties technical changes to what real users experience; 'celebrate the builder' is about flattering the developer, not describing user impact. Worse, 'focus on the exciting opportunity' when something is broken instructs the agent to hide problems from the user, which is actively harmful for a coding assistant."},"latency_ms":7372,"stop_reason":"end_turn","usage":{"input_tokens":558,"output_tokens":436},"cost_usd":0.02738} +{"config":"voice","item":"a07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point,' 'Be direct about quality,' 'Don't dance around judgments,' 'say so plainly,' 'End with what to do,' plus explicit permission to make flat judgments like 'this is a mess.' Concreteness is the standout section: it demands file, function, and line number, gives a before/after on commands (`bun test test/billing.test.ts`), real numbers (N+1, ~200ms, 50 items), and a precise example (auth.ts:47). Corporate/academic tone is explicitly banned ('never corporate, never academic, never PR, never hype'), framed with a clear contrast (builder to builder vs consultant to client), and given alternatives ('What's interesting here is...' beats 'It is important to understand...', curious not lecturing, short punchy sentences, parentheticals). AI vocabulary is the weak spot: it bans em dashes, cliches, 'words that sound machine-written,' filler, throat-clearing, and generic optimism, but never lists the actual tell words or phrases (no 'delve,' 'leverage,' 'robust,' 'it's worth noting,' 'in conclusion,' etc.), so the agent has to guess what counts. User outcomes gets a dedicated section with concrete examples ('3-second spinner on every page load,' 'the edge case you're skipping is the one that loses the customer's data') and the broader framing of pushing toward the user, job to be done, and bottleneck."},"latency_ms":13538,"stop_reason":"end_turn","usage":{"input_tokens":2426,"output_tokens":1030},"cost_usd":0.07576} +{"config":"voice","item":"f07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the conclusion and take a clear position.' Concreteness is actively inverted: the directive explicitly tells the agent NOT to name file names, line numbers, exact commands, or benchmark figures, and to describe problems by category rather than location. That is the opposite of the dimension being scored, so 1. Corporate ban is explicit with examples and an alternative ('Plain words, short sentences'). AI vocabulary ban includes a specific word list plus em dashes. User outcomes line is explicit and well-phrased (what they notice, what gets faster, what stops breaking)."},"latency_ms":3672,"stop_reason":"end_turn","usage":{"input_tokens":605,"output_tokens":268},"cost_usd":0.01945} +{"config":"voice","item":"p08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'End with what to do', plus a good/bad example contrasting a hedged statement with a direct one. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example uses 'auth.ts:47' and 'Two lines'. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, and offers the alternative 'builder talking to a builder, not a consultant presenting to a client'. AI vocabulary: concrete banned-word list (delve, crucial, robust, etc.), banned phrases list, and no em dashes. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', reinforced in the example ('Users hit a white screen'). Minor nit: the bounded-closer paragraph itself uses an em dash, contradicting its own rule, but the directive content covers every dimension explicitly with examples."},"latency_ms":9108,"stop_reason":"end_turn","usage":{"input_tokens":1190,"output_tokens":415},"cost_usd":0.03265} +{"config":"voice","item":"f08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of a good voice directive. Directness: it instructs the agent to open with hype filler rather than the point, and to avoid 'dwelling on the problem' when something is broken—actively discouraging taking clear positions. Concreteness: zero mention of files, commands, line numbers, or real numbers; it pushes vague superlatives instead. Avoids corporate: no ban on formal/academic tone, and it substitutes an equally hollow marketing register. Avoids AI vocabulary: it explicitly mandates classic AI-tell words (revolutionary, seamless, cutting-edge, next-level, world-class, game-changer) and emoji padding. Connects user outcomes: nothing ties technical work to what users actually experience; it frames everything as celebration of the builder rather than impact on end users."},"latency_ms":6637,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":335},"cost_usd":0.02746} +{"config":"voice","item":"f07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the conclusion and take a clear position' and builder-to-builder framing. Concreteness is actively inverted: the directive explicitly tells the agent NOT to name file names, line numbers, exact commands, or benchmark figures, and to describe problems by category rather than location. This is the opposite of what the dimension asks for, so it scores the minimum. Corporate/formal/academic tone is explicitly banned with concrete examples ('we are pleased to', 'leverage', 'going forward') and an alternative is given (plain words, short sentences). AI vocabulary is banned with a specific list of words plus em dashes. User outcomes are directly addressed: 'Always connect the work to the people using the product: what they will notice, what gets faster, what stops breaking for them.'"},"latency_ms":3957,"stop_reason":"end_turn","usage":{"input_tokens":605,"output_tokens":331},"cost_usd":0.0226} +{"config":"voice","item":"p06","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Rule 1 mandates point-first and taking a position with a concrete good/bad pair, and rule 7 reinforces ending with the action. Rule 2 explicitly requires file, function, line number, command, and real numbers, with an example containing all of them. Rule 4 names and bans corporate, academic, PR, and hype registers, provides a good/bad contrast, and offers the alternative ('engineer at the next desk'). Rule 5 gives a specific banned-word list of 17 AI-tell words plus em dashes. Rule 3 requires every technical point to end in user impact with a clear example translating float drift into customers overcharged. Each dimension is addressed directly, specifically, and with examples."},"latency_ms":6206,"stop_reason":"end_turn","usage":{"input_tokens":888,"output_tokens":297},"cost_usd":0.02373} +{"config":"voice","item":"p06","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: rule 1 'Point first' demands the first line state the finding or recommendation and 'take a position', with a concrete good/bad pair; rule 7 reinforces by requiring a closing action. Concreteness: rule 2 explicitly lists file, function, line number, command, and real number, with a strong example using `cart.ts:88` and a bun test command with actual counts. Avoids corporate: rule 4 names corporate, academic, PR, and hype registers as banned, gives the alternative ('engineer at the next desk') and a good/bad pair. Avoids AI vocabulary: rule 5 provides an explicit banned word list (delve, crucial, robust, comprehensive, nuanced, multifaceted, furthermore, moreover, additionally, pivotal, landscape, tapestry, underscore, foster, showcase, intricate, vibrant) plus a ban on em dashes. Connects user outcomes: rule 3 requires every technical point to end in user impact (see, lose, wait for, can now do) with a concrete example about customers being overcharged. Every dimension is covered explicitly with examples; the directive is compact and actionable."},"latency_ms":7410,"stop_reason":"end_turn","usage":{"input_tokens":888,"output_tokens":446},"cost_usd":0.03118} +{"config":"voice","item":"f06","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"directness":3,"concreteness":3,"avoids_corporate":4,"avoids_ai_vocabulary":3,"connects_user_outcomes":1,"reasoning":"Directness: says 'direct, sharp, no filler, no throat-clearing, end with what to do' but never tells the agent to lead with the point or take a position; it asserts a tone without operationalizing it. Concreteness: 'Name the file, the function, the command' is good but omits line numbers and real numbers/metrics. Avoids corporate: explicit 'never corporate, never academic' with a contrast ('builder, not a consultant') is solid, though terse and without examples of what to say instead. Avoids AI vocabulary: provides a short named list (delve, crucial, robust, comprehensive, nuanced) plus the em-dash ban, but trails off with 'etc.' and misses many common tells (leverage, seamless, streamline, 'it's worth noting', 'in today's', etc.); the directive also uses an em dash itself. Connects user outcomes: nothing instructs the agent to tie technical changes to what users actually experience; the 'user has context you don't' line is about deference, not outcomes."},"latency_ms":11607,"stop_reason":"end_turn","usage":{"input_tokens":500,"output_tokens":772},"cost_usd":0.0436} +{"config":"voice","item":"p06","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: Rule 1 demands the first line state the point and take a position, with a sharp good/bad pair ('Don't merge...' vs hedging), and Rule 7 forces an action-oriented close. Concreteness: Rule 2 explicitly requires file, function, line number, command, and real numbers, illustrated with a realistic `cart.ts:88` / `bun test` example and actual counts (0.01 drift, 3 of 12). Avoids corporate: Rule 4 names and bans corporate, academic, PR, and hype registers, supplies the alternative ('engineer at the next desk'), and contrasts 'This is broken, here's the fix' with consultant-speak. Avoids AI vocabulary: Rule 5 gives a specific banned list of 17 common AI-tell words plus an em-dash ban. Connects user outcomes: Rule 3 requires every technical point to end in what the user sees, loses, waits for, or can do, with a concrete example (customers overcharged a cent on ~1 in 4 orders) vs the abstract 'affects correctness'. The directive is tight, example-driven, and closes with a healthy recommend-don't-decide stance."},"latency_ms":8017,"stop_reason":"end_turn","usage":{"input_tokens":888,"output_tokens":456},"cost_usd":0.03168} +{"config":"voice","item":"a07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness is excellent: 'Lead with the point,' 'Don't dance around judgments,' 'say so plainly,' 'End with what to do,' punchy sentence examples. Concreteness is excellent with a dedicated section giving before/after examples (bun test command, N+1 ~200ms, auth.ts:47). Corporate tone is explicitly banned ('never corporate, never academic, never PR, never hype') with a clear alternative framing (builder-to-builder, not consultant-to-client, 'What's interesting here is...' over 'It is important to understand...'). User outcomes get a dedicated section with concrete examples (3-second spinner, lost customer data). The weak spot is AI vocabulary: it bans em dashes and says 'avoid cliches and words that sound machine-written' and 'avoid filler, throat-clearing, generic optimism,' but provides no specific list of AI-tell words or phrases (delve, leverage, robust, 'it's worth noting', 'in today's fast-paced', etc.). A vague instruction to avoid machine-sounding words is much less effective than an enumerated banlist."},"latency_ms":11438,"stop_reason":"end_turn","usage":{"input_tokens":1913,"output_tokens":985},"cost_usd":0.06838} +{"config":"voice","item":"f07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'lead with the conclusion and take a clear position.' Concreteness is actively inverted; the directive explicitly forbids naming file names, line numbers, exact commands, or benchmark figures and tells the agent to describe problems by vague category rather than location, which is the opposite of what the dimension asks for. Corporate/academic/PR tone is explicitly banned with concrete examples and a plain-words alternative. AI vocabulary is banned with a specific list of words plus em dashes. User outcomes are explicitly required: what users notice, what gets faster, what stops breaking."},"latency_ms":3483,"stop_reason":"end_turn","usage":{"input_tokens":605,"output_tokens":264},"cost_usd":0.01925} +{"config":"voice","item":"f06","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":3,"connects_user_outcomes":1,"reasoning":"Directness: 'direct, concrete, sharp', 'no filler, no throat-clearing', 'end with what to do' covers leading with the point and action, though it never explicitly tells the agent to take positions or state opinions. Concreteness: 'Name the file, the function, the command' is strong, but omits line numbers and real numbers/measurements. Avoids corporate: bans corporate and academic tone and gives one alternative ('builder, not a consultant'), but offers no concrete substitutions or examples of what the banned tone looks like. Avoids AI vocabulary: provides a short list (delve, crucial, robust, comprehensive, nuanced) plus an em dash ban, but the list is minimal, ends in 'etc.', and covers no phrase-level tells (e.g. 'it's worth noting', 'great question', tricolons). Connects user outcomes: nothing here ties technical work to what the end user experiences; the 'user has context' note is about deference to the operator, not about user outcomes."},"latency_ms":5541,"stop_reason":"end_turn","usage":{"input_tokens":1013,"output_tokens":395},"cost_usd":0.02988} +{"config":"voice","item":"p07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit rule that the first sentence carries the point, take a position, hedge only with a stated reason; table examples model it. Concreteness: rule names files, functions, line numbers, commands, measured numbers, and every table row demonstrates it (auth.ts:47, 51 queries/200ms, bun test 412/412). Avoids corporate: explicitly bans corporate, academic, PR, hype tone, throat-clearing, generic optimism, and the table provides concrete alternatives; slightly short of 5 because the ban is a single line without naming specific corporate phrases beyond the table examples (e.g. 'leverage', 'stakeholders', 'going forward'). Avoids AI vocabulary: a long specific banned-word list covering the usual tells plus em dashes, and the table shows the replacement. Connects user outcomes: direct rule that every finding says what the real user sees, loses, or waits for, with a matching table row (white screen, log in again)."},"latency_ms":5155,"stop_reason":"end_turn","usage":{"input_tokens":798,"output_tokens":385},"cost_usd":0.02723} +{"config":"voice","item":"a07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point,' 'Don't dance around judgments,' 'say so plainly,' 'Avoid filler, throat-clearing,' and 'End with what to do.' Concreteness is excellent with a dedicated section giving paired bad/good examples (file:line, exact bun test command, N+1 with ~200ms numbers). Corporate tone is explicitly banned ('never corporate, never academic, never PR, never hype') with clear alternatives (builder-to-builder, YC partner / senior eng / technical blog post energy, curious-not-lecturing phrasing examples). User outcomes get a dedicated section with concrete examples (3-second spinner, lost customer data) plus the recurring 'make the user's user real' framing. The weak spot is AI vocabulary: it bans em dashes, cliches, 'words that sound machine-written,' and AI self-reference, but never provides a specific list of AI-tell words or phrases (e.g., delve, leverage, robust, 'it's worth noting'), so the agent has to guess what counts."},"latency_ms":12492,"stop_reason":"end_turn","usage":{"input_tokens":2426,"output_tokens":896},"cost_usd":0.06906} +{"config":"voice","item":"f07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the conclusion and take a clear position', builder-to-builder framing. Concreteness is actively inverted: the directive explicitly tells the agent NOT to name file names, line numbers, exact commands, or benchmark figures, and to describe problems by category rather than location. This is the opposite of the dimension's intent, so it scores the minimum. Corporate/academic/PR tone is explicitly banned with concrete examples ('we are pleased to', 'leverage', 'going forward') and an alternative is given ('Plain words, short sentences'). AI vocabulary is banned with a specific list of 13 words plus em dashes. User outcomes are explicitly required with concrete framing ('what they will notice, what gets faster, what stops breaking for them')."},"latency_ms":3854,"stop_reason":"end_turn","usage":{"input_tokens":605,"output_tokens":326},"cost_usd":0.02235} +{"config":"voice","item":"a08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Answer first, then evidence. Pick a side' explicitly instructs leading with the point and taking positions. Concreteness: cites file:line, the command run, and the printed number, with a sharp before/after example ('412ms p95 on /search'). Avoids corporate: bans press-release tone with three concrete banned phrases and offers the alternative ('talk like an engineer'), but doesn't address academic/formal register or hedging patterns explicitly. Avoids AI vocabulary: provides a specific banned-word list, though it's short (six words) and omits common AI-tell phrases like 'it's worth noting', 'in today's landscape', 'navigate', 'tapestry', 'multifaceted'. Connects user outcomes: directly instructs saying what changes for the person using the product in concrete sensory terms (see, wait for, can no longer do)."},"latency_ms":6652,"stop_reason":"end_turn","usage":{"input_tokens":1008,"output_tokens":353},"cost_usd":0.02773} +{"config":"voice","item":"f08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive actively works against every dimension. Directness: it tells the agent to open with hype filler and avoid 'dwelling on the problem' when something is broken, which is the opposite of leading with the point and taking positions. Concreteness: nothing about files, commands, line numbers, or real numbers; it pushes vague superlatives instead. Avoids corporate: no ban on formal/academic tone, and it mandates a different kind of hollow register (marketing hype). Avoids AI vocabulary: it explicitly prescribes AI-tell/marketing words (revolutionary, seamless, cutting-edge, next-level, world-class) and forced openers/closers rather than banning them. Connects user outcomes: no mention of real user experience at all; it's about flattering the builder and inflating the work. This is essentially an anti-pattern document."},"latency_ms":7205,"stop_reason":"end_turn","usage":{"input_tokens":558,"output_tokens":340},"cost_usd":0.02258} +{"config":"voice","item":"f07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the conclusion and take a clear position' and 'builder talking to a builder' directly instruct leading with the point and taking stances. Concreteness is the opposite of what the dimension asks for: the directive explicitly tells the agent NOT to use file names, line numbers, exact commands, or benchmark figures, and to describe problems by category rather than location. That actively works against specificity, so it scores the minimum. Avoids corporate is excellent: it bans corporate, academic, and PR tone, gives concrete examples ('we are pleased to', 'leverage', 'going forward'), and offers the alternative (plain words, short sentences). Avoids AI vocabulary is excellent: a specific banned-word list (delve, crucial, robust, comprehensive, nuanced, multifaceted, furthermore, moreover, pivotal, landscape, tapestry, foster, showcase) plus a ban on em dashes. Connects user outcomes is excellent: 'Always connect the work to the people using the product: what they will notice, what gets faster, what stops breaking for them' is explicit and framed in user-experience terms."},"latency_ms":5649,"stop_reason":"end_turn","usage":{"input_tokens":1118,"output_tokens":443},"cost_usd":0.03333} +{"config":"voice","item":"a08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Answer first, then evidence. Pick a side' is explicit and concise. Concreteness: demands file:line, the command, and the printed number, with a sharp contrast example ('412ms p95 on /search'). Avoids corporate: bans press-release tone with three concrete phrase examples and gives the alternative ('talk like an engineer'), but the list is short and doesn't address academic/formal hedging. Avoids AI vocabulary: provides a specific banned-word list, but it's only six words and omits common AI phrase-tells (e.g., 'it's worth noting', 'in today's landscape', 'navigate', 'tapestry'). Connects user outcomes: directly instructs naming what the user sees, waits for, or loses — concrete and well-framed."},"latency_ms":8915,"stop_reason":"end_turn","usage":{"input_tokens":495,"output_tokens":323},"cost_usd":0.0211} +{"config":"voice","item":"f06","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness: explicitly calls for direct, sharp, no filler or throat-clearing, and 'end with what to do', but never says to lead with the point or take positions; the closing 'user decides' line slightly softens stance-taking. Concreteness: 'Name the file, the function, the command' is solid, but omits line numbers and real numbers. Avoids corporate: bans corporate and academic tone and offers a clear alternative (builder, not consultant), though briefly. Avoids AI vocabulary: provides a specific banned-word list (delve, crucial, robust, comprehensive, nuanced) plus an em dash ban, but the list is short and trails off with 'etc.', and the directive itself violates its own em dash rule. Connects user outcomes: nothing instructs the agent to tie technical work to real user experience or impact; the only user mention is about deference to the user's context, which is a different concern."},"latency_ms":10927,"stop_reason":"end_turn","usage":{"input_tokens":500,"output_tokens":837},"cost_usd":0.04685} +{"config":"voice","item":"p06","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Rule 1 explicitly mandates leading with the point and taking a position, with a good/bad pair. Rule 2 names file, function, line number, command, and real number with a concrete example. Rule 4 bans corporate, academic, PR, and hype register and gives the alternative ('engineer at the next desk') with examples. Rule 5 provides an explicit list of 17 banned AI-tell words plus em dashes. Rule 3 requires every technical point to end in user impact (what they see, lose, wait for, or can now do) with an example. Every dimension is directly addressed with instructions and contrasting examples."},"latency_ms":5941,"stop_reason":"end_turn","usage":{"input_tokens":1401,"output_tokens":267},"cost_usd":0.02736} +{"config":"voice","item":"p07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit rule 'First sentence carries the point. Take a position; hedge only with a stated reason' plus table examples replacing hedges with assertions. Concreteness: rule names files, functions, line numbers, commands, measured numbers, and every table row demonstrates it (auth.ts:47, 51 queries, 412/412). Avoids corporate: bans corporate/academic/PR/hype tone and throat-clearing, and the table supplies alternatives, but gives no named alternative register beyond 'builder to builder' and no examples specifically targeting PR/hype phrasing, so a 4. Avoids AI vocabulary: concrete banned-word list of 19 terms plus an em dash ban, and the table shows 'delve/robust/comprehensive' being cut. Connects user outcomes: 'Every finding says what the real user sees, loses, or waits for' with a matching example (white screen, log in again)."},"latency_ms":5565,"stop_reason":"end_turn","usage":{"input_tokens":798,"output_tokens":372},"cost_usd":0.02658} +{"config":"voice","item":"p08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'End with what to do', plus a good/bad example contrasting a hedged corporate sentence with a blunt one. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers; the example cites auth.ts:47 and '/login'. Avoids corporate: explicit ban on corporate, academic, PR, hype, filler, throat-clearing, generic optimism, with the alternative 'builder talking to a builder, not a consultant presenting to a client'. AI vocabulary: enumerated word list (delve, crucial, robust, etc.) plus a banned-phrases list and em dash ban. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', demonstrated in the example ('Users hit a white screen'). The bounded closer section reinforces directness. Minor nit: the directive itself uses an em dash in the closer exemption paragraph, but that doesn't weaken the instructions."},"latency_ms":9351,"stop_reason":"end_turn","usage":{"input_tokens":1190,"output_tokens":420},"cost_usd":0.0329} +{"config":"voice","item":"f07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the conclusion and take a clear position' is exactly the instruction. Concreteness is the opposite of what the rubric wants: the directive explicitly tells the agent NOT to name file names, line numbers, exact commands, or benchmark figures, and to describe problems by category rather than location. That actively works against specificity, so it earns the lowest score. Corporate/formal/academic tone is explicitly banned with concrete examples ('we are pleased to', 'leverage', 'going forward') and an alternative ('Plain words, short sentences'). AI vocabulary is banned with a specific, substantial word list plus em dashes. User outcomes are directly addressed: connect the work to what people will notice, what gets faster, what stops breaking."},"latency_ms":4608,"stop_reason":"end_turn","usage":{"input_tokens":605,"output_tokens":320},"cost_usd":0.02205} +{"config":"voice","item":"a07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":3,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the point,' 'Be direct about quality... Don't dance around judgments,' 'say so plainly,' 'End with what to do.' Concreteness has its own dedicated section with explicit before/after examples (exact command `bun test test/billing.test.ts`, 'auth.ts:47', '~200ms per page load with 50 items'). Corporate tone is explicitly banned ('never corporate, never academic, never PR, never hype'), with clear alternatives offered (builder-to-builder not consultant-to-client, YC partner / senior eng / technical-blog energy, 'What's interesting here is...' over 'It is important to understand...', bans on filler and throat-clearing). User outcomes get a dedicated section with concrete examples ('3-second spinner on every page load', 'the edge case you're skipping is the one that loses the customer's data'). The weak spot is AI vocabulary: it bans em dashes and says 'avoid cliches and words that sound machine-written' and 'never self-referential about being AI,' but there is no specific list of AI-tell words or phrases (e.g., delve, leverage, robust, seamless, 'it's worth noting', 'in today's fast-paced world'), so the agent has little to pattern-match against beyond the em dash rule."},"latency_ms":12363,"stop_reason":"end_turn","usage":{"input_tokens":1913,"output_tokens":1097},"cost_usd":0.07398} +{"config":"voice","item":"a07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness is explicit and repeated: 'Lead with the point,' 'Be direct about quality,' 'Don't dance around judgments,' 'say so plainly,' 'End with what to do.' Concreteness has its own section with before/after examples (file:line, exact bun test command, N+1 with ~200ms numbers), plus a reinforcing writing rule. Corporate tone is explicitly banned ('never corporate, never academic, never PR, never hype') with named alternatives (builder-to-builder, YC partner / senior eng / technical-blog-post energy) and a contrastive example ('What's interesting here is...' over 'It is important to understand...'). User outcomes get a dedicated section with concrete sample sentences and the 'make the user's user real' framing. The weak spot is AI vocabulary: it bans em dashes, 'cliches and words that sound machine-written,' filler, throat-clearing, and generic optimism, but provides no actual list of AI-tell words or phrases (no 'delve,' 'leverage,' 'robust,' 'it's worth noting,' 'in today's landscape,' etc.), so the rubric's 'specific lists' requirement is largely unmet."},"latency_ms":10531,"stop_reason":"end_turn","usage":{"input_tokens":1913,"output_tokens":888},"cost_usd":0.06353} +{"config":"voice","item":"a08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":5,"reasoning":"Directness: 'Answer first, then evidence' and 'Pick a side' directly cover leading with the point and taking positions. Concreteness: explicitly requires file:line, the command run, and the printed number, with a sharp before/after example (412ms p95 on /search). Avoids corporate: bans press-release tone with three specific phrases, but the alternative is only implied ('talk like an engineer') rather than shown with rewrites, and academic/formal register isn't addressed. Avoids AI vocabulary: a real banned-word list, but short (six words) and no phrase-level tells like 'it's worth noting' or 'in today's landscape'. Connects user outcomes: 'what they see, wait for, or can no longer do' is a concrete, actionable framing of user impact."},"latency_ms":6325,"stop_reason":"end_turn","usage":{"input_tokens":495,"output_tokens":321},"cost_usd":0.021} +{"config":"voice","item":"a07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":2,"connects_user_outcomes":5,"reasoning":"Directness is strong and repeated: 'Lead with the point,' 'Don't dance around judgments,' 'say so plainly,' 'End with what to do.' Concreteness is exemplary, with a dedicated section giving before/after examples (bun test command, N+1 at ~200ms, auth.ts:47). Corporate/academic/PR/hype tone is explicitly banned and replaced with concrete alternatives (builder-to-builder, YC partner / senior eng / technical blog energy, 'What's interesting here is...' over 'It is important to understand...'). User outcomes get their own section with vivid examples ('3-second spinner,' 'loses the customer's data') plus the whole framing around the job to be done. The weak spot is AI vocabulary: it bans em dashes and says 'avoid cliches and words that sound machine-written' and 'filler, throat-clearing, generic optimism,' but provides no actual list of AI-tell words or phrases (delve, leverage, robust, 'it's worth noting,' 'in today's fast-paced,' etc.). Without a concrete list the agent has little to enforce against, so this dimension scores low despite the intent being present."},"latency_ms":11833,"stop_reason":"end_turn","usage":{"input_tokens":1913,"output_tokens":969},"cost_usd":0.06758} +{"config":"voice","item":"p07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit rule 'First sentence carries the point. Take a position; hedge only with a stated reason,' reinforced by table rows that replace hedges with assertions. Concreteness: 'Name files, functions, line numbers, commands, and measured numbers,' with every table row modeling it (auth.ts:47, 51 queries/200ms, bun test 412/412). Avoids corporate: explicitly bans corporate, academic, PR, hype tone, throat-clearing, and generic optimism, and the table supplies concrete alternatives for each. Avoids AI vocabulary: a specific 20-word banned list plus em dashes, with table examples (delve, robust, comprehensive) showing the translation. User outcomes: 'Every finding says what the real user sees, loses, or waits for,' demonstrated by the white-screen and 200ms rows. Compact, example-driven, and every dimension has both a rule and a worked example."},"latency_ms":4905,"stop_reason":"end_turn","usage":{"input_tokens":798,"output_tokens":376},"cost_usd":0.02678} +{"config":"voice","item":"f07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the conclusion and take a clear position' and 'builder talking to a builder' directly instruct position-taking and leading with the point. Concreteness is the opposite of what is wanted: the directive explicitly tells the agent NOT to name file names, line numbers, exact commands, or benchmark figures, and to describe problems by category rather than location, so it actively undermines this dimension. Corporate/formal tone is explicitly banned with concrete examples ('we are pleased to', 'leverage', 'going forward') and an alternative is given ('Plain words, short sentences'). AI vocabulary is banned with a specific word list plus a ban on em dashes. User outcomes are covered well: 'Always connect the work to the people using the product: what they will notice, what gets faster, what stops breaking for them.'"},"latency_ms":4427,"stop_reason":"end_turn","usage":{"input_tokens":1118,"output_tokens":341},"cost_usd":0.02823} +{"config":"voice","item":"p06","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: Rule 1 demands the first line state the finding or recommendation and take a position, with a sharp good/bad pair; rule 7 reinforces with an action-oriented close. Concreteness: Rule 2 explicitly requires file, function, line number, command, and real numbers, and the example shows all of them (`cart.ts:88`, `bun test ...`, 3 of 12). Avoids corporate: Rule 4 names and bans corporate, academic, PR, and hype registers, gives a replacement register ('engineer at the next desk') and a contrasting example. Avoids AI vocabulary: Rule 5 lists 17 specific AI-tell words plus bans em dashes, which is a concrete, enforceable list. Connects user outcomes: Rule 3 requires every technical point to end in user impact with a clear good/bad pair. The directive is tight, every rule is illustrated, and it includes a pre-send check instruction. Minor possible additions (e.g., longer banned-phrase list like 'it's worth noting', or guidance on hedging) wouldn't move any dimension below 5."},"latency_ms":7811,"stop_reason":"end_turn","usage":{"input_tokens":888,"output_tokens":431},"cost_usd":0.03043} +{"config":"voice","item":"p08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'End with what to do', and the good/bad example contrast ('auth.ts:47 returns undefined' vs 'potential issue... may cause problems') all push toward taking positions. Concreteness: explicitly names files, functions, line numbers, commands, outputs, evals, real numbers, and the example demonstrates it (auth.ts:47, /login, 'two lines'). Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the alternative framing 'builder talking to a builder, not a consultant presenting to a client'. Avoids AI vocabulary: specific banned-word list (delve, crucial, robust, etc.), banned-phrase list (here's the kicker, plot twist, etc.), and no em dashes. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', reinforced by the example ('Users hit a white screen'). The bounded closer section adds a well-scoped anti-verbosity rule with exemptions. Minor nit: the directive itself uses an em dash in the closer exemption text, slightly undercutting its own rule, but this doesn't weaken the instructions to the agent."},"latency_ms":7857,"stop_reason":"end_turn","usage":{"input_tokens":1190,"output_tokens":508},"cost_usd":0.0373} +{"config":"voice","item":"f08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of good voice guidance on nearly every axis. Directness: it tells the agent to open with hype filler ('Love this!') and to avoid 'dwelling on the problem' when something is broken, which is the opposite of leading with the point or taking a position. Concreteness: no mention of files, commands, line numbers, or real numbers anywhere; it actively pushes inflation ('frame every change as a game-changer'). Avoids corporate: it doesn't ban corporate tone and instead mandates marketing-speak. Avoids AI vocabulary: it explicitly requires AI-tell words (revolutionary, seamless, cutting-edge, next-level, world-class) and emoji padding, with sycophantic openers and motivational closers. Connects user outcomes: nothing ties technical work to what users experience; the only 'outcome' is making the builder feel pumped. Bottom of the scale across the board."},"latency_ms":6896,"stop_reason":"end_turn","usage":{"input_tokens":1071,"output_tokens":370},"cost_usd":0.02921} +{"config":"voice","item":"a08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":4,"reasoning":"Directness: 'Answer first, then evidence. Pick a side when the evidence supports one' is a clear, complete instruction to lead with the point and take positions. Concreteness: explicitly requires file:line, the command run, and the printed number, with a sharp contrasting example ('Probably slow' vs '412ms p95 on /search'). Avoids corporate: bans press-release tone with three concrete banned phrases and gives the alternative framing ('talk like an engineer'), but the list is short and doesn't address academic/formal hedging explicitly. Avoids AI vocabulary: a specific list of six AI-tell words, but it's brief and omits common tells like 'additionally', 'it's worth noting', 'landscape', 'comprehensive', and phrase-level patterns. Connects user outcomes: directly instructs saying what changes for the person using the product with concrete dimensions (see, wait for, can no longer do); strong but terse, no guidance on when it's required or examples."},"latency_ms":7039,"stop_reason":"end_turn","usage":{"input_tokens":495,"output_tokens":389},"cost_usd":0.0244} +{"config":"voice","item":"p06","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: rule 1 explicitly demands the first line state the point and take a position, with a good/bad pair; rule 7 forces an action-oriented close. Concreteness: rule 2 names file, function, line number, command, and real number, with an example that includes a path:line, a runnable command, and measured drift. Avoids corporate: rule 4 bans corporate, academic, PR, and hype registers and offers the 'engineer at the next desk' alternative with contrasting examples. Avoids AI vocabulary: rule 5 gives an explicit banned word list (delve, crucial, robust, etc.) plus a ban on em dashes. Connects user outcomes: rule 3 requires every technical point to end in user impact and models it with a concrete customer-facing consequence. Each dimension is covered with an explicit instruction and a good/bad contrast, and the directive is short enough to be checkable."},"latency_ms":6739,"stop_reason":"end_turn","usage":{"input_tokens":888,"output_tokens":360},"cost_usd":0.02688} +{"config":"voice","item":"f06","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"directness":4,"concreteness":4,"avoids_corporate":3,"avoids_ai_vocabulary":3,"connects_user_outcomes":1,"reasoning":"Directness: says 'direct, sharp, no filler, no throat-clearing, end with what to do' but never explicitly says lead with the point or take positions. Concreteness: 'Name the file, the function, the command' is good but omits line numbers and real numbers. Avoids corporate: bans corporate/academic tone and gives the 'builder, not consultant' framing, but offers no concrete alternatives or examples of what to say instead. AI vocabulary: provides a short list (delve, crucial, robust, comprehensive, nuanced) plus em dash ban, but the list is thin and trails off with 'etc.' with no phrase-level bans; it also ironically uses an em dash in its own text. User outcomes: nothing instructs connecting technical work to real user experience; the 'user has context' line is about deferring to the human operator, not about end-user impact."},"latency_ms":4725,"stop_reason":"end_turn","usage":{"input_tokens":1013,"output_tokens":346},"cost_usd":0.02743} +{"config":"voice","item":"p08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: explicit 'Lead with the point', 'Be direct about quality', 'End with what to do', plus a good/bad example that models leading with the finding and the fix. Concreteness: directly instructs naming files, functions, line numbers, commands, outputs, evals, real numbers, and the example (auth.ts:47, /login, 'Two lines') demonstrates it. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, consultant tone, and offers the 'builder talking to a builder' alternative plus a bad example of consultant-speak; slightly short of 5 because the alternative is a framing rather than concrete substitutions for specific corporate phrases. AI vocabulary: specific word list and a specific banned-phrase list, plus the em dash ban. User outcomes: explicit 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', and the example ties a null return to 'Users hit a white screen.' Note the directive itself uses an em dash in the bounded closer exemption, a minor inconsistency with its own rule."},"latency_ms":7761,"stop_reason":"end_turn","usage":{"input_tokens":1703,"output_tokens":436},"cost_usd":0.03883} +{"config":"voice","item":"f08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it instructs the agent to open with filler hype ('Love this!') and to avoid 'dwelling on the problem' when something is broken — that is burying the lede and dodging positions, not leading with the point. Concreteness: nothing about files, commands, line numbers, or real numbers; it pushes vague superlatives instead. Avoids_corporate: it does not ban formal/corporate tone, and 'world-class', 'cutting-edge', 'seamless' are marketing-speak, so it actively prescribes a corporate-adjacent register. Avoids_ai_vocabulary: it mandates classic AI-tell words (revolutionary, seamless, cutting-edge, next-level, game-changer) and emojis rather than banning them. Connects_user_outcomes: no mention of real users or how technical work affects them; the only 'outcome' is making the builder feel pumped. Across the board, it would produce exactly the inflated, evasive, sycophantic prose a good voice directive is supposed to prevent."},"latency_ms":7111,"stop_reason":"end_turn","usage":{"input_tokens":558,"output_tokens":419},"cost_usd":0.02653} +{"config":"voice","item":"p08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'End with what to do', and a good/bad pair that contrasts a hedged consultant sentence with a flat statement of the bug and fix. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, and real numbers, with an example that uses auth.ts:47 and a two-line fix. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, and founder cosplay, and supplies the alternative (builder talking to a builder). Avoids AI vocabulary: a specific word list (delve, crucial, robust, etc.), a specific phrase list, and a no-em-dash rule. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', reinforced by the example ('Users hit a white screen'). Minor blemish: the bounded closer paragraph itself uses an em dash, contradicting the rule, but that does not weaken the instruction itself."},"latency_ms":7421,"stop_reason":"end_turn","usage":{"input_tokens":1190,"output_tokens":425},"cost_usd":0.03315} +{"config":"voice","item":"f07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"directness":5,"concreteness":1,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness is strong: 'Lead with the conclusion and take a clear position' is explicit. Concreteness is actively inverted: the directive forbids naming file names, line numbers, exact commands, and benchmark figures, and tells the agent to describe problems by category rather than location, which is the opposite of what the dimension asks for. Corporate/formal/academic tone is explicitly banned with concrete examples and an alternative ('Plain words, short sentences'). AI vocabulary is banned with a specific word list plus em dashes. User outcomes are explicitly required with concrete framing (what users notice, what gets faster, what stops breaking)."},"latency_ms":3703,"stop_reason":"end_turn","usage":{"input_tokens":605,"output_tokens":278},"cost_usd":0.01995} +{"config":"voice","item":"f08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of good voice guidance on every axis. Directness: it tells the agent to open with hype filler ('Love this!', 'Huge win!') and close with a motivational line, burying the point and explicitly discouraging taking a clear position on problems ('focus on the exciting opportunity rather than dwelling on the problem'). Concreteness: nothing about files, commands, line numbers, or real numbers; it pushes vague superlatives instead of specifics. Avoids corporate: it doesn't ban corporate/formal tone and actively prescribes marketing-speak ('revolutionary, seamless, cutting-edge, next-level, world-class'). Avoids AI vocabulary: it mandates the exact AI-tell words and sycophantic openers a good directive would ban, plus emoji spam. Connects user outcomes: no instruction to tie work to real user experience; it inflates every change ('frame every change as a game-changer') rather than honestly describing impact. It also instructs the agent to soften or hide broken behavior, which is a correctness and trust hazard."},"latency_ms":7406,"stop_reason":"end_turn","usage":{"input_tokens":558,"output_tokens":421},"cost_usd":0.02663} +{"config":"voice","item":"f06","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"directness":4,"concreteness":3,"avoids_corporate":4,"avoids_ai_vocabulary":4,"connects_user_outcomes":1,"reasoning":"Directness: explicitly calls for direct, sharp, no filler, no throat-clearing, and 'end with what to do', but never says to lead with the point or take a position, so strong but not complete. Concreteness: 'Name the file, the function, the command' is good but omits line numbers and real numbers/measurements. Avoids corporate: explicitly bans corporate and academic tone and offers the 'builder, not consultant' alternative, though it gives no examples of what corporate phrasing to replace. Avoids AI vocabulary: bans em dashes and gives a specific word list (delve, crucial, robust, comprehensive, nuanced), but the list is short, trails off with 'etc.', skips common phrase-level tells (e.g. 'it's worth noting', 'in today's landscape'), and the directive itself uses an em dash in its final line. Connects user outcomes: nothing instructs the agent to tie technical work to real user experience; the 'user has context you don't' line is about deference on decisions, not about framing work in terms of user impact."},"latency_ms":10654,"stop_reason":"end_turn","usage":{"input_tokens":500,"output_tokens":782},"cost_usd":0.0441} +{"config":"voice","item":"f08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"directness":1,"concreteness":1,"avoids_corporate":1,"avoids_ai_vocabulary":1,"connects_user_outcomes":1,"reasoning":"This directive is the inverse of what each dimension asks for. Directness: it instructs the agent to open with filler hype ('Love this!'), avoid dwelling on problems, and spin broken things as 'exciting opportunities' — the opposite of leading with the point and taking honest positions. Concreteness: nothing about naming files, commands, line numbers, or real numbers; it actively pushes toward vague inflation ('frame every change as a game-changer'). Avoids corporate: it doesn't ban formal tone at all, and instead mandates a different performative register plus emojis; no alternatives for plain speech. Avoids AI vocabulary: it explicitly *requires* classic AI-tell/marketing words (revolutionary, seamless, cutting-edge, next-level, world-class) and a motivational closing line — the exact patterns a good directive would ban. Connects user outcomes: no mention of end users or real-world impact; the focus is on flattering the builder, not on what the change does for users. Bottom scores across the board."},"latency_ms":7723,"stop_reason":"end_turn","usage":{"input_tokens":558,"output_tokens":401},"cost_usd":0.02563} +{"config":"voice","item":"p08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":5,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'End with what to do', plus the good/bad example contrasting a crisp diagnosis against hedged vagueness. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example shows auth.ts:47 with a two-line fix. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, generic optimism, founder cosplay, and gives the alternative ('builder talking to a builder, not a consultant presenting to a client'). Avoids AI vocabulary: a specific word list (delve, crucial, robust, etc.), a specific banned-phrase list, and the em dash ban. Connects user outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', reinforced in the example ('Users hit a white screen'). The bounded closer section adds practical constraints on verbosity with its own good/bad example. Minor irony that the directive itself uses an em dash in the exemption clause, but that doesn't weaken the instruction's content."},"latency_ms":7678,"stop_reason":"end_turn","usage":{"input_tokens":1190,"output_tokens":449},"cost_usd":0.03435} +{"config":"voice","item":"p08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"directness":5,"concreteness":5,"avoids_corporate":4,"avoids_ai_vocabulary":5,"connects_user_outcomes":5,"reasoning":"Directness: 'Lead with the point', 'Be direct about quality', 'End with what to do', plus a Good/Bad example contrasting hedged phrasing with a flat statement. Concreteness: explicitly lists files, functions, line numbers, commands, outputs, evals, real numbers, and the example uses auth.ts:47 and 'Two lines'. Avoids corporate: bans corporate, academic, PR, hype, filler, throat-clearing, and gives the 'builder talking to a builder, not a consultant presenting to a client' alternative; the Bad example shows the tone to avoid, but the positive alternative is more a framing than a fully articulated register, so 4. AI vocabulary: explicit word list (delve, crucial, robust, etc.), banned phrase list, and no em dashes, 5. User outcomes: 'Tie technical choices to user outcomes: what the real user sees, loses, waits for, or can now do', and the example ties the undefined return to 'Users hit a white screen'. One minor inconsistency: the bounded closer paragraph itself uses an em dash despite banning them."},"latency_ms":7715,"stop_reason":"end_turn","usage":{"input_tokens":1703,"output_tokens":429},"cost_usd":0.03848} +{"config":"voice","item":"f06","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"directness":4,"concreteness":3,"avoids_corporate":4,"avoids_ai_vocabulary":3,"connects_user_outcomes":1,"reasoning":"Directness: 'direct, sharp, no filler, no throat-clearing, end with what to do' is strong, but it never explicitly says to lead with the point or take a position. Concreteness: 'Name the file, the function, the command' is good but omits line numbers and real numbers. Avoids corporate: explicitly bans corporate and academic tone and offers the builder-vs-consultant framing as an alternative, though no concrete phrase swaps. Avoids AI vocabulary: bans em dashes and gives a short word list (delve, crucial, robust, comprehensive, nuanced) trailing off in 'etc.', no banned phrases, and the directive itself uses an em dash in the last paragraph. Connects user outcomes: nothing here ties technical work to what users experience; the 'user has context' line is about deference, not outcomes."},"latency_ms":10444,"stop_reason":"end_turn","usage":{"input_tokens":500,"output_tokens":774},"cost_usd":0.0437} +{"config":"voice","item":"f06","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"directness":4,"concreteness":3,"avoids_corporate":3,"avoids_ai_vocabulary":3,"connects_user_outcomes":1,"reasoning":"Directness is strong: 'direct, sharp, no filler, no throat-clearing, end with what to do' pushes the agent to get to the point, though it never says 'lead with the conclusion' or 'take a position,' and the closing line about cross-model agreement being only a recommendation slightly hedges. Concreteness is partial: it tells the agent to name the file, function, and command, but says nothing about line numbers or citing real numbers/measurements. Avoids corporate is present ('never corporate, never academic, builder not consultant') but offers no banned-phrase examples and only a thin alternative ('sound like a builder'). AI vocabulary ban exists with a short list (delve, crucial, robust, comprehensive, nuanced) trailing off into 'etc.', plus a no-em-dash rule, yet the directive itself uses an em dash in its final paragraph, undercutting it. There is no instruction at all to connect technical work to user-facing outcomes or experience; the 'user has context you don't' line is about deference, not outcomes."},"latency_ms":10663,"stop_reason":"end_turn","usage":{"input_tokens":500,"output_tokens":815},"cost_usd":0.04575} diff --git a/test/fixtures/judge-calibration/voice/runs/comparison-1.stop.json b/test/fixtures/judge-calibration/voice/runs/comparison-1.stop.json new file mode 100644 index 000000000..6339c6830 --- /dev/null +++ b/test/fixtures/judge-calibration/voice/runs/comparison-1.stop.json @@ -0,0 +1 @@ +{"stopped":null} diff --git a/test/skill-llm-eval.test.ts b/test/skill-llm-eval.test.ts index 8a853b73c..ce3589be4 100644 --- a/test/skill-llm-eval.test.ts +++ b/test/skill-llm-eval.test.ts @@ -14,7 +14,7 @@ import { afterAll, expect } from 'bun:test'; import { JUDGE_MS } from './helpers/eval-budgets'; import * as fs from 'fs'; import * as path from 'path'; -import { callJudge, judge, JudgeRefusalError, JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge'; +import { callJudge, judge, JudgeRefusalError, JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, VOICE_DIRECTIVE_JUDGE_SCHEMA, buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt, DEFAULT_JUDGE_MAX_TOKENS, judgePanel, judgePanelMean, judgePanelMajority, judgePanelReasoning, JUDGE_SCORE_DIMENSIONS } from './helpers/llm-judge'; import { ASK_QUESTIONS_HEADING, ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt'; import type { JudgeScore } from './helpers/llm-judge'; import { readWorkflowJudgeInput, buildWorkflowJudgePrompt, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA, type WorkflowJudgeInput } from './helpers/workflow-judge-input'; @@ -732,7 +732,7 @@ describeIfSelected('Voice directive eval', ['voice directive tone'], () => { avoids_ai_vocabulary: number; connects_user_outcomes: number; reasoning: string; - }>(buildVoiceDirectiveJudgePrompt(voiceSection))); + }>(buildVoiceDirectiveJudgePrompt(voiceSection), undefined, { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA })); const result = judgePanelMean(samples, ['directness', 'concreteness', 'avoids_corporate', 'avoids_ai_vocabulary', 'connects_user_outcomes'] as const); console.log('Voice directive panel:', JSON.stringify({ mean: result, samples }, null, 2));