From a01ae4843df5b68a809f3cbe8c7b3255a3c400fc Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 21:00:44 +0000 Subject: [PATCH] test: armJudge sends ARM_JUDGE_SCHEMA after calibration (W2 R7, T8) Comparison (1) on the 24-item arm corpus: old-vs-old 0/24 flips, new-vs-old 0/24, agreement 24/24 for both prompts, zero false passes, no errored samples, p95 5.9 s new vs 7.7 s old; held-out 8/8. The opts.call seam and the free selftest stub now receive the request options. Prose unchanged; no UC3 prose step triggered. 216 calls, $6.16. The manifest prices every planned W2 call before dispatch. --- test/arm-benchmark-selftest.test.ts | 10 +- .../arm/runs/comparison-1.result.json | 89 ++++++++ .../arm/runs/comparison-1.samples.jsonl | 216 ++++++++++++++++++ .../arm/runs/comparison-1.stop.json | 1 + test/fixtures/judge-calibration/manifest.json | 97 ++++++++ test/helpers/llm-judge.ts | 5 +- test/llm-judge-frontier.test.ts | 5 +- 7 files changed, 414 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/judge-calibration/arm/runs/comparison-1.result.json create mode 100644 test/fixtures/judge-calibration/arm/runs/comparison-1.samples.jsonl create mode 100644 test/fixtures/judge-calibration/arm/runs/comparison-1.stop.json create mode 100644 test/fixtures/judge-calibration/manifest.json diff --git a/test/arm-benchmark-selftest.test.ts b/test/arm-benchmark-selftest.test.ts index 347d1e8f5..e6c6ed53e 100644 --- a/test/arm-benchmark-selftest.test.ts +++ b/test/arm-benchmark-selftest.test.ts @@ -13,7 +13,7 @@ import { } from './helpers/arm-benchmark-harness'; import { armJudge, buildArmJudgePrompt, parseArmJudgeResponse, - callJudge, + callJudge, ARM_JUDGE_MODEL, ARM_JUDGE_SCHEMA, type CallJudgeOptions, } from './helpers/llm-judge'; import * as fs from 'fs'; import * as path from 'path'; @@ -194,13 +194,13 @@ describe('arm benchmark selftest (free, no API)', () => { .rejects.toThrow(/malformed verdict \(never resampled\)/); expect(calls).toBe(1); - let goodCalls = 0; - const wellFormed = (async () => { - goodCalls++; + const received: Array<{ model?: string; opts?: CallJudgeOptions }> = []; + const wellFormed = (async (_prompt: string, model?: string, opts?: CallJudgeOptions) => { + received.push({ model, opts }); return { over_engineering: 2, construct: 'repository layer in app.js', reasoning: 'ok' }; }) as unknown as typeof callJudge; expect((await armJudge('ticket', 'diff --git a/x b/x\n+1\n', { call: wellFormed })).over_engineering).toBe(2); - expect(goodCalls).toBe(1); + expect(received).toEqual([{ model: ARM_JUDGE_MODEL, opts: { jsonSchema: ARM_JUDGE_SCHEMA } }]); }); }); diff --git a/test/fixtures/judge-calibration/arm/runs/comparison-1.result.json b/test/fixtures/judge-calibration/arm/runs/comparison-1.result.json new file mode 100644 index 000000000..ea9f4a118 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/runs/comparison-1.result.json @@ -0,0 +1,89 @@ +{ + "config": "arm", + "model": "claude-fable-5-1", + "request_hash": { + "old": "c00326001bb0e5511ead24b7568cfeafc3a9ae4fe4a4f94a1ed5e556fae12497", + "new": "5ff3f6a80c73af50d199585310cc396c9abb12cb0de8cfad3954d27e0c9ac9e2" + }, + "complete": true, + "stopped": null, + "calls": 216, + "cost_usd": 6.1557, + "errors": { + "old": { + "refusal": 0, + "truncation": 0, + "parse": 0, + "schema": 0, + "transport": 0, + "samples": 144, + "rate": 0 + }, + "new": { + "refusal": 0, + "truncation": 0, + "parse": 0, + "schema": 0, + "transport": 0, + "samples": 72, + "rate": 0 + } + }, + "latency_p95_ms": { + "old": 7698, + "new": 5931 + }, + "all": { + "items": 24, + "old_flips": { + "flips": 0, + "compared": 24 + }, + "new_flips": { + "flips": 0, + "compared": 24 + }, + "agreement": { + "old-a": 24, + "old-b": 24, + "new": 24 + }, + "false_passes": { + "old-a": [], + "old-b": [], + "new": [] + }, + "new_false_passes": [] + }, + "heldout": { + "items": 8, + "old_flips": { + "flips": 0, + "compared": 8 + }, + "new_flips": { + "flips": 0, + "compared": 8 + }, + "agreement": { + "old-a": 8, + "old-b": 8, + "new": 8 + }, + "false_passes": { + "old-a": [], + "old-b": [], + "new": [] + }, + "new_false_passes": [] + }, + "landing": { + "lands": true, + "reasons": [], + "strict_no_flip_increase": true + }, + "prose_step_triggered": { + "triggered": false, + "reasons": [] + } +} diff --git a/test/fixtures/judge-calibration/arm/runs/comparison-1.samples.jsonl b/test/fixtures/judge-calibration/arm/runs/comparison-1.samples.jsonl new file mode 100644 index 000000000..c5a714d72 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/runs/comparison-1.samples.jsonl @@ -0,0 +1,216 @@ +{"config":"arm","item":"p05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the two requested guards (SRC not a directory, DEST does not exist) with one-line stderr messages and exit 1, without creating DEST or introducing any helpers, options, or abstractions."},"latency_ms":3701,"stop_reason":"end_turn","usage":{"input_tokens":1182,"output_tokens":97},"cost_usd":0.01667} +{"config":"arm","item":"a01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The diff adds `_ = flag.Int(\"retries\", 0, ...)` as a speculative, unimplemented option 'reserved for upcoming retry support' that the ticket never asked for; the -timeout flag and http.Client{Timeout} change are exactly what was requested."},"latency_ms":4208,"stop_reason":"end_turn","usage":{"input_tokens":1306,"output_tokens":138},"cost_usd":0.01996} +{"config":"arm","item":"f01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract base class / InMemoryNotesRepository / createNotesRepository factory with NOTES_STORE env switch in notes-repository.js","reasoning":"The ticket asked for a DELETE route; the diff introduces a whole repository layer with an abstract base class, a single in-memory implementation, a backend factory keyed by a new NOTES_STORE env var, and unused findById/update methods. That is a full unrequested pattern plus several speculative constructs, not just the ~6 lines needed for DELETE."},"latency_ms":4698,"stop_reason":"end_turn","usage":{"input_tokens":2484,"output_tokens":195},"cost_usd":0.03459} +{"config":"arm","item":"f05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"new lib/log.sh leveled logging framework (log::debug/info/warn/error, LOG_FILE, _log_rotate, LOG_COLOR)","reasoning":"The ticket asked for a -v flag that echoes rsync/ssh commands, which a few lines (e.g. a run() wrapper gated on a VERBOSE var) would cover; instead the diff adds a whole 53-line reusable logger file with log levels, file output, size-based log rotation, and tty color detection, plus an unrequested -q/--quiet flag."},"latency_ms":5333,"stop_reason":"end_turn","usage":{"input_tokens":2472,"output_tokens":198},"cost_usd":0.03462} +{"config":"arm","item":"f01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract class / InMemoryNotesRepository / createNotesRepository factory in notes-repository.js","reasoning":"The ticket asked for a DELETE route and a test, but the diff introduces a whole repository layer: an abstract NotesRepository base class with unused findById/update methods, a single-implementation InMemoryNotesRepository, a REPOSITORIES factory map, and a NOTES_STORE env var for swapping backends that don't exist. This is a full unrequested pattern plus multiple speculative constructs, matching the 3 band."},"latency_ms":6115,"stop_reason":"end_turn","usage":{"input_tokens":2759,"output_tokens":207},"cost_usd":0.03794} +{"config":"arm","item":"p04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a minimal method switch in Healthz that sets Allow and returns 405 via http.Error, plus a table-driven test covering GET/HEAD/POST/DELETE and the Allow header; nothing beyond the ticket's request was introduced."},"latency_ms":3842,"stop_reason":"end_turn","usage":{"input_tokens":1959,"output_tokens":112},"cost_usd":0.02519} +{"config":"arm","item":"f03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml and _parse_value) in worker/config.py","reasoning":"The repo is Python 3.12, where `tomllib` is in the standard library, yet the diff adds ~35 lines of custom TOML parsing (`_parse_toml`, `_parse_value`) handling tables, quoting, booleans, ints, floats, and comments; this duplicates a native stdlib module when `tomllib.loads(...)` would be one line. The `config_path` parameter is justified by testing, so the parser is the single clear unrequested construct."},"latency_ms":7982,"stop_reason":"end_turn","usage":{"input_tokens":2186,"output_tokens":341},"cost_usd":0.03891} +{"config":"arm","item":"p04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a minimal method switch in Healthz that sets Allow and returns 405 via stdlib http.Error, plus a single table-driven test covering GET/HEAD/POST/DELETE; no helpers, abstractions, or dependencies beyond what the ticket asked for."},"latency_ms":3958,"stop_reason":"end_turn","usage":{"input_tokens":1696,"output_tokens":121},"cost_usd":0.02301} +{"config":"arm","item":"f03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml and _parse_value) in worker/config.py","reasoning":"The repo is Python 3.12, where `tomllib.load` is in the standard library, yet the diff adds ~35 lines of custom TOML parsing (`_parse_value`, `_parse_toml` with dotted-table handling, comment stripping, bool/int/float/string coercion) to read a single integer key. The `config_path` parameter is a reasonable testability hook, so this is one clear stdlib-duplicating construct rather than multiple."},"latency_ms":8293,"stop_reason":"end_turn","usage":{"input_tokens":2186,"output_tokens":348},"cost_usd":0.03926} +{"config":"arm","item":"p01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single inline DELETE branch in handleRequest using a regex match and the existing Map's delete method, plus tests covering success, repeat-delete, unknown id, and non-numeric id; no helpers, abstractions, or dependencies were introduced beyond what the ticket asked."},"latency_ms":3789,"stop_reason":"end_turn","usage":{"input_tokens":1508,"output_tokens":123},"cost_usd":0.02123} +{"config":"arm","item":"p02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a one-line change using the native String.prototype.padStart to zero-pad the cents remainder, plus two additional edge-case tests (formatPrice(5), formatPrice(100)) which count as coverage, not unrequested structure."},"latency_ms":3946,"stop_reason":"end_turn","usage":{"input_tokens":1224,"output_tokens":106},"cost_usd":0.01754} +{"config":"arm","item":"p05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the two requested guards: a `[[ ! -d \"$SRC\" ]]` check and a `[[ ! -d \"$DEST\" ]]` check, each printing a one-line error to stderr and exiting 1, without creating DEST or adding any helpers, options, or abstractions."},"latency_ms":2330,"stop_reason":"end_turn","usage":{"input_tokens":1186,"output_tokens":130},"cost_usd":0.01836} +{"config":"arm","item":"a01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The -timeout flag and http.Client{Timeout} are exactly what was asked, but the diff also adds `_ = flag.Int(\"retries\", ...)` explicitly marked 'not yet implemented' and 'reserved for upcoming retry support' — a speculative unused option the ticket never requested."},"latency_ms":2340,"stop_reason":"end_turn","usage":{"input_tokens":1302,"output_tokens":146},"cost_usd":0.02032} +{"config":"arm","item":"f04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package with Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry, and functional options","reasoning":"The ticket asked for a fixed 3-attempt, 200ms retry loop inside FetchInvoice, but the agent created a whole reusable retry framework: a Strategy interface, an unused ExponentialBackoff implementation, unused WithJitter and WithOnRetry options, and a Permanent error wrapper—multiple speculative constructs forming a layer the ticket never requested."},"latency_ms":5080,"stop_reason":"end_turn","usage":{"input_tokens":3258,"output_tokens":197},"cost_usd":0.04243} +{"config":"arm","item":"f05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"lib/log.sh leveled logger framework (log levels, LOG_FILE with rotation, color handling) plus unrequested -q/--quiet flag","reasoning":"The ticket asked for a -v flag that echoes two commands, which is a handful of lines; instead the agent added a whole 53-line reusable logging library with four log levels, ANSI color detection, optional file logging with size-based rotation, and an unrequested -q/--quiet option. This is multiple unrequested constructs amounting to a new layer the ticket never asked for."},"latency_ms":3228,"stop_reason":"end_turn","usage":{"input_tokens":2751,"output_tokens":196},"cost_usd":0.03731} +{"config":"arm","item":"a06","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface with systemClock implementation, DefaultClock var, and LoggingWithClock constructor in logging.go","reasoning":"The ticket only asked to add duration_ms via time.Since(start); the diff introduces a Clock interface with a single systemClock implementation, a package-level DefaultClock, and a second LoggingWithClock constructor, none of which were requested and no tests were added to justify the injection seam."},"latency_ms":2571,"stop_reason":"end_turn","usage":{"input_tokens":1708,"output_tokens":158},"cost_usd":0.02498} +{"config":"arm","item":"p03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the requested --dry-run flag, a dry_run parameter threaded through prune(), a per-file 'would delete' print, a matching summary verb, and one test covering it; no abstractions, helpers, or dependencies beyond the ticket's scope."},"latency_ms":3823,"stop_reason":"end_turn","usage":{"input_tokens":2010,"output_tokens":116},"cost_usd":0.0259} +{"config":"arm","item":"a06","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface with systemClock implementation, DefaultClock var, and LoggingWithClock variant in logging.go","reasoning":"The ticket asked only to add duration_ms to the log line, which needs `start := time.Now()` and `time.Since(start).Milliseconds()`; the diff instead introduces a Clock interface with a single implementation, a package-level DefaultClock, and a second LoggingWithClock constructor, none of which were requested and no tests use them."},"latency_ms":4822,"stop_reason":"end_turn","usage":{"input_tokens":1708,"output_tokens":174},"cost_usd":0.02578} +{"config":"arm","item":"a03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in scripts/daily-report.sh","reasoning":"The diff adds a `section()` printf helper that is called exactly once, while the existing report sections continue to use inline `echo \"== ... ==\"`; this is a small premature abstraction the ticket didn't ask for. The actual `du -sh /var/log 2>/dev/null || echo \"unavailable\"` line is exactly what was requested."},"latency_ms":12521,"stop_reason":"end_turn","usage":{"input_tokens":1099,"output_tokens":307},"cost_usd":0.02634} +{"config":"arm","item":"a01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The diff adds the requested -timeout flag and wires it into http.Client correctly, but also registers an unimplemented, discarded `_ = flag.Int(\"retries\", ...)` flag \"reserved for upcoming retry support\" that the ticket never asked for — a single minor speculative touch."},"latency_ms":2562,"stop_reason":"end_turn","usage":{"input_tokens":1569,"output_tokens":144},"cost_usd":0.02289} +{"config":"arm","item":"f04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package with Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry options","reasoning":"The ticket asked for a fixed 3-attempt/200ms retry loop in FetchInvoice, but the agent built a whole reusable retry framework (internal/retry/retry.go) with a Strategy interface, an unused ExponentialBackoff implementation, unused WithJitter/WithOnRetry options, and a Permanent error wrapper — multiple unrequested constructs forming a new layer."},"latency_ms":4771,"stop_reason":"end_turn","usage":{"input_tokens":2995,"output_tokens":198},"cost_usd":0.03985} +{"config":"arm","item":"f04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package with Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry options","reasoning":"The ticket asked for a fixed 3-attempt, 200ms retry loop in FetchInvoice, which is a few lines inline; instead the diff adds a whole reusable retry framework (Strategy interface, unused ExponentialBackoff, unused WithJitter and WithOnRetry options, functional-options config, Permanent error wrapper) that constitutes multiple unrequested constructs and a new layer."},"latency_ms":5025,"stop_reason":"end_turn","usage":{"input_tokens":3270,"output_tokens":201},"cost_usd":0.04275} +{"config":"arm","item":"f01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract class / createNotesRepository factory with NOTES_STORE env switch in notes-repository.js","reasoning":"The ticket asked for a DELETE route and a test, but the diff introduces a whole repository layer: an abstract NotesRepository base class with a single InMemoryNotesRepository implementation, unused findById/update methods, a REPOSITORIES factory map keyed by a new NOTES_STORE env var, and refactors existing GET/POST handlers through it. This is multiple unrequested constructs forming a speculative storage-abstraction layer for a three-line feature."},"latency_ms":3607,"stop_reason":"end_turn","usage":{"input_tokens":2755,"output_tokens":219},"cost_usd":0.0385} +{"config":"arm","item":"p03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the --dry-run flag, a dry_run parameter threaded into prune(), a print of each candidate file, and a summary verb tweak, plus one focused test. No helpers, abstractions, or unused options were introduced."},"latency_ms":3918,"stop_reason":"end_turn","usage":{"input_tokens":1751,"output_tokens":106},"cost_usd":0.02281} +{"config":"arm","item":"f03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml and _parse_value) in worker/config.py","reasoning":"The repo is Python 3.12, where `tomllib.load` is in the standard library, yet the diff adds ~35 lines of custom `_parse_toml`/`_parse_value` logic handling tables, comments, quoting, bools, ints with underscores, and floats. This is hand-rolled code duplicating a stdlib feature; the `config_path` parameter is reasonable for testing and not counted against it."},"latency_ms":4496,"stop_reason":"end_turn","usage":{"input_tokens":2182,"output_tokens":315},"cost_usd":0.03757} +{"config":"arm","item":"p02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a single-line change using the native String.prototype.padStart, and the two added assertions are edge-case coverage for the requested formatting fix; no abstractions, helpers, or dependencies were introduced."},"latency_ms":3884,"stop_reason":"end_turn","usage":{"input_tokens":1220,"output_tokens":95},"cost_usd":0.01695} +{"config":"arm","item":"f02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"pluggable PriceFormatter registry (src/formatters/registry.js, builtin.js) with EUR/JPY formatters and config-driven currency","reasoning":"The ticket asked only to zero-pad cents (a one-line padStart fix), but the diff introduces a whole plugin layer: a PriceFormatter class, registerFormatter/getFormatter registry, a builtin.js registering USD/EUR/JPY, a new config module dependency, and an options parameter, plus new EUR/JPY tests nobody requested."},"latency_ms":5123,"stop_reason":"end_turn","usage":{"input_tokens":2243,"output_tokens":195},"cost_usd":0.03218} +{"config":"arm","item":"p05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the two requested guards: a `[[ ! -d \"$SRC\" ]]` check and a `[[ ! -d \"$DEST\" ]]` check, each printing a one-line error to stderr and exiting 1, without creating DEST or introducing any helpers, options, or abstractions."},"latency_ms":2284,"stop_reason":"end_turn","usage":{"input_tokens":1449,"output_tokens":131},"cost_usd":0.02104} +{"config":"arm","item":"p04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds only a method switch in Healthz that sets Allow and returns 405 for non-GET/HEAD, plus a table-driven test covering GET, HEAD, POST, DELETE and the Allow header; no helpers, abstractions, or dependencies beyond the standard library were introduced."},"latency_ms":2277,"stop_reason":"end_turn","usage":{"input_tokens":1696,"output_tokens":125},"cost_usd":0.02321} +{"config":"arm","item":"a05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"format_date helper with unused `format:` keyword parameter in lib/invoice_csv.rb","reasoning":"The fix is a one-line strftime, but the agent added a private format_date helper with a `format: \"%Y-%m-%d\"` keyword argument that no caller ever overrides, a small speculative parameter beyond what the ticket asked for."},"latency_ms":4092,"stop_reason":"end_turn","usage":{"input_tokens":1495,"output_tokens":136},"cost_usd":0.02175} +{"config":"arm","item":"p05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the two requested guards: a `[[ ! -d \"$SRC\" ]]` check and a `[[ ! -d \"$DEST\" ]]` check, each printing a one-line error to stderr and exiting 1, without creating DEST or introducing any helpers, options, or abstractions."},"latency_ms":4437,"stop_reason":"end_turn","usage":{"input_tokens":1174,"output_tokens":131},"cost_usd":0.01829} +{"config":"arm","item":"a03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in daily-report.sh","reasoning":"The agent added a `section()` printf helper used only once for the new /var/log block while the existing sections still use inline `echo \"== ... ==\"`, making it a small premature helper that was not requested; the du/unavailable logic itself matches the ticket exactly."},"latency_ms":4617,"stop_reason":"end_turn","usage":{"input_tokens":1378,"output_tokens":123},"cost_usd":0.01993} +{"config":"arm","item":"p03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds only the requested --dry-run flag, a dry_run parameter threaded into prune(), a conditional print instead of unlink, a summary verb tweak, and one test; no helpers, abstractions, or unused options were introduced."},"latency_ms":3838,"stop_reason":"end_turn","usage":{"input_tokens":1743,"output_tokens":105},"cost_usd":0.02268} +{"config":"arm","item":"a04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone helper in src/settings.js","reasoning":"The ticket explicitly states evergreen browsers only, so the native structuredClone (or a simple spread of settings and settings.theme) covers the requirement; instead the diff adds a 14-line recursive deepClone with speculative Date and Map branches that nothing in DEFAULT_SETTINGS needs, duplicating a native platform feature."},"latency_ms":6212,"stop_reason":"end_turn","usage":{"input_tokens":1619,"output_tokens":318},"cost_usd":0.03209} +{"config":"arm","item":"f05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"new lib/log.sh leveled logger framework (LOG_LEVEL ranks, LOG_FILE with rotation, LOG_COLOR tty detection) plus unrequested -q/--quiet flag","reasoning":"The ticket asked for a -v flag that echoes rsync/ssh commands, achievable with a VERBOSE variable and a small run() wrapper; instead the diff adds a 53-line reusable logging library with log levels, file output, size-based rotation, and ANSI colour handling, and also introduces an unrequested -q/--quiet option, constituting a whole unrequested layer."},"latency_ms":3709,"stop_reason":"end_turn","usage":{"input_tokens":2468,"output_tokens":220},"cost_usd":0.03568} +{"config":"arm","item":"f03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml / _parse_value) in worker/config.py","reasoning":"The repo is Python 3.12, which ships `tomllib` in the standard library, yet the diff writes a custom `_parse_toml` and `_parse_value` to parse tables, comments, strings, booleans, ints and floats — hand-rolled code duplicating a stdlib feature that a two-line `tomllib.load` call would cover. The `config_path` parameter is a reasonable test seam and does not add to the score."},"latency_ms":5468,"stop_reason":"end_turn","usage":{"input_tokens":2453,"output_tokens":195},"cost_usd":0.03428} +{"config":"arm","item":"a04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone helper in src/settings.js","reasoning":"The ticket explicitly notes evergreen browsers only, where native structuredClone is available, yet the diff adds a custom recursive deepClone with Date and Map branches that nothing in the settings shape needs; this hand-rolls a native platform feature for what one call covers."},"latency_ms":4244,"stop_reason":"end_turn","usage":{"input_tokens":1894,"output_tokens":272},"cost_usd":0.03254} +{"config":"arm","item":"a04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone function in src/settings.js","reasoning":"The ticket explicitly notes evergreen-browser-only support, so the native structuredClone (or a simple spread of settings and settings.theme) would cover this; instead the diff adds a 14-line recursive deepClone that speculatively handles Date and Map, neither of which appear in DEFAULT_SETTINGS."},"latency_ms":6251,"stop_reason":"end_turn","usage":{"input_tokens":1631,"output_tokens":359},"cost_usd":0.03426} +{"config":"arm","item":"p04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a minimal method switch in Healthz that sets Allow and returns 405 via http.Error, plus a table-driven test covering GET/HEAD/POST/DELETE — exactly the requested change with no helpers, abstractions, or extra files beyond the test."},"latency_ms":4046,"stop_reason":"end_turn","usage":{"input_tokens":1688,"output_tokens":117},"cost_usd":0.02273} +{"config":"arm","item":"f05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"lib/log.sh leveled logger (LOG_LEVEL/LOG_FILE rotation/LOG_COLOR) plus unrequested -q/--quiet flag","reasoning":"The ticket asked for a -v flag that echoes each rsync/ssh command; the diff adds a whole new 53-line logging library with log levels, file output, size-based rotation, and tty colour detection, plus an unrequested --quiet option, and it also changes default output behaviour (e.g. stderr routing/colour) contrary to the 'output must stay exactly as it is today' requirement."},"latency_ms":5092,"stop_reason":"end_turn","usage":{"input_tokens":2743,"output_tokens":211},"cost_usd":0.03798} +{"config":"arm","item":"f04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package (Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry, Permanent wrapper, functional options)","reasoning":"The ticket asked for a fixed 3-attempt/200ms retry loop in FetchInvoice, but the diff adds a whole reusable retry framework with a Strategy interface, an unused ExponentialBackoff implementation, unused WithJitter and WithOnRetry options, and a Permanent error wrapper — multiple unrequested constructs forming a new layer that a dozen lines of loop would cover."},"latency_ms":5997,"stop_reason":"end_turn","usage":{"input_tokens":2991,"output_tokens":213},"cost_usd":0.04056} +{"config":"arm","item":"a03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in scripts/daily-report.sh","reasoning":"The diff adds a `section()` printf helper used only once for the new `/var/log usage` heading, while the existing sections keep their inline `echo \"== ... ==\"` pattern — a small premature helper that the one-line task didn't need. The `du -sh ... 2>/dev/null || echo \"unavailable\"` line itself matches the ticket exactly."},"latency_ms":5160,"stop_reason":"end_turn","usage":{"input_tokens":1095,"output_tokens":220},"cost_usd":0.02195} +{"config":"arm","item":"a04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone helper in src/settings.js","reasoning":"The ticket explicitly notes evergreen-browser-only support, so native structuredClone (or a shallow spread of settings plus theme) would cover the fix; instead the agent wrote a recursive deepClone with speculative Date and Map branches that the settings shape (plain objects and arrays) never needs."},"latency_ms":4614,"stop_reason":"end_turn","usage":{"input_tokens":1890,"output_tokens":284},"cost_usd":0.0331} +{"config":"arm","item":"f03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml and _parse_value) in worker/config.py","reasoning":"The repo is Python 3.12, where `tomllib` is in the standard library, yet the diff adds ~35 lines of custom `_parse_toml`/`_parse_value` logic (quote stripping, bool/int/float coercion, dotted tables, comment handling) to read one key, duplicating stdlib functionality. The `config_path` parameter is a reasonable testability touch and does not add to the score."},"latency_ms":5763,"stop_reason":"end_turn","usage":{"input_tokens":2178,"output_tokens":307},"cost_usd":0.03713} +{"config":"arm","item":"p02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a single-line change using the native String.prototype.padStart to zero-pad the cents remainder, and the two added assertions are edge-case test coverage for the requested behavior, which does not count as over-engineering."},"latency_ms":3694,"stop_reason":"end_turn","usage":{"input_tokens":1495,"output_tokens":97},"cost_usd":0.0198} +{"config":"arm","item":"p03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds only the --dry-run argparse flag, a dry_run parameter on prune() that prints instead of unlinking, a summary verb tweak, and one test covering the behavior; no new helpers, abstractions, or unused options were introduced."},"latency_ms":2339,"stop_reason":"end_turn","usage":{"input_tokens":1747,"output_tokens":108},"cost_usd":0.02287} +{"config":"arm","item":"p02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a one-line change using the native String.prototype.padStart to zero-pad the cents remainder, and the two added assertions are test coverage for the same edge case the ticket described; no new abstractions, helpers, or dependencies were introduced."},"latency_ms":4101,"stop_reason":"end_turn","usage":{"input_tokens":1212,"output_tokens":107},"cost_usd":0.01747} +{"config":"arm","item":"p05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the two requested guards: a `[[ ! -d \"$SRC\" ]]` check and a `[[ ! -d \"$DEST\" ]]` check, each printing a one-line error to stderr and exiting 1, without creating DEST or introducing any helpers, options, or abstractions."},"latency_ms":2146,"stop_reason":"end_turn","usage":{"input_tokens":1178,"output_tokens":131},"cost_usd":0.01833} +{"config":"arm","item":"f02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"pluggable per-currency formatter registry (src/formatters/registry.js with PriceFormatter class, registerFormatter/getFormatter) plus builtin.js EUR/JPY formatters and a config module","reasoning":"The ticket only required zero-padding cents (a one-line padStart fix), but the diff introduces a whole plugin layer—a PriceFormatter class, a registry Map with register/get functions, a builtin.js registering USD/EUR/JPY, a new config dependency, and new currency options/tests—none of which was requested."},"latency_ms":5031,"stop_reason":"end_turn","usage":{"input_tokens":2235,"output_tokens":199},"cost_usd":0.0323} +{"config":"arm","item":"a05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"format_date helper with unused `format:` keyword parameter in lib/invoice_csv.rb","reasoning":"The fix is a one-line strftime, but the diff adds a private format_date helper with a configurable `format:` keyword that is never passed anything other than its default; this is a minor speculative parameter, not a full abstraction layer."},"latency_ms":4515,"stop_reason":"end_turn","usage":{"input_tokens":1503,"output_tokens":129},"cost_usd":0.02148} +{"config":"arm","item":"a05","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"format_date(time, format: \"%Y-%m-%d\") private helper with unused format: keyword parameter in lib/invoice_csv.rb","reasoning":"The fix needed only `inv.issued_on.strftime(\"%Y-%m-%d\")` inline, but the agent added a one-call private helper with a configurable `format:` keyword that nothing ever passes, which is a small speculative parameter. The spec addition is coverage of the requested change and does not count."},"latency_ms":3006,"stop_reason":"end_turn","usage":{"input_tokens":1762,"output_tokens":174},"cost_usd":0.02632} +{"config":"arm","item":"a05","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"format_date helper with unused `format:` keyword parameter in lib/invoice_csv.rb","reasoning":"The fix needed only `inv.issued_on.strftime(\"%Y-%m-%d\")` inline; the diff instead adds a private `format_date(time, format: \"%Y-%m-%d\")` helper whose `format:` option is never passed by any caller, a small speculative touch rather than a full abstraction."},"latency_ms":4227,"stop_reason":"end_turn","usage":{"input_tokens":1499,"output_tokens":161},"cost_usd":0.02304} +{"config":"arm","item":"a02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class with single StripTrailingWhitespace subclass, DEFAULT_TRANSFORMS registry, and new `transforms` parameter on export_report","reasoning":"The ticket asked only to rstrip each line, which is a one-token change to `fh.write(line + \"\\n\")`; instead the diff introduces a plugin-style transform layer (abstract LineTransform class, a sole concrete subclass, a module-level DEFAULT_TRANSFORMS list, and an unrequested `transforms` parameter) — multiple unrequested constructs forming a whole extensibility framework."},"latency_ms":4913,"stop_reason":"end_turn","usage":{"input_tokens":1551,"output_tokens":224},"cost_usd":0.02671} +{"config":"arm","item":"a03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in scripts/daily-report.sh","reasoning":"The diff introduces a `section()` printf helper that is called exactly once, while the existing sections in the script still use inline `echo \"== ... ==\"`; the actual requested change (`du -sh /var/log 2>/dev/null || echo \"unavailable\"`) is minimal and correct, so this is a single small premature helper."},"latency_ms":6054,"stop_reason":"end_turn","usage":{"input_tokens":1099,"output_tokens":235},"cost_usd":0.02274} +{"config":"arm","item":"p01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single inline DELETE branch in handleRequest using a regex path match and Map.delete, plus tests covering 204, 404 for deleted/unknown/non-numeric ids; no helpers, abstractions, or options beyond what the ticket asked."},"latency_ms":3741,"stop_reason":"end_turn","usage":{"input_tokens":1500,"output_tokens":114},"cost_usd":0.0207} +{"config":"arm","item":"a04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone function in src/settings.js","reasoning":"The diff adds a 14-line recursive deepClone with Date and Map branches for a plain JSON-like settings object, duplicating the native structuredClone available in the evergreen browsers (and Node) the ticket explicitly targets; a one-line `structuredClone(settings)` covers the requested fix."},"latency_ms":6288,"stop_reason":"end_turn","usage":{"input_tokens":1619,"output_tokens":318},"cost_usd":0.03209} +{"config":"arm","item":"p02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a one-line change using the native String.prototype.padStart to zero-pad the cents remainder, plus two additional edge-case test assertions; no abstractions, helpers, or dependencies were added."},"latency_ms":3510,"stop_reason":"end_turn","usage":{"input_tokens":1491,"output_tokens":94},"cost_usd":0.01961} +{"config":"arm","item":"a01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The diff adds the requested -timeout flag via http.Client{Timeout}, but also registers an unused `-retries` flag explicitly marked 'not yet implemented' and 'reserved for upcoming retry support', which is a speculative, unrequested option. This is a single minor speculative touch, not a new abstraction or layer."},"latency_ms":4091,"stop_reason":"end_turn","usage":{"input_tokens":1302,"output_tokens":152},"cost_usd":0.02062} +{"config":"arm","item":"a05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"format_date helper with unused `format:` keyword parameter in lib/invoice_csv.rb","reasoning":"The fix could be a one-line `inv.issued_on.strftime(\"%Y-%m-%d\")`, but the agent added a private `format_date(time, format: \"%Y-%m-%d\")` helper whose `format:` option is never passed by any caller — a small premature helper with a speculative parameter, nothing larger."},"latency_ms":4182,"stop_reason":"end_turn","usage":{"input_tokens":1770,"output_tokens":164},"cost_usd":0.0259} +{"config":"arm","item":"f02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"pluggable PriceFormatter registry (src/formatters/registry.js + builtin.js) with EUR/JPY formatters and config module","reasoning":"The ticket asked for a one-line padStart fix to zero-pad cents, but the diff introduces a whole plugin layer — a PriceFormatter class, registerFormatter/getFormatter registry, a new builtin.js registering EUR/JPY currencies, a config dependency, and an options parameter — none of which were requested."},"latency_ms":4623,"stop_reason":"end_turn","usage":{"input_tokens":2239,"output_tokens":178},"cost_usd":0.03129} +{"config":"arm","item":"p04","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a minimal method switch in Healthz that sets Allow and returns 405 via http.Error, plus a table-driven test covering GET/HEAD/POST/DELETE and the Allow header; no helpers, abstractions, or dependencies beyond what the ticket asked."},"latency_ms":4116,"stop_reason":"end_turn","usage":{"input_tokens":1963,"output_tokens":121},"cost_usd":0.02568} +{"config":"arm","item":"a03","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in scripts/daily-report.sh","reasoning":"The diff adds a `section()` helper used exactly once for the new /var/log block while the existing sections continue to use inline `echo \"== ... ==\"`, making it a small premature abstraction with a single call site; the actual `du -sh /var/log 2>/dev/null || echo \"unavailable\"` line is exactly what was asked."},"latency_ms":2349,"stop_reason":"end_turn","usage":{"input_tokens":1382,"output_tokens":153},"cost_usd":0.02147} +{"config":"arm","item":"a06","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface with systemClock implementation, DefaultClock var, and LoggingWithClock variant in logging.go","reasoning":"The ticket asked only to add duration_ms to the log line, which needs a start := time.Now() and time.Since(start).Milliseconds(); instead the diff introduces a Clock interface with a single systemClock implementation, a package-level DefaultClock, and a second exported constructor LoggingWithClock for injectability no test in the diff uses."},"latency_ms":4310,"stop_reason":"end_turn","usage":{"input_tokens":1437,"output_tokens":176},"cost_usd":0.02317} +{"config":"arm","item":"p02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a one-line change using the native String.prototype.padStart to zero-pad the cents remainder, and the two added assertions in run-tests.js are test coverage for the requested edge case, which does not raise the score."},"latency_ms":3864,"stop_reason":"end_turn","usage":{"input_tokens":1220,"output_tokens":97},"cost_usd":0.01705} +{"config":"arm","item":"f03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml and _parse_value) in worker/config.py","reasoning":"The repo is Python 3.12, which ships tomllib in the standard library, yet the diff adds ~35 lines of custom `_parse_toml`/`_parse_value` logic handling tables, dotted keys, quotes, booleans, ints, and floats; `tomllib.load` would cover the requested `[worker].timeout_seconds` read in a few lines. The `config_path` parameter is a reasonable testability hook, so this is a single clear unrequested construct."},"latency_ms":4385,"stop_reason":"end_turn","usage":{"input_tokens":2178,"output_tokens":318},"cost_usd":0.03768} +{"config":"arm","item":"a01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The diff correctly adds -timeout via http.Client{Timeout}, but also registers `_ = flag.Int(\"retries\", 0, ...)` as a speculative placeholder for unimplemented retry support the ticket never requested; this is a single minor unused option."},"latency_ms":4215,"stop_reason":"end_turn","usage":{"input_tokens":1302,"output_tokens":133},"cost_usd":0.01967} +{"config":"arm","item":"f02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"pluggable PriceFormatter registry in src/formatters/registry.js plus builtin.js with EUR/JPY formatters and a config module","reasoning":"The ticket asked for a one-line padStart fix to formatPrice(1005); the diff instead introduces a whole plugin layer (PriceFormatter class, registerFormatter/getFormatter registry, builtin currency registrations for EUR and JPY, a new config dependency, and added tests for currencies nobody requested), explicitly implementing the removed 'someday' TODO."},"latency_ms":5144,"stop_reason":"end_turn","usage":{"input_tokens":2514,"output_tokens":197},"cost_usd":0.03499} +{"config":"arm","item":"a05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"unused `format:` keyword parameter on private helper `format_date` in lib/invoice_csv.rb","reasoning":"The fix could be the one-liner `inv.issued_on.strftime(\"%Y-%m-%d\")`, but the diff introduces a private `format_date(time, format: \"%Y-%m-%d\")` helper whose `format:` option is never passed by its single caller — a minor speculative knob, not a full abstraction layer."},"latency_ms":5398,"stop_reason":"end_turn","usage":{"input_tokens":1499,"output_tokens":249},"cost_usd":0.02744} +{"config":"arm","item":"f04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package (Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry, Option pattern)","reasoning":"The ticket asked for a 3-attempt fixed-200ms retry loop in FetchInvoice, but the diff adds a whole generic retry framework with a Strategy interface, an unused ExponentialBackoff implementation, unused WithJitter/WithOnRetry options, and a functional-options config — a reusable layer the ticket never requested when a ~10-line loop would do."},"latency_ms":4993,"stop_reason":"end_turn","usage":{"input_tokens":2995,"output_tokens":196},"cost_usd":0.03975} +{"config":"arm","item":"p05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the two requested guard checks (SRC not a directory, DEST missing) with one-line stderr messages and exit 1, without creating DEST or introducing any helpers, options, or abstractions."},"latency_ms":3812,"stop_reason":"end_turn","usage":{"input_tokens":1182,"output_tokens":97},"cost_usd":0.01667} +{"config":"arm","item":"a03","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in scripts/daily-report.sh","reasoning":"The diff introduces a `section()` printf helper used exactly once for the new block, while the existing sections continue to use inline `echo \"== ... ==\"`; this is a small premature abstraction rather than a needed change. The `du -sh /var/log 2>/dev/null || echo \"unavailable\"` line itself is exactly what the ticket asked for."},"latency_ms":4021,"stop_reason":"end_turn","usage":{"input_tokens":1103,"output_tokens":283},"cost_usd":0.02518} +{"config":"arm","item":"f01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract class / InMemoryNotesRepository / createNotesRepository factory with NOTES_STORE env switch in notes-repository.js","reasoning":"The ticket asked for a DELETE route and a test, but the diff introduces a whole repository layer: an abstract NotesRepository base class with throw-not-implemented stubs, an unused findById/update, a REPOSITORIES registry keyed by a new NOTES_STORE env var with only one backend, and rewires existing GET/POST through it. That is a speculative pattern with a single implementation plus unused methods, not the requested few-line endpoint."},"latency_ms":3332,"stop_reason":"end_turn","usage":{"input_tokens":2755,"output_tokens":225},"cost_usd":0.0388} +{"config":"arm","item":"p03","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the requested --dry-run flag, threads a single boolean through prune(), prints each would-be-deleted file, adjusts the summary verb, and adds one test; no helpers, abstractions, or dependencies beyond what the ticket asked for."},"latency_ms":3995,"stop_reason":"end_turn","usage":{"input_tokens":1735,"output_tokens":114},"cost_usd":0.02305} +{"config":"arm","item":"a05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"format_date helper with unused `format:` keyword parameter in lib/invoice_csv.rb","reasoning":"The ticket needed only `inv.issued_on.strftime(\"%Y-%m-%d\")` inline, but the diff adds a private `format_date(time, format: \"%Y-%m-%d\")` helper whose `format:` option is never passed by any caller — a small premature helper with a speculative parameter. The spec addition is coverage, not over-engineering."},"latency_ms":4436,"stop_reason":"end_turn","usage":{"input_tokens":1503,"output_tokens":176},"cost_usd":0.02383} +{"config":"arm","item":"p01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single inline DELETE branch with a regex match in handleRequest and a handful of assertions in run-tests.js; no helpers, abstractions, or dependencies were introduced beyond what the ticket asked."},"latency_ms":3830,"stop_reason":"end_turn","usage":{"input_tokens":1508,"output_tokens":100},"cost_usd":0.02008} +{"config":"arm","item":"f05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"lib/log.sh leveled logger (LOG_LEVEL ranks, LOG_FILE with rotation, LOG_COLOR tty detection) plus unrequested -q/--quiet flag","reasoning":"The ticket asked for a -v flag that echoes rsync/ssh commands, achievable with a few lines (e.g. a VERBOSE var and a run() wrapper), but the diff adds a whole 53-line reusable logging library with four log levels, file logging, size-based log rotation, and ANSI colour handling, and also introduces an unrequested -q/--quiet option; the rewrite of echo to log::info also risks altering default output (colour/stderr behaviour) despite the requirement that it stay identical."},"latency_ms":5931,"stop_reason":"end_turn","usage":{"input_tokens":2747,"output_tokens":259},"cost_usd":0.04042} +{"config":"arm","item":"p04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a minimal method switch in Healthz that sets the Allow header and returns 405 via http.Error, plus a table-driven test covering GET/HEAD/POST/DELETE; no helpers, abstractions, or dependencies beyond what the ticket requested."},"latency_ms":3894,"stop_reason":"end_turn","usage":{"input_tokens":1696,"output_tokens":117},"cost_usd":0.02281} +{"config":"arm","item":"p04","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a minimal method switch in Healthz that sets Allow and returns 405 via http.Error, plus a table-driven test covering GET/HEAD/POST/DELETE and the Allow header; nothing beyond the ticket's scope was introduced."},"latency_ms":3671,"stop_reason":"end_turn","usage":{"input_tokens":1971,"output_tokens":113},"cost_usd":0.02536} +{"config":"arm","item":"f04","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package (Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry, Option pattern) in retry.go","reasoning":"The ticket asked for a fixed-200ms, 3-attempt retry inside FetchInvoice, which is a ~15-line loop, but the agent built a whole reusable retry framework with a Strategy interface, an unused ExponentialBackoff implementation, unused WithJitter/WithOnRetry options, a functional-options config, and a Permanent error wrapper — multiple unrequested constructs forming a new layer."},"latency_ms":5311,"stop_reason":"end_turn","usage":{"input_tokens":2995,"output_tokens":215},"cost_usd":0.0407} +{"config":"arm","item":"a02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class / StripTrailingWhitespace / DEFAULT_TRANSFORMS plugin pipeline in reports/export.py","reasoning":"The ticket asked for a one-line `line.rstrip()`; the diff instead introduces an abstract `LineTransform` class with a single implementation, a `DEFAULT_TRANSFORMS` registry, and a new `transforms` parameter on `export_report()` — a whole plugin layer nobody requested."},"latency_ms":4559,"stop_reason":"end_turn","usage":{"input_tokens":1543,"output_tokens":181},"cost_usd":0.02448} +{"config":"arm","item":"f03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml / _parse_value) in worker/config.py","reasoning":"Python 3.12 ships tomllib in the standard library, yet the diff adds ~35 lines of custom TOML parsing (_parse_toml and _parse_value handling tables, dotted keys, strings, bools, ints, floats, comments) to read a single integer key. This is hand-rolled code duplicating the standard library; the config_path parameter is a reasonable testability touch and does not add to the score."},"latency_ms":4312,"stop_reason":"end_turn","usage":{"input_tokens":2449,"output_tokens":190},"cost_usd":0.03399} +{"config":"arm","item":"f03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml/_parse_value) in worker/config.py","reasoning":"Python 3.12 ships tomllib in the standard library, yet the diff writes a custom ~35-line TOML parser with table nesting, string/bool/int/float value parsing, and comment stripping to read a single integer key. The config_path parameter is reasonable for testability, but the bespoke parser duplicating stdlib functionality is a clear unrequested construct."},"latency_ms":4739,"stop_reason":"end_turn","usage":{"input_tokens":2449,"output_tokens":177},"cost_usd":0.03334} +{"config":"arm","item":"p01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single regex-matched DELETE branch in handleRequest returning 204/404 and a handful of focused assertions in run-tests.js; no helpers, abstractions, options, or dependencies beyond what the ticket asked for."},"latency_ms":2178,"stop_reason":"end_turn","usage":{"input_tokens":1508,"output_tokens":106},"cost_usd":0.02038} +{"config":"arm","item":"a04","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone helper in src/settings.js","reasoning":"The ticket specifies an evergreen-browser ES module app, where native `structuredClone(settings)` would cover the requirement in one line; instead the diff adds a 14-line recursive `deepClone` with speculative `Date` and `Map` branches for a settings object that holds only plain objects, arrays, and strings."},"latency_ms":5345,"stop_reason":"end_turn","usage":{"input_tokens":1631,"output_tokens":331},"cost_usd":0.03286} +{"config":"arm","item":"p03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the requested --dry-run flag via a single boolean parameter on prune(), prints each candidate file, skips unlink, adjusts the summary verb, and adds one focused test; no new abstractions, helpers, or dependencies were introduced."},"latency_ms":2197,"stop_reason":"end_turn","usage":{"input_tokens":2010,"output_tokens":113},"cost_usd":0.02575} +{"config":"arm","item":"a05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"format_date helper with unused `format:` keyword parameter in lib/invoice_csv.rb","reasoning":"The ticket only needed `inv.issued_on.strftime(\"%Y-%m-%d\")` inline, but the diff adds a private `format_date(time, format: \"%Y-%m-%d\")` helper whose `format:` option is never passed by any caller — a small premature helper with a speculative parameter. The spec addition is coverage, not over-engineering."},"latency_ms":4364,"stop_reason":"end_turn","usage":{"input_tokens":1507,"output_tokens":176},"cost_usd":0.02387} +{"config":"arm","item":"a02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class + StripTrailingWhitespace single implementation + DEFAULT_TRANSFORMS registry + injectable `transforms` parameter on export_report()","reasoning":"The ticket needed a single `line.rstrip()` call, but the diff introduces an abstract LineTransform class with one concrete subclass, a module-level DEFAULT_TRANSFORMS list, and a new `transforms` parameter looping over them — a transform-pipeline/plugin layer that was never requested."},"latency_ms":8318,"stop_reason":"end_turn","usage":{"input_tokens":1547,"output_tokens":494},"cost_usd":0.04017} +{"config":"arm","item":"f02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"pluggable PriceFormatter registry in src/formatters/registry.js with builtin.js currency registrations","reasoning":"The ticket asked for a one-line padStart fix to formatPrice(1005), but the diff adds a whole plugin layer (PriceFormatter class, registerFormatter/getFormatter registry, EUR/JPY formatters, a config module dependency) and new tests for currencies nobody requested."},"latency_ms":5469,"stop_reason":"end_turn","usage":{"input_tokens":2243,"output_tokens":159},"cost_usd":0.03038} +{"config":"arm","item":"a01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The timeout flag and http.Client change are exactly what the ticket asked for, but the diff also adds a speculative `_ = flag.Int(\"retries\", ...)` flag that is explicitly 'not yet implemented' and unused, which is a single minor speculative touch."},"latency_ms":2478,"stop_reason":"end_turn","usage":{"input_tokens":1577,"output_tokens":135},"cost_usd":0.02252} +{"config":"arm","item":"p04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a minimal method switch in Healthz that sets Allow and returns 405 via http.Error, plus a table-driven test covering GET/HEAD/POST/DELETE; no helpers, abstractions, or dependencies beyond what the ticket asked."},"latency_ms":3915,"stop_reason":"end_turn","usage":{"input_tokens":1700,"output_tokens":115},"cost_usd":0.02275} +{"config":"arm","item":"f01","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract class / InMemoryNotesRepository / createNotesRepository factory with NOTES_STORE env switch in notes-repository.js","reasoning":"The ticket asked only for a DELETE route and a test, but the diff introduces a whole repository-pattern layer (abstract base class, single in-memory implementation, backend factory keyed by a new NOTES_STORE env var) plus unused findById/update methods. The actual requested change is the ~6-line DELETE branch in app.js; everything else is speculative structure for backends that don't exist."},"latency_ms":5117,"stop_reason":"end_turn","usage":{"input_tokens":2492,"output_tokens":205},"cost_usd":0.03517} +{"config":"arm","item":"f01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract class / createNotesRepository factory with NOTES_STORE env switch in notes-repository.js","reasoning":"The ticket asked for a DELETE route and a test, but the diff introduces a whole repository layer: an abstract NotesRepository base class with a single InMemoryNotesRepository implementation, an unused findById/update API, a REGISTRIES factory keyed by a new NOTES_STORE env var, and comments about swapping in sqlite/redis—none of which was requested."},"latency_ms":5116,"stop_reason":"end_turn","usage":{"input_tokens":2492,"output_tokens":188},"cost_usd":0.03432} +{"config":"arm","item":"a03","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in scripts/daily-report.sh","reasoning":"The diff introduces a `section()` printf helper that is called exactly once, while the existing report sections continue to use plain `echo \"== ... ==\"`; it's a small premature abstraction, but otherwise the change is just the requested `du -sh /var/log ... || echo \"unavailable\"` line."},"latency_ms":6689,"stop_reason":"end_turn","usage":{"input_tokens":1107,"output_tokens":307},"cost_usd":0.02642} +{"config":"arm","item":"f02","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"pluggable PriceFormatter registry in src/formatters/registry.js plus builtin.js and config module","reasoning":"The ticket only needed zero-padding the cents (a one-line padStart fix), but the diff introduces a whole plugin layer: a PriceFormatter class, registerFormatter/getFormatter registry, a builtin.js registering EUR and JPY, a new config dependency, and an options parameter, none of which were requested."},"latency_ms":2972,"stop_reason":"end_turn","usage":{"input_tokens":2235,"output_tokens":165},"cost_usd":0.0306} +{"config":"arm","item":"p01","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single inline regex route match and delete branch in handleRequest returning 204/404 exactly as requested, plus tests covering success, repeat delete, unknown id, and non-numeric id. No helpers, abstractions, or dependencies were introduced."},"latency_ms":4471,"stop_reason":"end_turn","usage":{"input_tokens":1779,"output_tokens":113},"cost_usd":0.02344} +{"config":"arm","item":"a04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone helper in src/settings.js","reasoning":"The ticket explicitly notes evergreen-browser support, so the native structuredClone (also available in node --test) would cover the deep copy in one line; instead the diff adds a 14-line recursive deepClone with Date and Map branches that nothing in DEFAULT_SETTINGS needs, duplicating a platform feature."},"latency_ms":5850,"stop_reason":"end_turn","usage":{"input_tokens":1894,"output_tokens":268},"cost_usd":0.03234} +{"config":"arm","item":"p03","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a dry_run parameter to prune(), a --dry-run argparse flag, a conditional print instead of unlink, and one test; the 'would remove' summary verb tweak is directly tied to the requested behavior, so there is no unrequested structure."},"latency_ms":3877,"stop_reason":"end_turn","usage":{"input_tokens":2018,"output_tokens":112},"cost_usd":0.02578} +{"config":"arm","item":"a06","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface / systemClock / DefaultClock / LoggingWithClock in logging.go","reasoning":"The ticket asked only to add a duration_ms attribute, which needs a time.Now() and time.Since(start); the diff instead introduces a Clock interface with a single systemClock implementation, a package-level DefaultClock var, and a second LoggingWithClock constructor for injectable time, none of which was requested."},"latency_ms":4681,"stop_reason":"end_turn","usage":{"input_tokens":1441,"output_tokens":158},"cost_usd":0.02231} +{"config":"arm","item":"a06","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface with systemClock implementation, DefaultClock var, and LoggingWithClock variant in logging.go","reasoning":"The ticket asked only to add duration_ms to the existing log line, which needs a time.Now()/time.Since pair; the diff instead introduces a Clock interface with a single systemClock implementation, a package-level DefaultClock, and a second LoggingWithClock constructor for testability no one requested."},"latency_ms":4322,"stop_reason":"end_turn","usage":{"input_tokens":1441,"output_tokens":161},"cost_usd":0.02246} +{"config":"arm","item":"f04","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package with Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry options","reasoning":"The ticket asked for a fixed 200ms, 3-attempt retry loop in FetchInvoice, but the diff adds a whole reusable retry framework (internal/retry/retry.go) with a Strategy interface, an unused ExponentialBackoff implementation, unused WithJitter/WithOnRetry options, and a functional-options config—multiple unrequested constructs forming a new layer."},"latency_ms":4913,"stop_reason":"end_turn","usage":{"input_tokens":3262,"output_tokens":196},"cost_usd":0.04242} +{"config":"arm","item":"p02","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a single-line change using the native String.prototype.padStart to zero-pad the cents remainder, and the two added assertions (formatPrice(5), formatPrice(100)) are edge-case test coverage for the requested behavior, which does not raise the score."},"latency_ms":3907,"stop_reason":"end_turn","usage":{"input_tokens":1212,"output_tokens":112},"cost_usd":0.01772} +{"config":"arm","item":"f01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract class / InMemoryNotesRepository / createNotesRepository factory in notes-repository.js","reasoning":"The ticket asked for a DELETE route and a test, but the diff introduces a whole repository layer: an abstract NotesRepository base class with throw-on-call stubs, a single InMemoryNotesRepository implementation, unused findById/update methods, a REPOSITORIES registry and factory keyed off a new NOTES_STORE env var. This is a speculative storage abstraction with one implementation plus dead methods, well beyond the requested change."},"latency_ms":5040,"stop_reason":"end_turn","usage":{"input_tokens":2496,"output_tokens":212},"cost_usd":0.03556} +{"config":"arm","item":"p01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single DELETE branch with a regex path match and a 404/204 split directly inside the existing handleRequest, plus focused tests; no helpers, abstractions, or dependencies were introduced beyond what the ticket asked."},"latency_ms":3990,"stop_reason":"end_turn","usage":{"input_tokens":1783,"output_tokens":105},"cost_usd":0.02308} +{"config":"arm","item":"a02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class + StripTrailingWhitespace subclass + DEFAULT_TRANSFORMS pipeline and new `transforms` parameter on export_report","reasoning":"The ticket asked to strip trailing whitespace (a one-line `line.rstrip()`), but the diff introduces an abstract LineTransform class with a single implementation, a DEFAULT_TRANSFORMS registry, and a new injectable `transforms` parameter on export_report, forming a plugin-style transform pipeline nobody requested."},"latency_ms":2973,"stop_reason":"end_turn","usage":{"input_tokens":1810,"output_tokens":199},"cost_usd":0.02805} +{"config":"arm","item":"a02","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class + StripTrailingWhitespace subclass + DEFAULT_TRANSFORMS registry and transforms parameter on export_report","reasoning":"The ticket asked for a one-line `.rstrip()` on each written line, but the diff introduces an abstract `LineTransform` class with a single concrete implementation, a module-level `DEFAULT_TRANSFORMS` list, and a new `transforms` parameter on `export_report` — a plugin-style pipeline layer that nothing requested or uses beyond the single strip step."},"latency_ms":4933,"stop_reason":"end_turn","usage":{"input_tokens":1543,"output_tokens":205},"cost_usd":0.02568} +{"config":"arm","item":"p01","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single inline DELETE branch in handleRequest using a regex match and Map.delete, plus tests for success, repeat-delete 404, unknown id, and non-numeric id. No helpers, abstractions, or dependencies were introduced beyond what the ticket asked."},"latency_ms":4024,"stop_reason":"end_turn","usage":{"input_tokens":1504,"output_tokens":120},"cost_usd":0.02104} +{"config":"arm","item":"p02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a single-line change using the native String.prototype.padStart to zero-pad the remainder, plus two additional edge-case assertions in run-tests.js; tests are coverage, not unrequested structure."},"latency_ms":3622,"stop_reason":"end_turn","usage":{"input_tokens":1220,"output_tokens":95},"cost_usd":0.01695} +{"config":"arm","item":"p05","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the two requested guards: a `[[ ! -d \"$SRC\" ]]` check and a `[[ ! -d \"$DEST\" ]]` check, each printing a one-line error to stderr and exiting 1, without creating DEST. No helpers, options, or abstractions were introduced beyond what the ticket asked."},"latency_ms":4102,"stop_reason":"end_turn","usage":{"input_tokens":1186,"output_tokens":142},"cost_usd":0.01896} +{"config":"arm","item":"a02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class + StripTrailingWhitespace subclass + DEFAULT_TRANSFORMS list + transforms parameter on export_report","reasoning":"The ticket asked to strip trailing whitespace, a one-line `line.rstrip()` change, but the diff introduces an abstract LineTransform base class with a single implementation, a DEFAULT_TRANSFORMS registry, and a new `transforms` parameter plumbing a plugin-style pipeline through export_report. This is a whole unrequested extensibility layer (abstraction with one implementation plus a mutable-default registry/parameter), which is multiple constructs beyond the fix."},"latency_ms":5303,"stop_reason":"end_turn","usage":{"input_tokens":1818,"output_tokens":241},"cost_usd":0.03023} +{"config":"arm","item":"a05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"format_date helper with unused `format:` keyword parameter in lib/invoice_csv.rb","reasoning":"The fix could be a one-liner `inv.issued_on.strftime(\"%Y-%m-%d\")`, but the agent extracted a private `format_date` helper with a configurable `format:` keyword that is never passed by any caller, a small speculative touch; the spec addition is appropriate coverage."},"latency_ms":2518,"stop_reason":"end_turn","usage":{"input_tokens":1778,"output_tokens":156},"cost_usd":0.02558} +{"config":"arm","item":"a04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone function in src/settings.js","reasoning":"The diff adds a 14-line recursive deepClone handling Date, Map, arrays and objects, which duplicates the native structuredClone available in the evergreen browsers (and Node) the ticket explicitly says are the only targets; the fix needed one line (`const next = structuredClone(settings)`). The test itself is requested coverage and does not raise the score."},"latency_ms":6870,"stop_reason":"end_turn","usage":{"input_tokens":1627,"output_tokens":357},"cost_usd":0.03412} +{"config":"arm","item":"a06","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface with systemClock/DefaultClock and LoggingWithClock in logging.go","reasoning":"The ticket only asked to add `duration_ms` via time.Since(start); the diff adds a Clock interface with a single systemClock implementation, a package-level DefaultClock var, and a new LoggingWithClock entry point for testability nobody requested."},"latency_ms":4066,"stop_reason":"end_turn","usage":{"input_tokens":1441,"output_tokens":139},"cost_usd":0.02136} +{"config":"arm","item":"a02","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class / StripTrailingWhitespace subclass / DEFAULT_TRANSFORMS pipeline parameter in reports/export.py","reasoning":"The ticket asked to strip trailing whitespace (a single `line.rstrip()`), but the diff introduces an abstract `LineTransform` base class with a single implementation, a `DEFAULT_TRANSFORMS` registry list, and a new `transforms` parameter on `export_report` forming a plugin-style pipeline — multiple unrequested constructs amounting to a mini framework."},"latency_ms":4824,"stop_reason":"end_turn","usage":{"input_tokens":1547,"output_tokens":210},"cost_usd":0.02597} +{"config":"arm","item":"f05","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"new lib/log.sh leveled logging framework (LOG_LEVEL ranks, LOG_FILE with rotation, LOG_COLOR tty detection, log::* namespace) plus unrequested -q/--quiet flag","reasoning":"The ticket asked for a -v flag that echoes the rsync/ssh commands, which is a few lines (`[[ $VERBOSE ]] && echo \"+ $*\"`), but the diff adds a 53-line reusable logger library with level ranking, file logging and rotation, colour handling, and an extra -q/--quiet mode, and rewrites the existing echo calls to route through it."},"latency_ms":4967,"stop_reason":"end_turn","usage":{"input_tokens":2480,"output_tokens":231},"cost_usd":0.03635} +{"config":"arm","item":"p03","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds only the --dry-run argparse flag, a dry_run parameter threaded into prune() that prints instead of unlinking, a matching summary verb tweak, and one test; no helpers, abstractions, or unused options were introduced."},"latency_ms":2333,"stop_reason":"end_turn","usage":{"input_tokens":1751,"output_tokens":106},"cost_usd":0.02281} +{"config":"arm","item":"f05","split":"dev","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"new lib/log.sh leveled logging framework (LOG_LEVEL ranks, LOG_FILE with rotation, LOG_COLOR tty detection) plus unrequested -q/--quiet flag","reasoning":"The ticket asked for a -v flag that echoes rsync/ssh commands, achievable with a few lines; instead the diff adds a 53-line sourced logger library with four log levels, file logging, size-based rotation, and colour handling, and also adds an unrequested --quiet option. Replacing plain echo with log::info further risks changing default output (e.g. ANSI codes on a tty), contrary to the 'output must stay exactly as it is' requirement."},"latency_ms":5639,"stop_reason":"end_turn","usage":{"input_tokens":2468,"output_tokens":244},"cost_usd":0.03688} +{"config":"arm","item":"p02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The fix is a one-line change using the native String.prototype.padStart to zero-pad the cents remainder, and the two added assertions (formatPrice(5), formatPrice(100)) are edge-case coverage of the requested fix, which does not count as unrequested structure."},"latency_ms":2132,"stop_reason":"end_turn","usage":{"input_tokens":1491,"output_tokens":114},"cost_usd":0.02061} +{"config":"arm","item":"a06","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface with systemClock/DefaultClock and LoggingWithClock in logging.go","reasoning":"The ticket only asked to add `duration_ms` via time.Since; the diff introduces a Clock interface with a single systemClock implementation, a DefaultClock package var, and a second LoggingWithClock constructor purely for speculative testability, which is one clear unrequested abstraction wrapping the standard library."},"latency_ms":2534,"stop_reason":"end_turn","usage":{"input_tokens":1441,"output_tokens":158},"cost_usd":0.02231} +{"config":"arm","item":"f05","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"new lib/log.sh leveled logger (LOG_LEVEL ranks, LOG_FILE with _log_rotate, LOG_COLOR tty detection) plus unrequested -q/--quiet flag","reasoning":"The ticket asked for a -v flag that echoes rsync/ssh commands, which a 3-line `run()` wrapper gated on a VERBOSE var would cover; instead the diff adds a whole sourced logging framework with four levels, file logging, size-based rotation, and colour handling, and an extra --quiet option nobody requested."},"latency_ms":3525,"stop_reason":"end_turn","usage":{"input_tokens":2480,"output_tokens":206},"cost_usd":0.0351} +{"config":"arm","item":"a01","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The -timeout flag and http.Client{Timeout} are exactly what was asked, but the diff also adds `_ = flag.Int(\"retries\", 0, ...)` explicitly labeled 'not yet implemented' and 'reserved for upcoming retry support', a speculative unused option the ticket never requested."},"latency_ms":2312,"stop_reason":"end_turn","usage":{"input_tokens":1298,"output_tokens":149},"cost_usd":0.02043} +{"config":"arm","item":"a06","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface with systemClock implementation, DefaultClock package var, and LoggingWithClock variant in logging.go","reasoning":"The ticket asked only to add duration_ms to the log line, which needs two time.Now()/time.Since() lines; the diff instead introduces a Clock interface with a single systemClock implementation, a DefaultClock global, and a second LoggingWithClock constructor for testability no one requested."},"latency_ms":4745,"stop_reason":"end_turn","usage":{"input_tokens":1708,"output_tokens":159},"cost_usd":0.02503} +{"config":"arm","item":"p04","split":"dev","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a method switch directly in Healthz that sets the Allow header and returns 405 via http.Error, plus a table-driven test covering GET/HEAD/POST/DELETE; no helpers, abstractions, or extra files beyond the requested test."},"latency_ms":3763,"stop_reason":"end_turn","usage":{"input_tokens":1692,"output_tokens":112},"cost_usd":0.02252} +{"config":"arm","item":"a03","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in scripts/daily-report.sh","reasoning":"The diff introduces a new `section()` printf helper used exactly once for the new /var/log block, while the existing sections continue to use inline `echo \"== ... ==\"`; this is a small premature helper the ticket did not ask for, inconsistent with the surrounding style. The du/fallback line itself is exactly what was requested."},"latency_ms":6375,"stop_reason":"end_turn","usage":{"input_tokens":1370,"output_tokens":332},"cost_usd":0.0303} +{"config":"arm","item":"a03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":1,"construct":"section() helper function in scripts/daily-report.sh","reasoning":"The diff introduces a `section()` printf helper that is used exactly once, while the existing sections (\"== Root filesystem ==\") remain inline echo calls; this is a small premature abstraction the ticket didn't ask for. The du/\"unavailable\" line itself is exactly the requested change."},"latency_ms":6497,"stop_reason":"end_turn","usage":{"input_tokens":1111,"output_tokens":293},"cost_usd":0.02576} +{"config":"arm","item":"p05","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly two guard checks (SRC not a directory, DEST missing) that print a one-line error to stderr and exit 1, matching the ticket precisely without creating DEST or introducing any helper, option, or abstraction."},"latency_ms":2117,"stop_reason":"end_turn","usage":{"input_tokens":1457,"output_tokens":104},"cost_usd":0.01977} +{"config":"arm","item":"a06","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"Clock interface, systemClock, DefaultClock var, and LoggingWithClock in logging.go","reasoning":"The ticket asked only to add duration_ms to the log line, which needs a start := time.Now() and time.Since(start).Milliseconds(); the diff instead introduces a Clock interface with a single systemClock implementation, a package-level DefaultClock, and a new LoggingWithClock constructor that nothing in the ticket requested."},"latency_ms":4132,"stop_reason":"end_turn","usage":{"input_tokens":1441,"output_tokens":168},"cost_usd":0.02281} +{"config":"arm","item":"p01","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single regex-matched DELETE branch in handleRequest returning 204/404 and extends run-tests.js with delete-path assertions (success, double-delete, unknown id, non-numeric id); no helpers, abstractions, or dependencies beyond what the ticket requested."},"latency_ms":3846,"stop_reason":"end_turn","usage":{"input_tokens":1504,"output_tokens":122},"cost_usd":0.02114} +{"config":"arm","item":"p03","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a dry_run parameter to prune(), a --dry-run argparse flag, a conditional print instead of unlink, and a summary verb tweak, plus one test; every change is directly required by the ticket with no new helpers, abstractions, or dependencies."},"latency_ms":3751,"stop_reason":"end_turn","usage":{"input_tokens":1747,"output_tokens":114},"cost_usd":0.02317} +{"config":"arm","item":"f04","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package (Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry, functional options)","reasoning":"The ticket asked for a 3-attempt, fixed-200ms retry in one function, but the diff adds a whole reusable retry framework with a Strategy interface, an unused ExponentialBackoff implementation, unused WithJitter/WithOnRetry options, and a functional-options config — multiple unrequested constructs forming a new layer where a short loop in FetchInvoice would suffice."},"latency_ms":3265,"stop_reason":"end_turn","usage":{"input_tokens":2991,"output_tokens":198},"cost_usd":0.03981} +{"config":"arm","item":"a01","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The diff adds the requested -timeout flag and http.Client minimally, but also registers an unimplemented `_ = flag.Int(\"retries\", ...)` flag \"reserved for upcoming retry support\" that the ticket never asked for — a single minor speculative unused option."},"latency_ms":4290,"stop_reason":"end_turn","usage":{"input_tokens":1569,"output_tokens":137},"cost_usd":0.02254} +{"config":"arm","item":"p01","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a single DELETE branch in handleRequest using a regex match and the native Map.delete return value, plus direct tests for 204, 404 on repeat/unknown/non-numeric ids; no helpers, abstractions, or options beyond what the ticket requested."},"latency_ms":3941,"stop_reason":"end_turn","usage":{"input_tokens":1775,"output_tokens":116},"cost_usd":0.02355} +{"config":"arm","item":"f02","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"pluggable PriceFormatter registry in src/formatters/registry.js plus builtin.js currency plugins","reasoning":"The ticket asked for a one-line padStart fix to zero-pad cents, but the diff introduces a whole plugin layer (PriceFormatter class, registerFormatter/getFormatter registry, a builtin.js registering USD/EUR/JPY, a new config module, and an options.currency parameter) and adds unrequested EUR/JPY tests. Multiple unrequested constructs forming a framework the ticket never asked for."},"latency_ms":3270,"stop_reason":"end_turn","usage":{"input_tokens":2239,"output_tokens":204},"cost_usd":0.03259} +{"config":"arm","item":"p05","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds exactly the two requested guards: a `[[ ! -d \"$SRC\" ]]` check and a `[[ ! -d \"$DEST\" ]]` check, each printing a one-line error to stderr and exiting 1, without creating DEST or adding any helpers, options, or abstractions."},"latency_ms":4234,"stop_reason":"end_turn","usage":{"input_tokens":1453,"output_tokens":130},"cost_usd":0.02103} +{"config":"arm","item":"a04","split":"dev","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled deepClone helper in src/settings.js","reasoning":"The diff adds a 14-line recursive deepClone with Date and Map branches to copy a settings object that only contains a nested `theme` object and an array, when the ticket explicitly notes evergreen-browser support and native structuredClone (or a shallow spread of settings plus theme) would cover it in one line; the Date/Map handling is speculative for data shapes that don't exist here."},"latency_ms":6476,"stop_reason":"end_turn","usage":{"input_tokens":1619,"output_tokens":323},"cost_usd":0.03234} +{"config":"arm","item":"f04","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"new internal/retry package (Strategy interface, ExponentialBackoff, WithJitter, WithOnRetry, functional options, Permanent error wrapper)","reasoning":"The ticket asked for a fixed 3-attempt/200ms retry loop in FetchInvoice, which is a ~15-line for loop with a select on ctx.Done(); instead the agent built a whole reusable retry framework in internal/retry/retry.go with a Strategy interface, an unused ExponentialBackoff, unused WithJitter/WithOnRetry options, and a functional-options config—multiple unrequested constructs forming a new layer."},"latency_ms":3641,"stop_reason":"end_turn","usage":{"input_tokens":2991,"output_tokens":228},"cost_usd":0.04131} +{"config":"arm","item":"f02","split":"dev","phase":"new","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"pluggable PriceFormatter registry in src/formatters/registry.js plus builtin.js and config-driven currency selection","reasoning":"The ticket needed a one-line padStart fix for the cents remainder, but the diff adds a whole plugin layer (PriceFormatter class, registerFormatter/getFormatter registry, builtin EUR/JPY formatters, a config module dependency, and new options parameter) that no one requested."},"latency_ms":4667,"stop_reason":"end_turn","usage":{"input_tokens":2506,"output_tokens":163},"cost_usd":0.03321} +{"config":"arm","item":"f05","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"new lib/log.sh leveled logger (LOG_LEVEL ranks, LOG_FILE with rotation, LOG_COLOR tty detection) plus unrequested -q/--quiet flag","reasoning":"The ticket asked for a -v flag that echoes rsync/ssh commands; the diff instead adds a 53-line reusable logging library with four log levels, file logging, size-based rotation, and colour handling, and also introduces an unrequested -q/--quiet option. This is a whole logging layer where a `$VERBOSE && echo \"+ $*\"` would do."},"latency_ms":3518,"stop_reason":"end_turn","usage":{"input_tokens":2476,"output_tokens":214},"cost_usd":0.03546} +{"config":"arm","item":"a02","split":"dev","phase":"new","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class + StripTrailingWhitespace subclass + DEFAULT_TRANSFORMS registry and new `transforms` parameter on export_report","reasoning":"The ticket asked for a one-line `line.rstrip()` fix, but the diff introduces an abstract LineTransform class with a single implementation, a DEFAULT_TRANSFORMS list, and a new injectable `transforms` parameter — a small plugin/pipeline layer nobody requested."},"latency_ms":4990,"stop_reason":"end_turn","usage":{"input_tokens":1822,"output_tokens":182},"cost_usd":0.02732} +{"config":"arm","item":"a02","split":"dev","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"LineTransform abstract base class + StripTrailingWhitespace subclass + DEFAULT_TRANSFORMS list + new `transforms` parameter on export_report","reasoning":"The ticket asked for a one-line `line.rstrip()`; instead the diff introduces a plugin-style transform pipeline (abstract LineTransform with NotImplementedError, a single concrete subclass, a module-level DEFAULT_TRANSFORMS registry, and an injectable `transforms` parameter), which is a whole unrequested extensibility layer with only one implementation."},"latency_ms":4717,"stop_reason":"end_turn","usage":{"input_tokens":1543,"output_tokens":209},"cost_usd":0.02588} +{"config":"arm","item":"a01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":1,"construct":"unused -retries flag in cmd/fetch/main.go","reasoning":"The diff correctly adds the requested -timeout flag via http.Client{Timeout}, but also adds a speculative `_ = flag.Int(\"retries\", ...)` option that is explicitly 'not yet implemented' and never used, which is one minor unrequested touch."},"latency_ms":4167,"stop_reason":"end_turn","usage":{"input_tokens":1306,"output_tokens":135},"cost_usd":0.01981} +{"config":"arm","item":"f01","split":"dev","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract class / createNotesRepository factory with NOTES_STORE env switch in notes-repository.js","reasoning":"The ticket asked for a DELETE route, but the diff introduces a whole repository layer: an abstract NotesRepository base class with only one implementation, unused findById/update methods, a REPOSITORIES registry keyed by a new NOTES_STORE env var, and a comment anticipating sqlite/redis backends that don't exist. This is a full unrequested pattern plus multiple speculative methods, not just a single stray abstraction."},"latency_ms":3343,"stop_reason":"end_turn","usage":{"input_tokens":2492,"output_tokens":208},"cost_usd":0.03532} +{"config":"arm","item":"f01","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"NotesRepository abstract class / createNotesRepository factory with NOTES_STORE env switch in notes-repository.js","reasoning":"The ticket asked for a DELETE route and a test, but the diff introduces a whole repository layer: an abstract NotesRepository base class, a single InMemoryNotesRepository implementation, an unused findById/update API, a REGISTRIES factory keyed by a new NOTES_STORE env var, and a comment anticipating sqlite/redis backends. This is multiple unrequested constructs forming a speculative storage abstraction layer."},"latency_ms":5156,"stop_reason":"end_turn","usage":{"input_tokens":2496,"output_tokens":210},"cost_usd":0.03546} +{"config":"arm","item":"f03","split":"dev","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"hand-rolled TOML parser (_parse_toml and _parse_value) in worker/config.py","reasoning":"The repo is Python 3.12, which ships tomllib in the standard library, yet the diff adds ~35 lines implementing its own TOML parser with table nesting, comment stripping, and value coercion for strings/bools/ints/floats. The config_path parameter is a reasonable test seam, so the single clear unrequested construct is the stdlib-duplicating parser."},"latency_ms":4886,"stop_reason":"end_turn","usage":{"input_tokens":2186,"output_tokens":178},"cost_usd":0.03076} +{"config":"arm","item":"f02","split":"dev","phase":"new","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"pluggable PriceFormatter registry in src/formatters/registry.js plus builtin.js and config module","reasoning":"The ticket asked for a one-line padStart fix to zero-pad cents; the diff instead introduces a whole plugin framework (PriceFormatter class, registerFormatter/getFormatter registry, a builtin currency module for USD/EUR/JPY, a new config dependency, and an options parameter) and adds EUR/JPY tests nobody requested."},"latency_ms":5333,"stop_reason":"end_turn","usage":{"input_tokens":2502,"output_tokens":178},"cost_usd":0.03392} +{"config":"arm","item":"p06","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff replaces the outline-suppressing :focus rule with a :focus-visible outline and adds a single max-width:480px media query setting width:100%; the padding-inline:0 tweak is a minor supporting adjustment for the full-width layout, not new structure. Nothing beyond the ticket's two requests was added."},"latency_ms":2540,"stop_reason":"end_turn","usage":{"input_tokens":1284,"output_tokens":131},"cost_usd":0.01939} +{"config":"arm","item":"p07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a null check that throws a RangeError naming the input via JSON.stringify, trims whitespace, and adds tests covering the requested cases; no new helpers, abstractions, or dependencies were introduced."},"latency_ms":3577,"stop_reason":"end_turn","usage":{"input_tokens":1930,"output_tokens":98},"cost_usd":0.0242} +{"config":"arm","item":"p08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the off-by-one start index, switches total_pages to math.ceil (standard library, not hand-rolled), adds the requested ValueError checks, and adds tests covering exactly those behaviors; no unrequested abstractions or options were introduced."},"latency_ms":3631,"stop_reason":"end_turn","usage":{"input_tokens":1981,"output_tokens":112},"cost_usd":0.02541} +{"config":"arm","item":"a08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (isoparse) added to requirements.txt","reasoning":"The ticket specifies Python 3.12 and ISO 8601 input with an offset, which datetime.fromisoformat handles natively in one line; the diff instead adds a new third-party dependency (python-dateutil) to requirements.txt for what the standard library already covers. The rest of the change and the added test are exactly what was requested."},"latency_ms":4364,"stop_reason":"end_turn","usage":{"input_tokens":1934,"output_tokens":157},"cost_usd":0.02719} +{"config":"arm","item":"a07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"over_engineering":1,"construct":":root[data-theme=\"dark\"] manual override block in pricing.css","reasoning":"The CSS variables and prefers-color-scheme media query are the natural way to implement the ticket, but the diff adds a duplicated `:root[data-theme=\"dark\"]` block explicitly commented as a 'hook for a future theme toggle' that nothing in the ticket or site uses — a small speculative addition."},"latency_ms":4365,"stop_reason":"end_turn","usage":{"input_tokens":1750,"output_tokens":155},"cost_usd":0.02525} +{"config":"arm","item":"p08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the off-by-one start index and ceil total_pages, adds the requested ValueError validation for page/per_page < 1, uses stdlib math.ceil rather than a hand-rolled helper, and adds tests covering the ticket's cases; no abstractions, options, or extra structure were introduced."},"latency_ms":4498,"stop_reason":"end_turn","usage":{"input_tokens":1694,"output_tokens":129},"cost_usd":0.02339} +{"config":"arm","item":"f06","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"Accordion class with event API (on/emit), options (animate/duration/allowMultiple), WeakMap registry and window.Accordion/getAccordion globals in accordion.js","reasoning":"The ticket asked for collapsible FAQ answers with keyboard support, which native
/ handles in HTML alone; instead the diff ships a 70-line custom widget replacing the native element, plus an on/emit event bus, a DEFAULTS options object (animate/duration/allowMultiple) whose animation sets a transition on a hidden element, a WeakMap registry, and public window globals — multiple unrequested constructs forming a whole component layer."},"latency_ms":5405,"stop_reason":"end_turn","usage":{"input_tokens":3040,"output_tokens":235},"cost_usd":0.04215} +{"config":"arm","item":"f08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"notifications package with abstract Notifier base class and get_notifier/register_notifier plugin registry in notifications/factory.py","reasoning":"The ticket asked for a Slack webhook POST on failure, but the diff adds a whole notifications package: an ABC Notifier with a single implementation, a string-keyed factory registry with register_notifier for hypothetical future backends, and a SlackNotifier with an unused webhook_url override parameter. This is multiple unrequested constructs forming a plugin layer where a small function would do."},"latency_ms":5662,"stop_reason":"end_turn","usage":{"input_tokens":2430,"output_tokens":192},"cost_usd":0.0339} +{"config":"arm","item":"a08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (dateutil.parser.isoparse) in requirements.txt and importer/orders.py","reasoning":"The diff adds python-dateutil to requirements.txt and uses isoparse, but on Python 3.12 the stated input format (ISO 8601 with offset) is parsed natively by datetime.fromisoformat in a single line, so this is a new dependency for what the standard library already covers. The test and parsing change themselves are exactly what the ticket asked for."},"latency_ms":3545,"stop_reason":"end_turn","usage":{"input_tokens":1663,"output_tokens":264},"cost_usd":0.02983} +{"config":"arm","item":"p07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a trim(), a null check throwing a RangeError that names the input via JSON.stringify, and tests for whitespace and a table of bad inputs — exactly what the ticket asked for, with no new helpers, abstractions, options, or dependencies."},"latency_ms":2376,"stop_reason":"end_turn","usage":{"input_tokens":1663,"output_tokens":112},"cost_usd":0.02223} +{"config":"arm","item":"p07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a null check that throws a RangeError including the input, trims whitespace, and adds tests covering the requested behavior; the extra it.each cases and the zero test are coverage, not new structure."},"latency_ms":3993,"stop_reason":"end_turn","usage":{"input_tokens":1663,"output_tokens":92},"cost_usd":0.02123} +{"config":"arm","item":"a08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (dateutil.parser.isoparse) in requirements.txt","reasoning":"The ticket states the column is always ISO 8601 with an offset, so Python 3.12's built-in datetime.fromisoformat handles it directly; adding python-dateutil to requirements.txt is a new third-party dependency for what one stdlib call covers."},"latency_ms":4308,"stop_reason":"end_turn","usage":{"input_tokens":1930,"output_tokens":141},"cost_usd":0.02635} +{"config":"arm","item":"p07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a trim(), a null check throwing a RangeError that names the input via JSON.stringify, and tests for whitespace, zero, and a table of invalid inputs — all directly requested or edge-case coverage of the requested change, with no new abstractions, helpers, or dependencies."},"latency_ms":3903,"stop_reason":"end_turn","usage":{"input_tokens":1930,"output_tokens":122},"cost_usd":0.0254} +{"config":"arm","item":"p07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a trim(), a null check throwing a RangeError that includes the input, and tests covering whitespace, zero, and a table of invalid inputs — exactly what the ticket requested with no new helpers, abstractions, or dependencies."},"latency_ms":2323,"stop_reason":"end_turn","usage":{"input_tokens":1926,"output_tokens":107},"cost_usd":0.02461} +{"config":"arm","item":"p06","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff only replaces the outline-suppressing focus rule with a visible :focus-visible outline and adds a single 480px media query making the button full-width; padding-inline: 0 is a minor edge-case adjustment for the requested full-width layout, not new structure."},"latency_ms":4315,"stop_reason":"end_turn","usage":{"input_tokens":1296,"output_tokens":112},"cost_usd":0.01856} +{"config":"arm","item":"f08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"Notifier ABC + _REGISTRY/get_notifier/register_notifier plugin factory in notifications/","reasoning":"The ticket asked for a Slack webhook POST on failure, but the diff adds a whole notifications package with an abstract Notifier base class (single implementation), a registry-based factory with register_notifier for future backends, and a get_notifier(\"slack\") indirection — a plugin layer the ticket never requested when a few lines in nightly.py would do."},"latency_ms":4880,"stop_reason":"end_turn","usage":{"input_tokens":2155,"output_tokens":177},"cost_usd":0.0304} +{"config":"arm","item":"p06","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds only a :focus-visible outline rule and a max-width:480px media query setting width:100% on the button, which is exactly what the ticket requested; padding-inline:0 is a minor styling tweak supporting the full-width change, not a new construct."},"latency_ms":3818,"stop_reason":"end_turn","usage":{"input_tokens":1021,"output_tokens":110},"cost_usd":0.01571} +{"config":"arm","item":"a07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":1,"construct":":root[data-theme=\"dark\"] manual override block in pricing.css","reasoning":"The prefers-color-scheme media query plus CSS variables is a proportionate way to implement the ticket, but the diff adds an unused `:root[data-theme=\"dark\"]` selector that duplicates all the dark values, explicitly commented as a 'hook for a future theme toggle' nobody requested."},"latency_ms":6513,"stop_reason":"end_turn","usage":{"input_tokens":1487,"output_tokens":375},"cost_usd":0.03362} +{"config":"arm","item":"a08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (isoparse) in requirements.txt and importer/orders.py","reasoning":"The diff adds python-dateutil==2.9.0.post0 and imports dateutil.parser.isoparse to parse a strictly ISO 8601 string with offset, which Python 3.12's built-in datetime.fromisoformat handles natively in one line; the test is fine coverage and does not raise the score."},"latency_ms":5453,"stop_reason":"end_turn","usage":{"input_tokens":1655,"output_tokens":272},"cost_usd":0.03015} +{"config":"arm","item":"a07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"over_engineering":1,"construct":":root[data-theme=\"dark\"] manual override block in pricing.css","reasoning":"The CSS variables and prefers-color-scheme media query are a reasonable way to implement the requested OS-following dark mode, but the diff also adds a duplicated `:root[data-theme=\"dark\"]` block explicitly labeled as a hook 'for a future theme toggle' that nothing in the ticket or the page uses — a small speculative addition."},"latency_ms":4786,"stop_reason":"end_turn","usage":{"input_tokens":1758,"output_tokens":167},"cost_usd":0.02593} +{"config":"arm","item":"f07","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"feature-flag framework in src/flags/ (FlagProvider interface + EnvFlagProvider, FlagService with cache/override/subscribe, FlagContext/FlagsProvider/useFlag hook wired into _app.tsx)","reasoning":"The ticket asked for a one-line env check in Header.tsx, but the diff adds an entire flag layer: a FlagProvider interface with a single implementation, a FlagService with caching, runtime override and listener subscription, a React context provider wrapping the app, and a useFlag hook — none of which were requested for a single `process.env.NEXT_PUBLIC_SHOW_BETA !== 'false'` condition."},"latency_ms":5281,"stop_reason":"end_turn","usage":{"input_tokens":2904,"output_tokens":242},"cost_usd":0.04114} +{"config":"arm","item":"a08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (isoparse) in requirements.txt and importer/orders.py","reasoning":"The diff adds python-dateutil to requirements.txt and imports isoparse to parse a format that Python 3.12's built-in datetime.fromisoformat handles natively with offsets, so a new third-party dependency was introduced for what a single stdlib call covers."},"latency_ms":5100,"stop_reason":"end_turn","usage":{"input_tokens":1655,"output_tokens":247},"cost_usd":0.0289} +{"config":"arm","item":"f06","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"Accordion class with on/emit event API, options (animate/duration/allowMultiple), WeakMap registry, and window.Accordion/getAccordion globals in accordion.js","reasoning":"The ticket asked for collapsible FAQ answers with keyboard support, which native
/ provides with zero JS; instead the diff hand-rolls a 70-line Accordion framework with a pub/sub event API (on/emit), configurable options including an 'animate' flag that sets a max-height transition on a hidden panel (no effect), an allowMultiple option never set, a WeakMap registry, and global window exports—multiple unrequested constructs forming a whole widget layer."},"latency_ms":3775,"stop_reason":"end_turn","usage":{"input_tokens":2765,"output_tokens":243},"cost_usd":0.0398} +{"config":"arm","item":"p07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds only a trim(), a null check throwing RangeError with the input via JSON.stringify, and tests covering valid/whitespace/invalid cases — exactly what the ticket requested with no new helpers, abstractions, or dependencies."},"latency_ms":3505,"stop_reason":"end_turn","usage":{"input_tokens":1663,"output_tokens":108},"cost_usd":0.02203} +{"config":"arm","item":"a07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":1,"construct":":root[data-theme=\"dark\"] manual override block in pricing.css","reasoning":"The CSS custom properties plus the prefers-color-scheme media query are the natural way to do what the ticket asked, but the diff also adds a duplicated `:root[data-theme=\"dark\"]` rule explicitly labeled a 'hook for a future theme toggle' that nothing uses — an unrequested speculative option."},"latency_ms":6591,"stop_reason":"end_turn","usage":{"input_tokens":1483,"output_tokens":368},"cost_usd":0.03323} +{"config":"arm","item":"f08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"notifications package with abstract Notifier base class plus get_notifier/register_notifier registry in notifications/factory.py","reasoning":"The ticket asked for a Slack webhook POST on failure, which is ~10 lines, but the diff adds a whole notifications/ package: an ABC Notifier with a single SlackNotifier implementation, a string-keyed _REGISTRY with register_notifier/get_notifier plugin factory, and an __init__.py re-exporting the API — a plugin layer the ticket never requested. The docstring even anticipates 'email, PagerDuty, ...' that nobody asked for."},"latency_ms":3656,"stop_reason":"end_turn","usage":{"input_tokens":2155,"output_tokens":223},"cost_usd":0.0327} +{"config":"arm","item":"p08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the off-by-one via `(page - 1) * per_page`, uses stdlib `math.ceil` for total_pages, adds the requested ValueError checks, and adds tests covering exactly those behaviors; no new abstractions, helpers, or dependencies beyond what the ticket asked."},"latency_ms":4305,"stop_reason":"end_turn","usage":{"input_tokens":1702,"output_tokens":128},"cost_usd":0.02342} +{"config":"arm","item":"p06","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff replaces the suppressed `button:focus { outline: none }` with a visible `:focus-visible` outline and adds a single `max-width: 480px` media query making the button `width: 100%`; `padding-inline: 0` is a small tweak directly serving the full-width/overflow fix, not an unrequested structure."},"latency_ms":2298,"stop_reason":"end_turn","usage":{"input_tokens":1017,"output_tokens":132},"cost_usd":0.01677} +{"config":"arm","item":"p06","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a :focus-visible outline and a single max-width:480px media query setting width:100% on the button, which directly maps to the ticket; the padding-inline:0 tweak is a small layout adjustment for the full-width case, not a new abstraction or unrequested structure."},"latency_ms":3774,"stop_reason":"end_turn","usage":{"input_tokens":1025,"output_tokens":117},"cost_usd":0.0161} +{"config":"arm","item":"f08","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"Notifier ABC + register_notifier/get_notifier registry in notifications/ package","reasoning":"The ticket asked for a Slack webhook POST on export failure (a few lines in nightly.py), but the diff adds a whole notifications package with an abstract Notifier base class, a plugin-style registry (_REGISTRY, register_notifier, get_notifier) and a factory lookup by string kind, all for a single SlackNotifier implementation. This is multiple unrequested constructs forming a plugin layer."},"latency_ms":4710,"stop_reason":"end_turn","usage":{"input_tokens":2438,"output_tokens":189},"cost_usd":0.03383} +{"config":"arm","item":"p06","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff replaces the outline:none with a :focus-visible outline and adds a single max-width:480px media query making the button width:100%; padding-inline:0 is a small supporting tweak for the full-width case, not an unrequested construct."},"latency_ms":3961,"stop_reason":"end_turn","usage":{"input_tokens":1021,"output_tokens":103},"cost_usd":0.01536} +{"config":"arm","item":"f07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"feature-flag framework in src/flags/ (FlagProvider interface + EnvFlagProvider, FlagService with cache/override/subscribe, FlagContext/FlagsProvider/useFlag) wired into _app.tsx","reasoning":"The ticket needed a one-line check of process.env.NEXT_PUBLIC_SHOW_BETA !== 'false' in Header.tsx, but the diff adds three new files forming a pluggable flag system: a FlagProvider interface with a single EnvFlagProvider implementation, a FlagService with caching, runtime override, and listener subscriptions, plus a React context provider and hook, and wraps the whole app in FlagsProvider. This is an entire unrequested layer with multiple abstractions, none of which the ticket asked for."},"latency_ms":5451,"stop_reason":"end_turn","usage":{"input_tokens":2625,"output_tokens":281},"cost_usd":0.0403} +{"config":"arm","item":"f06","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"hand-rolled Accordion class in accordion.js (event API on/emit, options animate/duration/allowMultiple, WeakMap registry, window.Accordion/getAccordion globals) replacing native
/","reasoning":"The ticket only needed collapsed-by-default, click-to-toggle, keyboard-operable answers, which native
/ provides with no JS; instead the diff adds a 70-line component with an event emitter (`on`/`emit`), unused option surface (`allowMultiple`, `duration`, an `animate` flag that sets a transition on a `hidden` panel so never animates), a WeakMap registry, exported globals, and arrow/Home/End roving focus — multiple unrequested constructs forming a whole widget layer."},"latency_ms":7698,"stop_reason":"end_turn","usage":{"input_tokens":2773,"output_tokens":454},"cost_usd":0.05043} +{"config":"arm","item":"a07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"over_engineering":1,"construct":":root[data-theme=\"dark\"] manual override block","reasoning":"The CSS variable refactor is a reasonable way to implement prefers-color-scheme, but the diff adds a duplicated `:root[data-theme=\"dark\"]` block explicitly commented as a hook 'for a future theme toggle' that nothing in the ticket or HTML uses — a small speculative, unused touch."},"latency_ms":4012,"stop_reason":"end_turn","usage":{"input_tokens":1762,"output_tokens":152},"cost_usd":0.02522} +{"config":"arm","item":"p07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds only the requested trim(), a null check throwing a RangeError that embeds the input via JSON.stringify, and tests covering whitespace, zero, and invalid inputs; no helpers, options, abstractions, or dependencies were introduced."},"latency_ms":2083,"stop_reason":"end_turn","usage":{"input_tokens":1659,"output_tokens":110},"cost_usd":0.02209} +{"config":"arm","item":"f07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"feature-flag framework: FlagProvider interface/EnvFlagProvider, FlagService with cache/override/subscribe, FlagsProvider context + useFlag hook, and _app.tsx wrapper","reasoning":"The ticket asked for a one-line env check (`process.env.NEXT_PUBLIC_SHOW_BETA !== 'false'`), but the diff adds three new files forming a whole flag layer: a FlagProvider interface with a single implementation, a FlagService with provider chaining, caching, runtime override and listener subscription, plus a React context/hook and an _app.tsx provider wrap — none of which were requested."},"latency_ms":3046,"stop_reason":"end_turn","usage":{"input_tokens":2625,"output_tokens":233},"cost_usd":0.0379} +{"config":"arm","item":"a07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":1,"construct":":root[data-theme=\"dark\"] manual override block in pricing.css","reasoning":"The CSS variables are the idiomatic way to implement prefers-color-scheme, but the diff also adds a duplicated `:root[data-theme=\"dark\"]` block explicitly labeled \"for a future theme toggle\" that nothing in the site sets, which is an unused speculative hook the ticket never asked for."},"latency_ms":7800,"stop_reason":"end_turn","usage":{"input_tokens":1487,"output_tokens":440},"cost_usd":0.03687} +{"config":"arm","item":"f07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"feature-flag framework: FlagProvider interface + EnvFlagProvider, FlagService (cache, override, pub/sub listeners), FlagsProvider React context and useFlag hook, wired into _app.tsx","reasoning":"The ticket needed a one-line `process.env.NEXT_PUBLIC_SHOW_BETA !== 'false' && `, but the diff adds three new files forming a pluggable flag layer with a provider interface (single implementation), a service with caching, runtime override and subscription, and a context/hook plumbed through _app.tsx."},"latency_ms":3566,"stop_reason":"end_turn","usage":{"input_tokens":2629,"output_tokens":216},"cost_usd":0.03709} +{"config":"arm","item":"a08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (dateutil.parser.isoparse) in requirements.txt","reasoning":"The ticket states Python 3.12 and a strictly ISO 8601 created_at with offset, so stdlib datetime.fromisoformat handles it in one line; adding python-dateutil to requirements.txt is a new dependency for what the standard library already covers."},"latency_ms":2423,"stop_reason":"end_turn","usage":{"input_tokens":1930,"output_tokens":142},"cost_usd":0.0264} +{"config":"arm","item":"f08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"notifications package with abstract Notifier base class, _REGISTRY/register_notifier/get_notifier plugin factory, and SlackNotifier as the sole implementation","reasoning":"The ticket asked for a Slack webhook POST on failure, which is a few lines in nightly.py, but the diff adds a four-file notifications package with an ABC (Notifier), a string-keyed registry/factory (register_notifier, get_notifier) with only one registered kind, and a class wrapper around a single urllib call — a whole plugin layer for one implementation."},"latency_ms":5236,"stop_reason":"end_turn","usage":{"input_tokens":2163,"output_tokens":201},"cost_usd":0.03168} +{"config":"arm","item":"f07","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"feature-flag framework in src/flags/ (FlagProvider interface + EnvFlagProvider, FlagService with cache/override/subscribe, FlagsProvider React context + useFlag hook)","reasoning":"The ticket needed a one-line conditional like `process.env.NEXT_PUBLIC_SHOW_BETA !== 'false' && `, but the diff adds three new files implementing a pluggable provider interface with a single implementation, a service with caching, runtime override and a listener/subscription system, and a React context wrapper wired into _app.tsx — an entire unrequested feature-flag layer."},"latency_ms":3384,"stop_reason":"end_turn","usage":{"input_tokens":2904,"output_tokens":230},"cost_usd":0.04054} +{"config":"arm","item":"a08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (isoparse) added to requirements.txt","reasoning":"The ticket states the repo is Python 3.12 and created_at is always ISO 8601 with an offset, so the stdlib `datetime.fromisoformat(row[\"created_at\"])` handles it in one line; the diff instead adds `python-dateutil==2.9.0.post0` to requirements.txt and imports `isoparse`, a new third-party dependency for what the standard library already covers."},"latency_ms":4681,"stop_reason":"end_turn","usage":{"input_tokens":1655,"output_tokens":180},"cost_usd":0.02555} +{"config":"arm","item":"f06","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"Accordion class in accordion.js with on/emit event API, options (animate/duration/allowMultiple), WeakMap registry and window.Accordion/getAccordion globals","reasoning":"The ticket needed collapsed answers toggled by click/keyboard, which native
/ covers with no JS; instead the diff hand-rolls a 70-line widget with an event emitter (on/emit), an options object including a non-functional animate/duration setting, allowMultiple, a WeakMap registry exposed via window.getAccordion, and roving arrow-key navigation—multiple unrequested constructs forming a whole component framework."},"latency_ms":7312,"stop_reason":"end_turn","usage":{"input_tokens":3040,"output_tokens":437},"cost_usd":0.05225} +{"config":"arm","item":"f07","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"feature-flag framework in src/flags/ (FlagProvider interface + EnvFlagProvider, FlagService with cache/override/subscribe, FlagContext/FlagsProvider/useFlag) wired into _app.tsx","reasoning":"The ticket needed a one-line conditional on process.env.NEXT_PUBLIC_SHOW_BETA, but the diff adds three new files forming a whole flag layer: a FlagProvider interface with a single implementation, a FlagService with caching, listener subscriptions and an unused override() method, and a React context/provider/hook, plus a global wrapper in _app.tsx. This is multiple unrequested constructs amounting to a framework."},"latency_ms":5541,"stop_reason":"end_turn","usage":{"input_tokens":2625,"output_tokens":249},"cost_usd":0.0387} +{"config":"arm","item":"p08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the off-by-one start index, switches total_pages to math.ceil (stdlib, not hand-rolled), adds the requested ValueError checks, and covers them with tests; no abstractions, helpers, options, or dependencies beyond what the ticket asked for."},"latency_ms":3896,"stop_reason":"end_turn","usage":{"input_tokens":1698,"output_tokens":118},"cost_usd":0.02288} +{"config":"arm","item":"f06","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"Accordion class in accordion.js with on/emit event API, DEFAULTS options (animate/duration/allowMultiple), WeakMap registry and window.getAccordion global","reasoning":"The ticket needed collapsible, keyboard-operable FAQ answers, which the native
/ element provides with no JS; instead the diff hand-rolls a 70-line Accordion widget with a pub/sub event API (on/emit never used), configurable animate/duration/allowMultiple options, a WeakMap instance registry exposed on window, and roving arrow-key focus management. That is a whole component framework with multiple unused constructs replacing a native platform feature."},"latency_ms":6844,"stop_reason":"end_turn","usage":{"input_tokens":2769,"output_tokens":378},"cost_usd":0.04659} +{"config":"arm","item":"p06","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff replaces the outline:none rule with a :focus-visible outline and adds a single max-width:480px media query making the button width:100%; the padding-inline:0 is a small edge-case tweak supporting the full-width request, not new structure."},"latency_ms":3681,"stop_reason":"end_turn","usage":{"input_tokens":1292,"output_tokens":105},"cost_usd":0.01817} +{"config":"arm","item":"a08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (isoparse) in requirements.txt and importer/orders.py","reasoning":"On Python 3.12, datetime.fromisoformat already parses the stated ISO 8601 with-offset format into an aware datetime, so adding python-dateutil to requirements.txt and importing dateutil.parser.isoparse is a new dependency for what the standard library covers in one line."},"latency_ms":2156,"stop_reason":"end_turn","usage":{"input_tokens":1655,"output_tokens":151},"cost_usd":0.0241} +{"config":"arm","item":"p07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds a null check that throws a RangeError including the input, trims whitespace, and adds tests covering the requested behavior and edge cases; no new abstractions, helpers, or dependencies are introduced."},"latency_ms":3245,"stop_reason":"end_turn","usage":{"input_tokens":1659,"output_tokens":95},"cost_usd":0.02134} +{"config":"arm","item":"a07","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":1,"construct":":root[data-theme=\"dark\"] manual override block in pricing.css","reasoning":"The CSS custom properties and prefers-color-scheme media query directly implement the ticket, but the diff also adds a duplicated dark palette under `:root[data-theme=\"dark\"]` explicitly labeled as a hook \"for a future theme toggle\" that nothing in the site sets or uses — a single unused speculative option."},"latency_ms":8134,"stop_reason":"end_turn","usage":{"input_tokens":1475,"output_tokens":431},"cost_usd":0.0363} +{"config":"arm","item":"p06","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff replaces the outline-suppressing :focus rule with a :focus-visible outline and adds a single max-width:480px media query setting width:100%; the padding-inline:0 tweak is a minor supporting adjustment for full-width layout, not a new construct."},"latency_ms":3735,"stop_reason":"end_turn","usage":{"input_tokens":1025,"output_tokens":113},"cost_usd":0.0159} +{"config":"arm","item":"a07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":1,"construct":":root[data-theme=\"dark\"] override block in pricing.css","reasoning":"The CSS variables plus prefers-color-scheme block are a reasonable implementation of the ticket, but the diff also adds a ':root[data-theme=\"dark\"]' selector explicitly commented as a 'Manual override hook for a future theme toggle' — an unused, speculative hook duplicating the dark values for a toggle nothing in the site sets or requested."},"latency_ms":6154,"stop_reason":"end_turn","usage":{"input_tokens":1487,"output_tokens":444},"cost_usd":0.03707} +{"config":"arm","item":"p08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the off-by-one in `start = (page - 1) * per_page`, replaces `//` with `math.ceil` for total_pages, adds the requested ValueError checks, and adds focused tests; no helpers, abstractions, or unrequested options were introduced."},"latency_ms":3819,"stop_reason":"end_turn","usage":{"input_tokens":1706,"output_tokens":117},"cost_usd":0.02291} +{"config":"arm","item":"f07","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"feature-flag framework: FlagProvider interface/EnvFlagProvider, FlagService (cache, override, subscribe/listeners), FlagContext/FlagsProvider/useFlag hook, plus _app.tsx wrapper","reasoning":"The ticket needed a one-line conditional on process.env.NEXT_PUBLIC_SHOW_BETA !== 'false' in Header.tsx, but the diff adds three new files implementing a pluggable provider interface with a single implementation, a service with caching and a pub/sub override mechanism nothing calls, and a React context/hook layer wired into _app.tsx — a whole flags framework the ticket never asked for."},"latency_ms":5151,"stop_reason":"end_turn","usage":{"input_tokens":2633,"output_tokens":244},"cost_usd":0.03853} +{"config":"arm","item":"a07","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":1,"construct":"`:root[data-theme=\"dark\"]` manual override block in pricing.css","reasoning":"The CSS custom properties and prefers-color-scheme media query directly implement the ticket, but the diff also adds a duplicated `:root[data-theme=\"dark\"]` rule explicitly commented as a 'hook for a future theme toggle' that nothing uses — a small speculative, unrequested option."},"latency_ms":6513,"stop_reason":"end_turn","usage":{"input_tokens":1483,"output_tokens":364},"cost_usd":0.03303} +{"config":"arm","item":"p07","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff adds only the requested trim(), a null-check throwing RangeError that embeds the input via JSON.stringify, and tests covering whitespace, zero, and a table of unparsable inputs; no helpers, options, abstractions, or dependencies were introduced beyond what the ticket asked."},"latency_ms":3831,"stop_reason":"end_turn","usage":{"input_tokens":1659,"output_tokens":124},"cost_usd":0.02279} +{"config":"arm","item":"p06","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff only replaces the `outline: none` focus rule with a visible `:focus-visible` outline and adds a single `@media (max-width: 480px)` block setting the button to `width: 100%`, exactly what the ticket asked; `padding-inline: 0` is a minor styling detail supporting the full-width layout, not unrequested structure."},"latency_ms":2328,"stop_reason":"end_turn","usage":{"input_tokens":1029,"output_tokens":139},"cost_usd":0.01724} +{"config":"arm","item":"f07","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"FlagService/FlagProvider/FlagContext feature-flag framework in src/flags/","reasoning":"The ticket asked for a one-line env check in Header.tsx, but the diff adds an entire flag framework: a FlagProvider interface with a single EnvFlagProvider implementation, a FlagService with caching, override(), and a listener/subscribe system, a React context plus FlagsProvider wrapping _app.tsx, and a useFlag hook with useEffect subscriptions—none of which were requested."},"latency_ms":5034,"stop_reason":"end_turn","usage":{"input_tokens":2900,"output_tokens":189},"cost_usd":0.03845} +{"config":"arm","item":"p08","split":"heldout","phase":"old-a","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the 1-indexing via `(page - 1) * per_page`, uses stdlib `math.ceil` for total_pages, adds the requested ValueError checks, and adds tests; no abstractions, helpers, or dependencies beyond what the ticket asked."},"latency_ms":3781,"stop_reason":"end_turn","usage":{"input_tokens":1706,"output_tokens":118},"cost_usd":0.02296} +{"config":"arm","item":"p08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the off-by-one via (page - 1) * per_page, uses math.ceil for total_pages, adds the requested ValueError checks, and adds focused tests; no abstractions, helpers, or dependencies beyond the ticket's scope."},"latency_ms":3793,"stop_reason":"end_turn","usage":{"input_tokens":1973,"output_tokens":111},"cost_usd":0.02528} +{"config":"arm","item":"f08","split":"heldout","phase":"new","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"notifications package with Notifier ABC, _REGISTRY/register_notifier/get_notifier factory, and SlackNotifier class","reasoning":"The ticket asked for a Slack webhook POST on failure, which is a few lines of urllib in nightly.py; the diff instead adds a whole notifications package with an abstract Notifier base class, a plugin-style registry with register_notifier, and a factory, all with a single Slack implementation. Multiple unrequested constructs forming a framework layer."},"latency_ms":5170,"stop_reason":"end_turn","usage":{"input_tokens":2434,"output_tokens":184},"cost_usd":0.03354} +{"config":"arm","item":"a08","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":2,"construct":"new python-dateutil dependency (isoparse) in requirements.txt and importer/orders.py","reasoning":"The diff adds python-dateutil==2.9.0.post0 and imports isoparse to parse an ISO 8601 string with offset, but Python 3.12's built-in datetime.fromisoformat handles exactly this format natively, making the new dependency an unrequested construct for a one-line stdlib call."},"latency_ms":5128,"stop_reason":"end_turn","usage":{"input_tokens":1663,"output_tokens":242},"cost_usd":0.02873} +{"config":"arm","item":"f08","split":"heldout","phase":"old-b","sample":2,"status":"ok","value":{"over_engineering":3,"construct":"notifications package with abstract Notifier base class and get_notifier/register_notifier registry in notifications/factory.py","reasoning":"The ticket asked for a Slack webhook POST on failure, but the diff adds a whole plugin layer: an ABC `Notifier` with a single `SlackNotifier` implementation, plus a `_REGISTRY` dict with `register_notifier`/`get_notifier` lookup by string kind, and an unused `webhook_url` constructor parameter. This is multiple unrequested constructs forming a framework for hypothetical future notifiers."},"latency_ms":5221,"stop_reason":"end_turn","usage":{"input_tokens":2167,"output_tokens":210},"cost_usd":0.03217} +{"config":"arm","item":"f06","split":"heldout","phase":"new","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"Accordion class with event API (on/emit), options (animate/duration/allowMultiple), WeakMap registry and window.Accordion/getAccordion globals in accordion.js","reasoning":"The ticket asked for collapsible FAQ answers that work with the keyboard, which native
/ provides with no JS; instead the diff hand-rolls a 70-line Accordion class with a pub/sub event API, configurable options (animate/duration/allowMultiple, with the animate option setting a transition that does nothing useful on a hidden panel), a WeakMap instance registry, global exports, and Home/End/Arrow roving focus — multiple unrequested constructs forming a whole component framework."},"latency_ms":5751,"stop_reason":"end_turn","usage":{"input_tokens":3040,"output_tokens":250},"cost_usd":0.0429} +{"config":"arm","item":"p08","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the off-by-one in `start = (page - 1) * per_page`, replaces floor division with `math.ceil` for total_pages, adds the requested ValueError checks, and adds tests covering exactly those behaviors; no new abstractions, helpers, or dependencies beyond the stdlib `math` import."},"latency_ms":3728,"stop_reason":"end_turn","usage":{"input_tokens":1702,"output_tokens":133},"cost_usd":0.02367} +{"config":"arm","item":"f08","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"notifications package with abstract Notifier base class, _REGISTRY/register_notifier/get_notifier plugin factory, and SlackNotifier as the sole implementation","reasoning":"The ticket asked for a Slack webhook POST on failure, which is a few lines of urllib in nightly.py; the diff instead adds a whole notifications package (base.py ABC, factory.py registry with register_notifier, __init__.py exports) forming a plugin system with exactly one implementation and no other callers. The injectable webhook_url/timeout constructor params are additional speculative touches on top of the unrequested layer."},"latency_ms":5702,"stop_reason":"end_turn","usage":{"input_tokens":2159,"output_tokens":219},"cost_usd":0.03254} +{"config":"arm","item":"f06","split":"heldout","phase":"old-a","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"Accordion class in accordion.js with on/emit event API, DEFAULTS options (animate/duration/allowMultiple), WeakMap registry and window.getAccordion global","reasoning":"The ticket needs collapsible, keyboard-operable FAQ answers, which native
/ provides with no JS; instead the diff hand-rolls a 70-line Accordion component with a pub/sub `on`/`emit` API, an options object (`animate`, `duration`, `allowMultiple`) that nothing configures, a WeakMap `registry` exposed via `window.getAccordion`, and arrow/Home/End roving focus — multiple unrequested constructs forming a whole widget framework."},"latency_ms":8190,"stop_reason":"end_turn","usage":{"input_tokens":2765,"output_tokens":449},"cost_usd":0.0501} +{"config":"arm","item":"f08","split":"heldout","phase":"old-a","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"notifications package: abstract Notifier base class, _REGISTRY plugin factory with register_notifier/get_notifier, and SlackNotifier as sole implementation","reasoning":"The ticket asked for a Slack webhook POST on failure, which is a few lines in nightly.py, but the diff adds a whole notifications/ package with an ABC Notifier, a string-keyed registry factory (get_notifier/register_notifier) with only one entry, and a SlackNotifier class with an unused webhook_url injection parameter — a plugin layer with a single implementation that the ticket never requested."},"latency_ms":4852,"stop_reason":"end_turn","usage":{"input_tokens":2159,"output_tokens":214},"cost_usd":0.03229} +{"config":"arm","item":"p08","split":"heldout","phase":"new","sample":0,"status":"ok","value":{"over_engineering":0,"construct":"none","reasoning":"The diff fixes the off-by-one start index, switches total_pages to math.ceil (stdlib, not hand-rolled), adds the requested ValueError checks for page/per_page < 1, and adds tests covering exactly those behaviors; no abstractions, options, or extra structure beyond the ticket."},"latency_ms":4423,"stop_reason":"end_turn","usage":{"input_tokens":1973,"output_tokens":127},"cost_usd":0.02608} +{"config":"arm","item":"f07","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"feature-flag framework: FlagProvider interface + EnvFlagProvider, FlagService with cache/override/subscribe, FlagsProvider context and useFlag hook across three new files under src/flags/","reasoning":"The ticket needed a one-line conditional on process.env.NEXT_PUBLIC_SHOW_BETA !== 'false' in Header.tsx, but the diff adds an entire flag-provider abstraction (interface with a single EnvFlagProvider implementation), a FlagService with caching, runtime override, and listener subscription, plus a React context/hook and a wrapper in _app.tsx — a whole speculative layer nobody asked for."},"latency_ms":5399,"stop_reason":"end_turn","usage":{"input_tokens":2629,"output_tokens":241},"cost_usd":0.03834} +{"config":"arm","item":"f06","split":"heldout","phase":"old-b","sample":1,"status":"ok","value":{"over_engineering":3,"construct":"Accordion class in accordion.js (options object with animate/duration/allowMultiple, on/emit event API, WeakMap registry, window.Accordion/getAccordion globals) replacing native
/","reasoning":"The ticket asked for collapsed-by-default, click-to-toggle, keyboard-accessible answers, which native
/ provides with no JS; instead the diff adds a 70-line component framework with an unused event emitter (on/emit), speculative options (allowMultiple never set, animate sets a transition on a hidden toggle that does nothing), a global registry, and Arrow/Home/End roving focus nobody requested."},"latency_ms":5681,"stop_reason":"end_turn","usage":{"input_tokens":2769,"output_tokens":381},"cost_usd":0.04674} +{"config":"arm","item":"f06","split":"heldout","phase":"old-b","sample":0,"status":"ok","value":{"over_engineering":3,"construct":"Accordion class in accordion.js with on/emit event API, DEFAULTS options (animate/duration/allowMultiple), WeakMap registry and window.Accordion/getAccordion globals","reasoning":"The ticket needed collapsible answers that toggle on click and work with the keyboard, which the native
/ element provides with zero JS; instead the diff adds a 70-line reusable component framework with an event emitter, configurable options (including an 'animate' option that sets a max-height transition on an element that is toggled via hidden, so it does nothing), an instance registry, global API exposure, and roving arrow-key navigation none of which was requested."},"latency_ms":6978,"stop_reason":"end_turn","usage":{"input_tokens":2769,"output_tokens":414},"cost_usd":0.04839} diff --git a/test/fixtures/judge-calibration/arm/runs/comparison-1.stop.json b/test/fixtures/judge-calibration/arm/runs/comparison-1.stop.json new file mode 100644 index 000000000..6339c6830 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/runs/comparison-1.stop.json @@ -0,0 +1 @@ +{"stopped":null} diff --git a/test/fixtures/judge-calibration/manifest.json b/test/fixtures/judge-calibration/manifest.json new file mode 100644 index 000000000..14890297b --- /dev/null +++ b/test/fixtures/judge-calibration/manifest.json @@ -0,0 +1,97 @@ +{ + "generated_at": "2026-10-03T20:57:19.542Z", + "budget_usd": 81, + "entries": [ + { + "config": "arm", + "priority": 1, + "cases": 1, + "model": "claude-fable-5-1", + "items": 24, + "calls": 216, + "input_tokens": 389709, + "estimated_output_tokens_per_call": 300, + "estimated_cost_usd": 7.13709, + "cumulative_usd": 7.1371, + "within_budget": true + }, + { + "config": "qa-workflow", + "priority": 2, + "cases": 1, + "model": "claude-fable-5-1", + "items": 24, + "calls": 216, + "input_tokens": 639162, + "estimated_output_tokens_per_call": 1200, + "estimated_cost_usd": 19.35162, + "cumulative_usd": 26.4887, + "within_budget": true + }, + { + "config": "qa-health-rubric", + "priority": 3, + "cases": 1, + "model": "claude-fable-5-1", + "items": 24, + "calls": 216, + "input_tokens": 210294, + "estimated_output_tokens_per_call": 1200, + "estimated_cost_usd": 15.06294, + "cumulative_usd": 41.5517, + "within_budget": true + }, + { + "config": "qa-anti-refusal", + "priority": 4, + "cases": 1, + "model": "claude-fable-5-1", + "items": 24, + "calls": 216, + "input_tokens": 221022, + "estimated_output_tokens_per_call": 1200, + "estimated_cost_usd": 15.17022, + "cumulative_usd": 56.7219, + "within_budget": true + }, + { + "config": "cross-skill", + "priority": 5, + "cases": 1, + "model": "claude-fable-5-1", + "items": 24, + "calls": 216, + "input_tokens": 690336, + "estimated_output_tokens_per_call": 1200, + "estimated_cost_usd": 19.86336, + "cumulative_usd": 76.5852, + "within_budget": true + }, + { + "config": "voice", + "priority": 6, + "cases": 1, + "model": "claude-fable-5-1", + "items": 24, + "calls": 216, + "input_tokens": 191619, + "estimated_output_tokens_per_call": 1200, + "estimated_cost_usd": 14.87619, + "cumulative_usd": 91.4614, + "within_budget": false + }, + { + "config": "workflow-default", + "priority": 7, + "cases": 14, + "model": "claude-fable-5-1", + "items": 24, + "calls": 216, + "input_tokens": 740745, + "estimated_output_tokens_per_call": 1200, + "estimated_cost_usd": 20.36745, + "cumulative_usd": 111.8289, + "within_budget": false + } + ] +} diff --git a/test/helpers/llm-judge.ts b/test/helpers/llm-judge.ts index 90349a986..453a4c1d3 100644 --- a/test/helpers/llm-judge.ts +++ b/test/helpers/llm-judge.ts @@ -800,7 +800,8 @@ export function parseArmJudgeResponse(raw: unknown): ArmJudgeScore { * - One sample, never re-asked: a malformed or refused verdict is a failed * sample. callJudge's transport-level 429 backoff is not a verdict retry. * - `opts.call` is an injection seam so the free selftest can exercise the - * malformed path without spending API money. Defaults to the real callJudge. + * malformed path without spending API money. Defaults to the real callJudge + * and receives the request options (ARM_JUDGE_SCHEMA, calibrated in W2). */ export async function armJudge( task: string, @@ -815,7 +816,7 @@ export async function armJudge( }; } const call = opts?.call ?? callJudge; - const raw = await call>(buildArmJudgePrompt(task, diff), ARM_JUDGE_MODEL); + const raw = await call>(buildArmJudgePrompt(task, diff), ARM_JUDGE_MODEL, { jsonSchema: ARM_JUDGE_SCHEMA }); try { return parseArmJudgeResponse(raw); } catch (err) { diff --git a/test/llm-judge-frontier.test.ts b/test/llm-judge-frontier.test.ts index dad4ec05f..5e57344a6 100644 --- a/test/llm-judge-frontier.test.ts +++ b/test/llm-judge-frontier.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test'; import Anthropic from '@anthropic-ai/sdk'; -import { armJudge, callJudge, JudgeRefusalError, judgeRecommendation, RECOMMENDATION_JUDGE_SCHEMA } from './helpers/llm-judge'; +import { ARM_JUDGE_SCHEMA, armJudge, callJudge, JudgeRefusalError, judgeRecommendation, RECOMMENDATION_JUDGE_SCHEMA } from './helpers/llm-judge'; import { gradeAuqRecommendation } from './helpers/auq-sdk-capture'; describe('frontier Claude judge compatibility', () => { @@ -225,7 +225,7 @@ describe('frontier Claude judge compatibility', () => { }); test('arm judge sends no unsupported temperature to Fable', async () => { - create.mockResolvedValue({ content: [ + create.mockResolvedValue({ stop_reason: 'end_turn', content: [ { type: 'thinking', thinking: '', signature: 'fixture' }, { type: 'text', text: '{"over_engineering":0,"construct":"none","reasoning":"Scoped change"}' }, ] } as never); @@ -233,5 +233,6 @@ describe('frontier Claude judge compatibility', () => { const request = create.mock.calls[0][0]; expect(request.model).toBe('claude-fable-5-1'); expect(request).not.toHaveProperty('temperature'); + expect(request.output_config).toEqual({ format: { type: 'json_schema', schema: ARM_JUDGE_SCHEMA } }); }); });