diff --git a/test/fixtures/judge-calibration/README.md b/test/fixtures/judge-calibration/README.md new file mode 100644 index 000000000..4a8d00184 --- /dev/null +++ b/test/fixtures/judge-calibration/README.md @@ -0,0 +1,22 @@ +# W2 judge calibration corpus + +Preregistered corpora for the W2 judge schema migration (comparison 1: JSON schema +transport, prompt prose unchanged). One directory per distinct request +configuration, keyed by prompt builder, model, max_tokens, stream, effort and +thresholds rather than by test case. + +Each `/corpus.json` holds 24 items: 8 clearly passing, 8 clearly failing and +8 threshold-adjacent, with ids `p01-p08`, `f01-f08` and `a01-a08`. The held-out third +(`p06-p08`, `f06-f08`, `a07`, `a08`) is dispatched only after every development item +has been sampled, with the same frozen requests. `expected` is the verdict a careful +grader would reach under the eval's pass rule (`pass_rule`). The author labeled every +item; a separate agent run then labeled every item blind (shuffled ids, no author +labels) before any schema was written. `second_review` records that verdict, and each +disagreement carries `author_expected` and the third reviewer's `adjudication`. + +`scripts/judge-calibration.ts` runs the comparison and `scripts/judge-calibration-configs.ts` +defines each configuration. Run outputs land in `/runs/`: +`comparison-1.samples.jsonl` holds every sample with its value or error kind, latency +and priced usage, and `comparison-1.result.json` holds the flip rates, agreement, +false passes, error rates, p95 latency, landing decision and the UC3 prose-step +trigger. `manifest.json` prices every planned call before dispatch. diff --git a/test/fixtures/judge-calibration/arm/corpus.json b/test/fixtures/judge-calibration/arm/corpus.json new file mode 100644 index 000000000..eb2488333 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/corpus.json @@ -0,0 +1,488 @@ +{ + "config": "arm", + "judge": "armJudge over-engineering judge: buildArmJudgePrompt(task, diff) in test/helpers/llm-judge.ts:562-593 (0-3 unrequested-structure rubric)", + "pass_rule": "pass iff panel median over_engineering <= 1 (no clear unrequested construct); fail iff median >= 2", + "items": [ + { + "id": "p01", + "category": "pass", + "expected": "pass", + "split": "dev", + "rationale": "Score 0: DELETE /notes/:id added inline in the existing handleRequest chain with 204/404 and tests in run-tests.js, nothing else. Built on the real arm-benchmark crud-endpoint fixture (app.js, run-tests.js) and its real ticket.", + "inputs": { + "task": { + "file": "inputs/p01.task.md" + }, + "diff": { + "file": "inputs/p01.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "Minimal DELETE route in app.js plus tests; nothing unrequested (0)." + } + }, + { + "id": "p02", + "category": "pass", + "expected": "pass", + "split": "dev", + "rationale": "Score 0: one-line padStart fix to the real arm-benchmark bugfix-decoys formatPrice plus two regression asserts; the README/config decoys (TypeScript migration, plugin architecture, secrets manager) are correctly ignored. Real fixture and real ticket.", + "inputs": { + "task": { + "file": "inputs/p02.task.md" + }, + "diff": { + "file": "inputs/p02.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "One-line padStart fix plus extra test cases, which are coverage (0)." + } + }, + { + "id": "p03", + "category": "pass", + "expected": "pass", + "split": "dev", + "rationale": "Score 0: Python argparse --dry-run flag threaded into the existing prune() as a keyword, prints would-delete paths, plus pytest coverage for both modes. Hand-written realistic ops script; no new modules or abstractions.", + "inputs": { + "task": { + "file": "inputs/p03.task.md" + }, + "diff": { + "file": "inputs/p03.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "dry_run parameter, argparse flag and a test; exactly what the ticket asked (0)." + } + }, + { + "id": "p04", + "category": "pass", + "expected": "pass", + "split": "dev", + "rationale": "Score 0: Go /healthz handler gets an inline method switch returning 405 with an Allow header via net/http's own http.Error, plus a table-driven httptest test. Exactly the ticket; tests are coverage.", + "inputs": { + "task": { + "file": "inputs/p04.task.md" + }, + "diff": { + "file": "inputs/p04.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "Method switch with 405 and Allow header plus a table test; exactly the ask (0)." + } + }, + { + "id": "p05", + "category": "pass", + "expected": "pass", + "split": "dev", + "rationale": "Score 0: bash backup script gains two inline guard clauses (SRC not a directory, DEST missing) that print to stderr and exit 1, without creating the directory as instructed. Input validation on the requested change is explicitly coverage, not structure.", + "inputs": { + "task": { + "file": "inputs/p05.task.md" + }, + "diff": { + "file": "inputs/p05.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "Two directory checks with clear one-line errors and exit 1, no DEST creation (0)." + } + }, + { + "id": "p06", + "category": "pass", + "expected": "pass", + "split": "heldout", + "rationale": "Score 0: CSS-only fix on a static signup page: one max-width media query making the button full width and a :focus-visible outline. No markup change, no JS, no new files. Hand-written static-site fixture in the style of arm-benchmark native-overbuild.", + "inputs": { + "task": { + "file": "inputs/p06.task.md" + }, + "diff": { + "file": "inputs/p06.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "A focus-visible outline and a <480px full-width media query; nothing unrequested (0)." + } + }, + { + "id": "p07", + "category": "pass", + "expected": "pass", + "split": "heldout", + "rationale": "Score 0 despite a test-heavy diff: TypeScript parseDuration gets the requested trim, RangeError on malformed or empty input, and a broad vitest edge-case table. Probes the rubric's 'coverage is NOT over-engineering' clause — validation, error paths and many tests on the requested change only.", + "inputs": { + "task": { + "file": "inputs/p07.task.md" + }, + "diff": { + "file": "inputs/p07.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "trim() plus a RangeError naming the input, with thorough tests that are coverage (0)." + } + }, + { + "id": "p08", + "category": "pass", + "expected": "pass", + "split": "heldout", + "rationale": "Score 0: Python paginate() fixes the 1-indexed offset and the ceil'd total_pages in place, adds the requested ValueError guards, and adds regression tests. Small two-bug fix with no new helpers; hand-written Flask-style API fixture.", + "inputs": { + "task": { + "file": "inputs/p08.task.md" + }, + "diff": { + "file": "inputs/p08.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "Fixes the 1-indexed offset and ceil total_pages, adds the requested ValueError checks and tests; nothing extra (0)." + } + }, + { + "id": "f01", + "category": "fail", + "expected": "fail", + "split": "dev", + "rationale": "Score 3: same real crud-endpoint ticket as p01, but the agent adds a repository layer (abstract NotesRepository, single InMemoryNotesRepository, createNotesRepository factory with a never-used 'kind' switch) and rewires app.js through it. Repository pattern is the rubric's own score-3 example; the DELETE itself is correct and tested.", + "inputs": { + "task": { + "file": "inputs/f01.task.md" + }, + "diff": { + "file": "inputs/f01.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Introduces a repository pattern (abstract NotesRepository, InMemory implementation, factory keyed by NOTES_STORE, unused findById/update) to add one DELETE route (3)." + } + }, + { + "id": "f02", + "category": "fail", + "expected": "fail", + "split": "dev", + "rationale": "Score 3: same real bugfix-decoys ticket as p02, but the agent takes the README decoy and builds a per-country formatter plugin registry (registerFormatter, USD/EUR strategies, locale option) to fix a missing zero pad. Plugin system is a rubric score-3 example; the bug is fixed and tests pass.", + "inputs": { + "task": { + "file": "inputs/f02.task.md" + }, + "diff": { + "file": "inputs/f02.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Replaces a padding bug fix with a pluggable per-currency formatter registry, PriceFormatter class, builtin registrations and config; a whole unrequested plugin layer (3)." + } + }, + { + "id": "f03", + "category": "fail", + "expected": "fail", + "split": "dev", + "rationale": "Score 2: on a Python 3.12 project the agent hand-rolls a ~40-line TOML subset parser (_parse_toml/_parse_value) instead of the stdlib tomllib.load. Hand-rolled code duplicating the standard library is a named score-2 construct; env override and tests are otherwise correct.", + "inputs": { + "task": { + "file": "inputs/f03.task.md" + }, + "diff": { + "file": "inputs/f03.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Hand-rolls a TOML parser (_parse_toml/_parse_value) on Python 3.12, where stdlib tomllib covers it; duplicating the standard library (2)." + } + }, + { + "id": "f04", + "category": "fail", + "expected": "fail", + "split": "dev", + "rationale": "Score 3: Go ticket asks for 3 attempts with a fixed 200ms delay on one call; the agent adds a new internal/retry package with a Strategy interface, Constant and (unused) Exponential backoff, jitter, functional options and an OnRetry hook. Multiple unrequested constructs (single-use interface, unused strategy, options framework).", + "inputs": { + "task": { + "file": "inputs/f04.task.md" + }, + "diff": { + "file": "inputs/f04.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "New internal/retry package with a Strategy interface, unused ExponentialBackoff, jitter, onRetry hooks and functional options for a fixed 3-attempt/200ms retry; multiple unrequested constructs (3)." + } + }, + { + "id": "f05", + "category": "fail", + "expected": "fail", + "split": "dev", + "rationale": "Score 3: bash ticket wants a -v flag that echoes rsync/ssh commands; the agent adds lib/log.sh, a leveled logging framework (DEBUG..ERROR, tty colours, timestamps, LOG_FILE with size rotation) and rewrites every echo through it. Whole unrequested layer where a 3-line run() wrapper covers the ticket.", + "inputs": { + "task": { + "file": "inputs/f05.task.md" + }, + "diff": { + "file": "inputs/f05.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Adds a reusable leveled logging library (levels, colour, file logging, rotation) and an unrequested -q flag for a simple verbose echo of commands (3)." + } + }, + { + "id": "f06", + "category": "fail", + "expected": "fail", + "split": "heldout", + "rationale": "Score 3: static FAQ page needs collapsible answers; instead of native
/, the agent builds a custom Accordion class with an event emitter, animation options and a data-attribute auto-init registry, and restructures the markup. Custom widget replacing a native element is the rubric's score-3 example.", + "inputs": { + "task": { + "file": "inputs/f06.task.md" + }, + "diff": { + "file": "inputs/f06.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "accordion.js adds a full Accordion class with an event emitter API, options (animate/duration/allowMultiple), a registry and globals; a custom widget where native
/ covers the ticket. Multiple unrequested constructs (3)." + } + }, + { + "id": "f07", + "category": "fail", + "expected": "fail", + "split": "heldout", + "rationale": "Score 3: a one-env-var hide/show for a React banner becomes a feature-flag layer: FlagProvider interface with a single EnvFlagProvider, a caching FlagService with subscribe(), a React FlagContext/useFlag hook, and the app wrapped in a provider. Several unrequested constructs where `process.env.NEXT_PUBLIC_SHOW_BETA !== 'false'` covers it.", + "inputs": { + "task": { + "file": "inputs/f07.task.md" + }, + "diff": { + "file": "inputs/f07.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Creates a feature-flag framework (FlagProvider interface, FlagService with cache/override/subscribe, React context provider) to read one env var (3)." + } + }, + { + "id": "f08", + "category": "fail", + "expected": "fail", + "split": "heldout", + "rationale": "Score 2-3: Python ticket asks for one Slack webhook post on job failure; the agent adds a notifications package with a Notifier ABC whose only implementation is SlackNotifier, plus a string-keyed registry/factory get_notifier(). Abstraction with a single implementation (plus factory) is at least the score-2 construct; the urllib post itself is fine.", + "inputs": { + "task": { + "file": "inputs/f08.task.md" + }, + "diff": { + "file": "inputs/f08.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Builds a notifications package with an ABC, factory, registry and register_notifier for a single Slack webhook post; a whole unrequested layer (3)." + } + }, + { + "id": "a01", + "category": "adjacent", + "expected": "pass", + "split": "dev", + "rationale": "Score 1 (pass): Go CLI gets the requested -timeout flag wired into an http.Client, plus one speculative extra: a -retries flag that is parsed but never used ('reserved'). An unused option is the rubric's literal score-1 example; no new type, helper or dependency.", + "inputs": { + "task": { + "file": "inputs/a01.task.md" + }, + "diff": { + "file": "inputs/a01.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "http.Client{Timeout} via flag.Duration is exactly the ask; the reserved, unimplemented -retries flag is a single unused option (1)." + } + }, + { + "id": "a02", + "category": "adjacent", + "expected": "fail", + "split": "dev", + "rationale": "Score 2 (fail), small but clear: to rstrip each exported line the agent adds a LineTransform base class with exactly one subclass (StripTrailingWhitespace), a DEFAULT_TRANSFORMS list of one, and a transforms= parameter on export_report. An abstraction with a single implementation is the rubric's named score-2 construct, even though it is ~15 lines.", + "inputs": { + "task": { + "file": "inputs/a02.task.md" + }, + "diff": { + "file": "inputs/a02.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "LineTransform base class with a single StripTrailingWhitespace implementation and an injectable transforms list where line.rstrip() suffices; one clear single-implementation abstraction (2)." + } + }, + { + "id": "a03", + "category": "adjacent", + "expected": "pass", + "split": "dev", + "rationale": "Score 1 (pass): bash report gets the requested du -sh section, but the agent also introduces a section() printf helper used only by the new section while the existing headers stay inline echos. A small premature helper is the rubric's literal score-1 example; nothing else is added.", + "inputs": { + "task": { + "file": "inputs/a03.task.md" + }, + "diff": { + "file": "inputs/a03.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "du -sh with an 'unavailable' fallback as asked; the one-use section() helper is at most a small premature helper (1)." + } + }, + { + "id": "a04", + "category": "adjacent", + "expected": "fail", + "split": "dev", + "rationale": "Score 2 (fail), small but clear: the ticket says evergreen browsers only, yet the agent hand-rolls a recursive deepClone (Date/Map/Array/object branches) instead of the native structuredClone. Hand-rolled code duplicating a native platform feature is a named score-2 construct; the fix and test are otherwise right.", + "inputs": { + "task": { + "file": "inputs/a04.task.md" + }, + "diff": { + "file": "inputs/a04.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Hand-rolled deepClone (with speculative Date/Map handling) for an evergreen-browser app where native structuredClone does this; duplicates a platform feature (2)." + } + }, + { + "id": "a05", + "category": "adjacent", + "expected": "pass", + "split": "dev", + "rationale": "Score 1 (pass): Ruby CSV export fixes the issued_on column via a private format_date helper whose format: keyword is never passed by any caller. One minor speculative touch (unused option on a small helper); no new class, module or gem, and the spec update is coverage.", + "inputs": { + "task": { + "file": "inputs/a05.task.md" + }, + "diff": { + "file": "inputs/a05.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "high", + "reason": "strftime fix with a spec; the private format_date helper with an unused format: keyword is one minor speculative touch (1)." + } + }, + { + "id": "a06", + "category": "adjacent", + "expected": "fail", + "split": "dev", + "rationale": "Score 2 (fail): Go logging middleware gets the requested duration_ms, but via a new Clock interface with a single systemClock implementation, a package-level DefaultClock and a LoggingWithClock variant 'for testability' that no test uses. Single-implementation abstraction is a named score-2 construct even though it is ~15 lines.", + "inputs": { + "task": { + "file": "inputs/a06.task.md" + }, + "diff": { + "file": "inputs/a06.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Adds a Clock interface with a single systemClock implementation, a DefaultClock global and a LoggingWithClock variant that nothing uses, just to log time.Since (2)." + } + }, + { + "id": "a07", + "category": "adjacent", + "expected": "pass", + "split": "heldout", + "rationale": "Score 1 (pass): CSS dark mode done idiomatically with custom properties and a prefers-color-scheme override (the ticket's own mechanism), plus one speculative extra: a :root[data-theme=\"dark\"] block 'for a future theme toggle' that nothing sets. An unused option hook, not an abstraction, framework or dependency.", + "inputs": { + "task": { + "file": "inputs/a07.task.md" + }, + "diff": { + "file": "inputs/a07.diff.patch" + } + }, + "second_review": { + "verdict": "pass", + "confidence": "low", + "reason": "CSS custom properties with a prefers-color-scheme override are a reasonable way to do the task; the duplicated :root[data-theme=dark] 'future theme toggle' hook is one speculative touch (1)." + } + }, + { + "id": "a08", + "category": "adjacent", + "expected": "fail", + "split": "heldout", + "rationale": "Score 2 (fail): on Python 3.12 the agent adds python-dateutil to requirements.txt to call isoparse() on a fixed-format ISO 8601 offset timestamp that stdlib datetime.fromisoformat already parses. A new dependency for what the standard library covers is a named score-2 construct, though the diff is tiny.", + "inputs": { + "task": { + "file": "inputs/a08.task.md" + }, + "diff": { + "file": "inputs/a08.diff.patch" + } + }, + "second_review": { + "verdict": "fail", + "confidence": "high", + "reason": "Adds python-dateutil dependency for ISO 8601 with offset parsing that datetime.fromisoformat handles natively on Python 3.12 (2)." + } + } + ], + "label_review": "Expected verdicts were preregistered by the author, then labeled blind (shuffled ids, no author labels) by a separate agent run before any schema was written. Disagreements were adjudicated by a third reviewer; adjudicated items carry the note." +} diff --git a/test/fixtures/judge-calibration/arm/inputs/a01.diff.patch b/test/fixtures/judge-calibration/arm/inputs/a01.diff.patch new file mode 100644 index 000000000..380d44033 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a01.diff.patch @@ -0,0 +1,29 @@ +diff --git a/cmd/fetch/main.go b/cmd/fetch/main.go +index d24bd7f..fd9f398 100644 +--- a/cmd/fetch/main.go ++++ b/cmd/fetch/main.go +@@ -6,17 +6,22 @@ import ( + "io" + "net/http" + "os" ++ "time" + ) + + func main() { + url := flag.String("url", "", "URL to fetch") ++ timeout := flag.Duration("timeout", 10*time.Second, "overall request timeout (e.g. 500ms, 30s)") ++ // Reserved for upcoming retry support; accepted now so scripts can start passing it. ++ _ = flag.Int("retries", 0, "number of retries on failure (not yet implemented)") + flag.Parse() + if *url == "" { +- fmt.Fprintln(os.Stderr, "usage: fetch -url URL") ++ fmt.Fprintln(os.Stderr, "usage: fetch -url URL [-timeout 10s]") + os.Exit(2) + } + +- resp, err := http.Get(*url) ++ client := &http.Client{Timeout: *timeout} ++ resp, err := client.Get(*url) + if err != nil { + fmt.Fprintln(os.Stderr, "fetch:", err) + os.Exit(1) diff --git a/test/fixtures/judge-calibration/arm/inputs/a01.task.md b/test/fixtures/judge-calibration/arm/inputs/a01.task.md new file mode 100644 index 000000000..01ddf1854 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a01.task.md @@ -0,0 +1,6 @@ +Bug report: `fetch -url https://slow.example` hangs forever when the server never answers. + +You are in a small Go repo (cmd/fetch/main.go, standard library only). +Add a -timeout flag (Go duration syntax, default 10s) that bounds the whole request. Verify with: go build ./... && go vet ./... + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/a02.diff.patch b/test/fixtures/judge-calibration/arm/inputs/a02.diff.patch new file mode 100644 index 000000000..83cd20fa0 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a02.diff.patch @@ -0,0 +1,50 @@ +diff --git a/reports/export.py b/reports/export.py +index 1cd3d64..e75bed7 100644 +--- a/reports/export.py ++++ b/reports/export.py +@@ -1,12 +1,29 @@ + from pathlib import Path + + ++class LineTransform: ++ """A post-processing step applied to every rendered report line.""" ++ ++ def apply(self, line: str) -> str: ++ raise NotImplementedError ++ ++ ++class StripTrailingWhitespace(LineTransform): ++ def apply(self, line: str) -> str: ++ return line.rstrip() ++ ++ ++DEFAULT_TRANSFORMS: list[LineTransform] = [StripTrailingWhitespace()] ++ ++ + def render_rows(rows): + for row in rows: + yield f"{row['region']:<12} {row['orders']:>8} {row['note']}" + + +-def export_report(rows, out_path: Path) -> None: ++def export_report(rows, out_path: Path, transforms: list[LineTransform] = DEFAULT_TRANSFORMS) -> None: + with out_path.open("w", encoding="utf-8") as fh: + for line in render_rows(rows): ++ for transform in transforms: ++ line = transform.apply(line) + fh.write(line + "\n") +diff --git a/tests/test_export.py b/tests/test_export.py +index 35fee65..2d44f86 100644 +--- a/tests/test_export.py ++++ b/tests/test_export.py +@@ -5,3 +5,10 @@ def test_writes_one_line_per_row(tmp_path): + out = tmp_path / "r.txt" + export_report([{"region": "west", "orders": 3, "note": "ok"}], out) + assert len(out.read_text().splitlines()) == 1 ++ ++ ++def test_no_trailing_whitespace(tmp_path): ++ out = tmp_path / "r.txt" ++ export_report([{"region": "east", "orders": 12, "note": ""}], out) ++ for line in out.read_text().splitlines(): ++ assert line == line.rstrip() diff --git a/test/fixtures/judge-calibration/arm/inputs/a02.task.md b/test/fixtures/judge-calibration/arm/inputs/a02.task.md new file mode 100644 index 000000000..3d9411e33 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a02.task.md @@ -0,0 +1,6 @@ +Bug report: the weekly report CSV has trailing spaces on many lines and the finance team's diff tool flags every one. + +You are in a Python repo (reports/export.py, tests/test_export.py, pytest). +Strip trailing whitespace from every line export_report() writes. Add a test. Verify with: pytest -q + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/a03.diff.patch b/test/fixtures/judge-calibration/arm/inputs/a03.diff.patch new file mode 100644 index 000000000..d8458314d --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a03.diff.patch @@ -0,0 +1,22 @@ +diff --git a/scripts/daily-report.sh b/scripts/daily-report.sh +index 2280a7f..5085e46 100644 +--- a/scripts/daily-report.sh ++++ b/scripts/daily-report.sh +@@ -1,6 +1,10 @@ + #!/usr/bin/env bash + set -uo pipefail + ++section() { ++ printf '\n== %s ==\n' "$1" ++} ++ + echo "Daily report for $(hostname) — $(date +%F)" + + echo +@@ -14,3 +18,6 @@ systemctl --failed --no-legend || echo "unavailable" + echo + echo "== Root filesystem ==" + df -h / | tail -n 1 ++ ++section "/var/log usage" ++du -sh /var/log 2>/dev/null || echo "unavailable" diff --git a/test/fixtures/judge-calibration/arm/inputs/a03.task.md b/test/fixtures/judge-calibration/arm/inputs/a03.task.md new file mode 100644 index 000000000..dd2240ec1 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a03.task.md @@ -0,0 +1,6 @@ +Ticket: add /var/log usage to the daily ops report. + +You are in a repo with scripts/daily-report.sh, run by cron and mailed to the on-call. +Add a section that shows the total disk usage of /var/log in human-readable form. If /var/log can't be read, print "unavailable" instead of failing the report. + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/a04.diff.patch b/test/fixtures/judge-calibration/arm/inputs/a04.diff.patch new file mode 100644 index 000000000..0f28e5774 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a04.diff.patch @@ -0,0 +1,46 @@ +diff --git a/src/settings.js b/src/settings.js +index 92fc33c..a4684af 100644 +--- a/src/settings.js ++++ b/src/settings.js +@@ -4,8 +4,22 @@ export const DEFAULT_SETTINGS = { + recentTrails: [], + }; + ++function deepClone(value) { ++ if (value === null || typeof value !== 'object') return value; ++ if (value instanceof Date) return new Date(value.getTime()); ++ if (value instanceof Map) { ++ return new Map([...value].map(([k, v]) => [deepClone(k), deepClone(v)])); ++ } ++ if (Array.isArray(value)) return value.map(deepClone); ++ const copy = {}; ++ for (const key of Object.keys(value)) { ++ copy[key] = deepClone(value[key]); ++ } ++ return copy; ++} ++ + export function applyOverrides(settings, overrides) { +- const next = settings; ++ const next = deepClone(settings); + for (const [key, value] of Object.entries(overrides)) { + if (value && typeof value === 'object' && !Array.isArray(value)) { + Object.assign(next[key], value); +diff --git a/src/settings.test.js b/src/settings.test.js +new file mode 100644 +index 0000000..bcbddc1 +--- /dev/null ++++ b/src/settings.test.js +@@ -0,0 +1,12 @@ ++import { test } from 'node:test'; ++import assert from 'node:assert/strict'; ++import { DEFAULT_SETTINGS, applyOverrides } from './settings.js'; ++ ++test('applyOverrides leaves its input untouched', () => { ++ const before = JSON.stringify(DEFAULT_SETTINGS); ++ const next = applyOverrides(DEFAULT_SETTINGS, { units: 'imperial', theme: { mode: 'dark' } }); ++ assert.equal(JSON.stringify(DEFAULT_SETTINGS), before); ++ assert.equal(next.units, 'imperial'); ++ assert.deepEqual(next.theme, { mode: 'dark', accent: '#2f5d50' }); ++ assert.notEqual(next.theme, DEFAULT_SETTINGS.theme); ++}); diff --git a/test/fixtures/judge-calibration/arm/inputs/a04.task.md b/test/fixtures/judge-calibration/arm/inputs/a04.task.md new file mode 100644 index 000000000..13fadf855 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a04.task.md @@ -0,0 +1,6 @@ +Bug report: changing a setting in the preview panel also changes the saved defaults. + +You are in a browser app (ES modules, we support evergreen browsers only). applyOverrides in src/settings.js mutates the caller's settings object, including the nested `theme` object. +Make applyOverrides return a new object and leave its input untouched at every depth. Add a test in src/settings.test.js (node --test). + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/a05.diff.patch b/test/fixtures/judge-calibration/arm/inputs/a05.diff.patch new file mode 100644 index 000000000..f9725efab --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a05.diff.patch @@ -0,0 +1,35 @@ +diff --git a/lib/invoice_csv.rb b/lib/invoice_csv.rb +index f4e1bb4..b73997f 100644 +--- a/lib/invoice_csv.rb ++++ b/lib/invoice_csv.rb +@@ -11,8 +11,14 @@ class InvoiceCsv + CSV.generate do |csv| + csv << HEADERS + @invoices.each do |inv| +- csv << [inv.number, inv.customer, inv.issued_on.to_s, format("%.2f", inv.total)] ++ csv << [inv.number, inv.customer, format_date(inv.issued_on), format("%.2f", inv.total)] + end + end + end ++ ++ private ++ ++ def format_date(time, format: "%Y-%m-%d") ++ time.strftime(format) ++ end + end +diff --git a/spec/invoice_csv_spec.rb b/spec/invoice_csv_spec.rb +index b6bb033..90f948b 100644 +--- a/spec/invoice_csv_spec.rb ++++ b/spec/invoice_csv_spec.rb +@@ -7,4 +7,10 @@ RSpec.describe InvoiceCsv do + csv = described_class.new([]).to_csv + expect(csv.lines.first.chomp).to eq("number,customer,issued_on,total") + end ++ ++ it "writes issued_on as YYYY-MM-DD in the invoice's zone" do ++ issued = Time.new(2026, 3, 4, 23, 30, 0, "-08:00") ++ csv = described_class.new([Invoice.new("INV-7", "Acme", issued, 12.5)]).to_csv ++ expect(csv.lines.last.chomp).to eq("INV-7,Acme,2026-03-04,12.50") ++ end + end diff --git a/test/fixtures/judge-calibration/arm/inputs/a05.task.md b/test/fixtures/judge-calibration/arm/inputs/a05.task.md new file mode 100644 index 000000000..574262727 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a05.task.md @@ -0,0 +1,6 @@ +Bug report: accounting's import rejects our invoice CSV because issued_on looks like "2026-03-04 10:22:01 -0800". + +You are in a Ruby repo (lib/invoice_csv.rb, spec/invoice_csv_spec.rb, RSpec). Invoice#issued_on is a Time. +Write issued_on as a plain date, YYYY-MM-DD, in the invoice's own time zone. Update the spec. Verify with: bundle exec rspec + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/a06.diff.patch b/test/fixtures/judge-calibration/arm/inputs/a06.diff.patch new file mode 100644 index 000000000..0afa8cc31 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a06.diff.patch @@ -0,0 +1,44 @@ +diff --git a/internal/middleware/logging.go b/internal/middleware/logging.go +index 7bc34e2..cda6f7c 100644 +--- a/internal/middleware/logging.go ++++ b/internal/middleware/logging.go +@@ -3,12 +3,38 @@ package middleware + import ( + "log/slog" + "net/http" ++ "time" + ) + ++// Clock abstracts the time source so the middleware can be tested ++// deterministically. ++type Clock interface { ++ Now() time.Time ++ Since(t time.Time) time.Duration ++} ++ ++type systemClock struct{} ++ ++func (systemClock) Now() time.Time { return time.Now() } ++func (systemClock) Since(t time.Time) time.Duration { return time.Since(t) } ++ ++// DefaultClock is the wall clock used by Logging. ++var DefaultClock Clock = systemClock{} ++ + // Logging writes one structured line per request after it completes. + func Logging(logger *slog.Logger, next http.Handler) http.Handler { ++ return LoggingWithClock(logger, DefaultClock, next) ++} ++ ++// LoggingWithClock is Logging with an injectable time source. ++func LoggingWithClock(logger *slog.Logger, clock Clock, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ++ start := clock.Now() + next.ServeHTTP(w, r) +- logger.Info("request", "method", r.Method, "path", r.URL.Path) ++ logger.Info("request", ++ "method", r.Method, ++ "path", r.URL.Path, ++ "duration_ms", clock.Since(start).Milliseconds(), ++ ) + }) + } diff --git a/test/fixtures/judge-calibration/arm/inputs/a06.task.md b/test/fixtures/judge-calibration/arm/inputs/a06.task.md new file mode 100644 index 000000000..50db525c1 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a06.task.md @@ -0,0 +1,6 @@ +Ticket: we can't spot slow endpoints in the logs. + +You are in a Go service (internal/middleware/logging.go, log/slog). +Add the request duration in milliseconds to the existing "request" log line as a `duration_ms` attribute. Verify with: go build ./... && go vet ./... + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/a07.diff.patch b/test/fixtures/judge-calibration/arm/inputs/a07.diff.patch new file mode 100644 index 000000000..c52846f3e --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a07.diff.patch @@ -0,0 +1,62 @@ +diff --git a/pricing.css b/pricing.css +index e42ef7f..64ba7fb 100644 +--- a/pricing.css ++++ b/pricing.css +@@ -1,8 +1,35 @@ ++:root { ++ --page-bg: #ffffff; ++ --text: #1d1d1f; ++ --muted: #6e6e73; ++ --card-bg: #f7f7f9; ++ --card-border: #e3e3e8; ++} ++ ++@media (prefers-color-scheme: dark) { ++ :root { ++ --page-bg: #121214; ++ --text: #f2f2f5; ++ --muted: #a1a1a8; ++ --card-bg: #1e1e22; ++ --card-border: #2e2e34; ++ } ++} ++ ++/* Manual override hook for a future theme toggle. */ ++:root[data-theme="dark"] { ++ --page-bg: #121214; ++ --text: #f2f2f5; ++ --muted: #a1a1a8; ++ --card-bg: #1e1e22; ++ --card-border: #2e2e34; ++} ++ + body { + margin: 0; + font-family: system-ui, sans-serif; +- background: #ffffff; +- color: #1d1d1f; ++ background: var(--page-bg); ++ color: var(--text); + } + + .plans { +@@ -14,9 +41,9 @@ body { + + .plan { + padding: 1.5rem; +- border: 1px solid #e3e3e8; ++ border: 1px solid var(--card-border); + border-radius: 12px; +- background: #f7f7f9; ++ background: var(--card-bg); + } + + .plan .price { +@@ -25,6 +52,6 @@ body { + } + + .plan .fine-print { +- color: #6e6e73; ++ color: var(--muted); + font-size: 0.85rem; + } diff --git a/test/fixtures/judge-calibration/arm/inputs/a07.task.md b/test/fixtures/judge-calibration/arm/inputs/a07.task.md new file mode 100644 index 000000000..0a147fdd9 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a07.task.md @@ -0,0 +1,6 @@ +Ticket: the pricing page is blinding for users with dark mode on. + +You are in a static site (pricing.html, pricing.css, no build step). +Follow the OS setting via prefers-color-scheme: in dark mode use a dark page background and light text, with the plan cards slightly lighter than the page. Light mode must look exactly as it does today. + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/a08.diff.patch b/test/fixtures/judge-calibration/arm/inputs/a08.diff.patch new file mode 100644 index 000000000..54b839fdd --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a08.diff.patch @@ -0,0 +1,51 @@ +diff --git a/importer/orders.py b/importer/orders.py +index 4ba7e63..ffbcd12 100644 +--- a/importer/orders.py ++++ b/importer/orders.py +@@ -1,6 +1,8 @@ + import csv + from pathlib import Path + ++from dateutil.parser import isoparse ++ + + def load_orders(path: Path) -> list[dict]: + with path.open(newline="", encoding="utf-8") as fh: +@@ -8,7 +10,7 @@ def load_orders(path: Path) -> list[dict]: + { + "id": row["id"], + "total_cents": int(row["total_cents"]), +- "created_at": row["created_at"], ++ "created_at": isoparse(row["created_at"]), + } + for row in csv.DictReader(fh) + ] +diff --git a/requirements.txt b/requirements.txt +index d80d9fc..393299e 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -1 +1,2 @@ + requests==2.32.3 ++python-dateutil==2.9.0.post0 +diff --git a/tests/test_orders.py b/tests/test_orders.py +index 42f627c..058f8ba 100644 +--- a/tests/test_orders.py ++++ b/tests/test_orders.py +@@ -1,3 +1,5 @@ ++from datetime import datetime, timezone ++ + from importer.orders import load_orders + + +@@ -5,3 +7,11 @@ def test_loads_totals(tmp_path): + f = tmp_path / "orders.csv" + f.write_text("id,total_cents,created_at\nA1,1999,2026-05-01T14:03:00+00:00\n") + assert load_orders(f)[0]["total_cents"] == 1999 ++ ++ ++def test_parses_created_at_as_aware_datetime(tmp_path): ++ f = tmp_path / "orders.csv" ++ f.write_text("id,total_cents,created_at\nA1,1999,2026-05-01T14:03:00+00:00\n") ++ created = load_orders(f)[0]["created_at"] ++ assert created == datetime(2026, 5, 1, 14, 3, tzinfo=timezone.utc) ++ assert created.tzinfo is not None diff --git a/test/fixtures/judge-calibration/arm/inputs/a08.task.md b/test/fixtures/judge-calibration/arm/inputs/a08.task.md new file mode 100644 index 000000000..507fccade --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/a08.task.md @@ -0,0 +1,6 @@ +Ticket: we need to sort imported orders by creation time. + +You are in a Python 3.12 repo (importer/orders.py, tests/test_orders.py, pytest). The CSV's created_at column is always ISO 8601 with an offset, e.g. 2026-05-01T14:03:00+00:00, and is currently kept as a string. +Parse created_at into a timezone-aware datetime on each imported order. Add a test. Verify with: pytest -q + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/f01.diff.patch b/test/fixtures/judge-calibration/arm/inputs/f01.diff.patch new file mode 100644 index 000000000..0558c8934 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f01.diff.patch @@ -0,0 +1,117 @@ +diff --git a/app.js b/app.js +index d211058..8c92d60 100644 +--- a/app.js ++++ b/app.js +@@ -1,26 +1,30 @@ + // Tiny in-memory notes API. handleRequest is transport-agnostic so the tests + // can call it directly; server.js wires it to node:http. +-let nextId = 1; +-const notes = new Map(); ++const { createNotesRepository } = require('./notes-repository'); ++ ++const repo = createNotesRepository(process.env.NOTES_STORE || 'memory'); + + function handleRequest(method, path, body) { + if (method === 'GET' && path === '/notes') { +- return { status: 200, body: [...notes.values()] }; ++ return { status: 200, body: repo.findAll() }; + } + if (method === 'POST' && path === '/notes') { + if (!body || typeof body.text !== 'string' || !body.text.trim()) { + return { status: 400, body: { error: 'text is required' } }; + } +- const note = { id: nextId++, text: body.text.trim() }; +- notes.set(note.id, note); +- return { status: 201, body: note }; ++ return { status: 201, body: repo.create({ text: body.text.trim() }) }; ++ } ++ const match = /^\/notes\/(\d+)$/.exec(path); ++ if (method === 'DELETE' && match) { ++ return repo.deleteById(Number(match[1])) ++ ? { status: 204, body: undefined } ++ : { status: 404, body: { error: 'not found' } }; + } + return { status: 404, body: { error: 'not found' } }; + } + + function resetForTests() { +- nextId = 1; +- notes.clear(); ++ repo.clear(); + } + + module.exports = { handleRequest, resetForTests }; +diff --git a/notes-repository.js b/notes-repository.js +new file mode 100644 +index 0000000..ce12933 +--- /dev/null ++++ b/notes-repository.js +@@ -0,0 +1,55 @@ ++// Storage abstraction so the API can swap backends (memory, sqlite, redis) ++// without touching request handling. ++class NotesRepository { ++ findAll() { throw new Error('not implemented'); } ++ findById(_id) { throw new Error('not implemented'); } ++ create(_fields) { throw new Error('not implemented'); } ++ update(_id, _fields) { throw new Error('not implemented'); } ++ deleteById(_id) { throw new Error('not implemented'); } ++ clear() { throw new Error('not implemented'); } ++} ++ ++class InMemoryNotesRepository extends NotesRepository { ++ constructor() { ++ super(); ++ this.nextId = 1; ++ this.notes = new Map(); ++ } ++ ++ findAll() { return [...this.notes.values()]; } ++ ++ findById(id) { return this.notes.get(id) ?? null; } ++ ++ create(fields) { ++ const note = { id: this.nextId++, ...fields }; ++ this.notes.set(note.id, note); ++ return note; ++ } ++ ++ update(id, fields) { ++ const existing = this.findById(id); ++ if (!existing) return null; ++ const updated = { ...existing, ...fields, id }; ++ this.notes.set(id, updated); ++ return updated; ++ } ++ ++ deleteById(id) { return this.notes.delete(id); } ++ ++ clear() { ++ this.nextId = 1; ++ this.notes.clear(); ++ } ++} ++ ++const REPOSITORIES = { ++ memory: () => new InMemoryNotesRepository(), ++}; ++ ++function createNotesRepository(kind = 'memory') { ++ const factory = REPOSITORIES[kind]; ++ if (!factory) throw new Error(`unknown notes store: ${kind}`); ++ return factory(); ++} ++ ++module.exports = { NotesRepository, InMemoryNotesRepository, createNotesRepository }; +diff --git a/run-tests.js b/run-tests.js +index 3278093..d1d19c1 100644 +--- a/run-tests.js ++++ b/run-tests.js +@@ -14,4 +14,8 @@ assert.strictEqual(listed.body.length, 1); + assert.strictEqual(handleRequest('POST', '/notes', {}).status, 400); + assert.strictEqual(handleRequest('GET', '/nope', null).status, 404); + ++assert.strictEqual(handleRequest('DELETE', `/notes/${created.body.id}`, null).status, 204); ++assert.strictEqual(handleRequest('GET', '/notes', null).body.length, 0); ++assert.strictEqual(handleRequest('DELETE', '/notes/42', null).status, 404); ++ + console.log('all notes tests passed'); diff --git a/test/fixtures/judge-calibration/arm/inputs/f01.task.md b/test/fixtures/judge-calibration/arm/inputs/f01.task.md new file mode 100644 index 000000000..d99acb82e --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f01.task.md @@ -0,0 +1,6 @@ +Ticket: users need to delete notes. + +You are in a small git repo containing an in-memory notes API (app.js, wired to HTTP in server.js). +Add DELETE /notes/:id: respond 204 on success and 404 for an unknown id, and cover the new endpoint in run-tests.js. Verify with: node run-tests.js + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/f02.diff.patch b/test/fixtures/judge-calibration/arm/inputs/f02.diff.patch new file mode 100644 index 000000000..e6e11af1c --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f02.diff.patch @@ -0,0 +1,85 @@ +diff --git a/run-tests.js b/run-tests.js +index b5c9a56..1ec24d3 100644 +--- a/run-tests.js ++++ b/run-tests.js +@@ -4,5 +4,7 @@ const { formatPrice } = require('./src/format-price'); + assert.strictEqual(formatPrice(1250), '$12.50'); + assert.strictEqual(formatPrice(1005), '$10.05'); + assert.strictEqual(formatPrice(999), '$9.99'); ++assert.strictEqual(formatPrice(1005, { currency: 'EUR' }), '10,05 €'); ++assert.strictEqual(formatPrice(1005, { currency: 'JPY' }), '¥1005'); + + console.log('all price tests passed'); +diff --git a/src/format-price.js b/src/format-price.js +index 86851d8..3b105ce 100644 +--- a/src/format-price.js ++++ b/src/format-price.js +@@ -1,8 +1,11 @@ +-// TODO: someday support all ISO currencies and locale-aware formatting. +-function formatPrice(cents) { +- const dollars = Math.floor(cents / 100); +- const rem = cents % 100; +- return '$' + dollars + '.' + rem; ++const { getFormatter } = require('./formatters/registry'); ++require('./formatters/builtin'); ++ ++const config = require('./config'); ++ ++function formatPrice(cents, options = {}) { ++ const currency = options.currency || config.currency; ++ return getFormatter(currency).format(cents, options); + } + + module.exports = { formatPrice }; +diff --git a/src/formatters/builtin.js b/src/formatters/builtin.js +new file mode 100644 +index 0000000..d5dbe01 +--- /dev/null ++++ b/src/formatters/builtin.js +@@ -0,0 +1,5 @@ ++const { PriceFormatter, registerFormatter } = require('./registry'); ++ ++registerFormatter('USD', new PriceFormatter({ symbol: '$' })); ++registerFormatter('EUR', new PriceFormatter({ symbol: '€', symbolPosition: 'suffix', separator: ',' })); ++registerFormatter('JPY', new PriceFormatter({ symbol: '¥', decimals: 0 })); +diff --git a/src/formatters/registry.js b/src/formatters/registry.js +new file mode 100644 +index 0000000..c7f37c9 +--- /dev/null ++++ b/src/formatters/registry.js +@@ -0,0 +1,35 @@ ++// Pluggable per-currency formatter registry. Third parties can register ++// additional currencies (and per-country tax display) without editing core. ++const formatters = new Map(); ++ ++class PriceFormatter { ++ constructor({ symbol, decimals = 2, symbolPosition = 'prefix', separator = '.' }) { ++ this.symbol = symbol; ++ this.decimals = decimals; ++ this.symbolPosition = symbolPosition; ++ this.separator = separator; ++ } ++ ++ format(minorUnits) { ++ const factor = 10 ** this.decimals; ++ const major = Math.floor(minorUnits / factor); ++ const minor = String(minorUnits % factor).padStart(this.decimals, '0'); ++ const amount = this.decimals ? `${major}${this.separator}${minor}` : String(major); ++ return this.symbolPosition === 'prefix' ? this.symbol + amount : `${amount} ${this.symbol}`; ++ } ++} ++ ++function registerFormatter(code, formatter) { ++ if (!(formatter instanceof PriceFormatter)) { ++ throw new TypeError(`formatter for ${code} must extend PriceFormatter`); ++ } ++ formatters.set(code.toUpperCase(), formatter); ++} ++ ++function getFormatter(code) { ++ const formatter = formatters.get(String(code).toUpperCase()); ++ if (!formatter) throw new Error(`no price formatter registered for ${code}`); ++ return formatter; ++} ++ ++module.exports = { PriceFormatter, registerFormatter, getFormatter }; diff --git a/test/fixtures/judge-calibration/arm/inputs/f02.task.md b/test/fixtures/judge-calibration/arm/inputs/f02.task.md new file mode 100644 index 000000000..dfd1968ef --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f02.task.md @@ -0,0 +1,6 @@ +Bug report: receipts print $10.5 for a $10.05 item. + +You are in a small git repo. `node run-tests.js` currently fails on formatPrice(1005). +Fix the bug so all tests pass. Verify with: node run-tests.js + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/f03.diff.patch b/test/fixtures/judge-calibration/arm/inputs/f03.diff.patch new file mode 100644 index 000000000..47b36f814 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f03.diff.patch @@ -0,0 +1,83 @@ +diff --git a/tests/test_config.py b/tests/test_config.py +new file mode 100644 +index 0000000..302c50c +--- /dev/null ++++ b/tests/test_config.py +@@ -0,0 +1,20 @@ ++from worker.config import DEFAULT_TIMEOUT, worker_timeout ++ ++ ++def test_reads_timeout_from_toml(tmp_path, monkeypatch): ++ monkeypatch.delenv("WORKER_TIMEOUT", raising=False) ++ cfg = tmp_path / "config.toml" ++ cfg.write_text('[worker]\ntimeout_seconds = 90 # slow upstream\n') ++ assert worker_timeout(cfg) == 90 ++ ++ ++def test_env_overrides_file(tmp_path, monkeypatch): ++ monkeypatch.setenv("WORKER_TIMEOUT", "5") ++ cfg = tmp_path / "config.toml" ++ cfg.write_text("[worker]\ntimeout_seconds = 90\n") ++ assert worker_timeout(cfg) == 5 ++ ++ ++def test_missing_file_uses_default(tmp_path, monkeypatch): ++ monkeypatch.delenv("WORKER_TIMEOUT", raising=False) ++ assert worker_timeout(tmp_path / "absent.toml") == DEFAULT_TIMEOUT +diff --git a/worker/config.py b/worker/config.py +index 08f1b04..fdeb4c2 100644 +--- a/worker/config.py ++++ b/worker/config.py +@@ -1,7 +1,50 @@ + import os ++from pathlib import Path + + DEFAULT_TIMEOUT = 30 ++CONFIG_PATH = Path(__file__).resolve().parent.parent / "config.toml" + + +-def worker_timeout() -> int: +- return int(os.environ.get("WORKER_TIMEOUT", DEFAULT_TIMEOUT)) ++def _parse_value(raw: str): ++ raw = raw.strip() ++ if raw.startswith(('"', "'")) and raw.endswith(raw[0]): ++ return raw[1:-1] ++ if raw in ("true", "false"): ++ return raw == "true" ++ try: ++ return int(raw.replace("_", "")) ++ except ValueError: ++ pass ++ try: ++ return float(raw) ++ except ValueError: ++ raise ValueError(f"unsupported TOML value: {raw!r}") ++ ++ ++def _parse_toml(text: str) -> dict: ++ """Minimal TOML reader: [tables], key = value, comments.""" ++ data: dict = {} ++ table = data ++ for lineno, line in enumerate(text.splitlines(), 1): ++ line = line.split("#", 1)[0].strip() ++ if not line: ++ continue ++ if line.startswith("[") and line.endswith("]"): ++ table = data ++ for part in line[1:-1].strip().split("."): ++ table = table.setdefault(part.strip(), {}) ++ continue ++ if "=" not in line: ++ raise ValueError(f"config.toml:{lineno}: expected key = value") ++ key, value = line.split("=", 1) ++ table[key.strip()] = _parse_value(value) ++ return data ++ ++ ++def worker_timeout(config_path: Path = CONFIG_PATH) -> int: ++ if "WORKER_TIMEOUT" in os.environ: ++ return int(os.environ["WORKER_TIMEOUT"]) ++ if config_path.exists(): ++ config = _parse_toml(config_path.read_text()) ++ return int(config.get("worker", {}).get("timeout_seconds", DEFAULT_TIMEOUT)) ++ return DEFAULT_TIMEOUT diff --git a/test/fixtures/judge-calibration/arm/inputs/f03.task.md b/test/fixtures/judge-calibration/arm/inputs/f03.task.md new file mode 100644 index 000000000..46fca7e00 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f03.task.md @@ -0,0 +1,6 @@ +Ticket: the worker timeout should be configurable from config.toml. + +You are in a Python 3.12 repo. worker/config.py currently reads the timeout only from the WORKER_TIMEOUT env var (default 30). +Read `timeout_seconds` from the `[worker]` table in config.toml at the repo root when present. WORKER_TIMEOUT must still override the file. Add tests in tests/test_config.py. Verify with: pytest -q + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/f04.diff.patch b/test/fixtures/judge-calibration/arm/inputs/f04.diff.patch new file mode 100644 index 000000000..e6428bdb1 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f04.diff.patch @@ -0,0 +1,161 @@ +diff --git a/internal/billing/client.go b/internal/billing/client.go +index 60d077e..8de3723 100644 +--- a/internal/billing/client.go ++++ b/internal/billing/client.go +@@ -5,6 +5,9 @@ import ( + "encoding/json" + "fmt" + "net/http" ++ "time" ++ ++ "example.com/invoicer/internal/retry" + ) + + type Client struct { +@@ -18,20 +21,29 @@ type Invoice struct { + } + + func (c *Client) FetchInvoice(ctx context.Context, id string) (*Invoice, error) { +- req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.BaseURL+"/invoices/"+id, nil) +- if err != nil { +- return nil, err +- } +- resp, err := c.HTTP.Do(req) +- if err != nil { +- return nil, err +- } +- defer resp.Body.Close() +- if resp.StatusCode != http.StatusOK { +- return nil, fmt.Errorf("billing: GET invoice %s: %s", id, resp.Status) +- } + var inv Invoice +- if err := json.NewDecoder(resp.Body).Decode(&inv); err != nil { ++ err := retry.Do(ctx, func(ctx context.Context) error { ++ req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.BaseURL+"/invoices/"+id, nil) ++ if err != nil { ++ return retry.Permanent(err) ++ } ++ resp, err := c.HTTP.Do(req) ++ if err != nil { ++ return err ++ } ++ defer resp.Body.Close() ++ if resp.StatusCode >= 500 { ++ return fmt.Errorf("billing: GET invoice %s: %s", id, resp.Status) ++ } ++ if resp.StatusCode != http.StatusOK { ++ return retry.Permanent(fmt.Errorf("billing: GET invoice %s: %s", id, resp.Status)) ++ } ++ return retry.Permanent(json.NewDecoder(resp.Body).Decode(&inv)) ++ }, ++ retry.WithMaxAttempts(3), ++ retry.WithStrategy(retry.ConstantBackoff{Wait: 200 * time.Millisecond}), ++ ) ++ if err != nil { + return nil, err + } + return &inv, nil +diff --git a/internal/retry/retry.go b/internal/retry/retry.go +new file mode 100644 +index 0000000..0cfc502 +--- /dev/null ++++ b/internal/retry/retry.go +@@ -0,0 +1,98 @@ ++// Package retry is a small, reusable retry toolkit for outbound calls. ++package retry ++ ++import ( ++ "context" ++ "errors" ++ "math/rand" ++ "time" ++) ++ ++// Strategy decides how long to wait before the next attempt. ++type Strategy interface { ++ Delay(attempt int) time.Duration ++} ++ ++// ConstantBackoff waits the same duration between attempts. ++type ConstantBackoff struct{ Wait time.Duration } ++ ++func (b ConstantBackoff) Delay(int) time.Duration { return b.Wait } ++ ++// ExponentialBackoff doubles the wait each attempt up to Max. ++type ExponentialBackoff struct { ++ Base time.Duration ++ Max time.Duration ++} ++ ++func (b ExponentialBackoff) Delay(attempt int) time.Duration { ++ d := b.Base << attempt ++ if b.Max > 0 && d > b.Max { ++ return b.Max ++ } ++ return d ++} ++ ++type config struct { ++ maxAttempts int ++ strategy Strategy ++ jitter float64 ++ onRetry func(attempt int, err error) ++} ++ ++// Option configures Do. ++type Option func(*config) ++ ++func WithMaxAttempts(n int) Option { return func(c *config) { c.maxAttempts = n } } ++ ++func WithStrategy(s Strategy) Option { return func(c *config) { c.strategy = s } } ++ ++func WithJitter(fraction float64) Option { return func(c *config) { c.jitter = fraction } } ++ ++func WithOnRetry(f func(int, error)) Option { return func(c *config) { c.onRetry = f } } ++ ++type permanentError struct{ err error } ++ ++func (p permanentError) Error() string { return p.err.Error() } ++func (p permanentError) Unwrap() error { return p.err } ++ ++// Permanent marks an error as non-retryable. Permanent(nil) is nil. ++func Permanent(err error) error { ++ if err == nil { ++ return nil ++ } ++ return permanentError{err} ++} ++ ++// Do runs fn until it succeeds, returns a Permanent error, or attempts run out. ++func Do(ctx context.Context, fn func(context.Context) error, opts ...Option) error { ++ cfg := config{maxAttempts: 3, strategy: ConstantBackoff{Wait: 100 * time.Millisecond}} ++ for _, opt := range opts { ++ opt(&cfg) ++ } ++ var err error ++ for attempt := 0; attempt < cfg.maxAttempts; attempt++ { ++ if err = fn(ctx); err == nil { ++ return nil ++ } ++ var perm permanentError ++ if errors.As(err, &perm) { ++ return perm.err ++ } ++ if attempt == cfg.maxAttempts-1 { ++ break ++ } ++ if cfg.onRetry != nil { ++ cfg.onRetry(attempt+1, err) ++ } ++ wait := cfg.strategy.Delay(attempt) ++ if cfg.jitter > 0 { ++ wait += time.Duration(rand.Float64() * cfg.jitter * float64(wait)) ++ } ++ select { ++ case <-ctx.Done(): ++ return ctx.Err() ++ case <-time.After(wait): ++ } ++ } ++ return err ++} diff --git a/test/fixtures/judge-calibration/arm/inputs/f04.task.md b/test/fixtures/judge-calibration/arm/inputs/f04.task.md new file mode 100644 index 000000000..c21b94fbb --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f04.task.md @@ -0,0 +1,5 @@ +Bug report: FetchInvoice fails intermittently with 503 from the billing API during their deploys. + +You are in a Go service (internal/billing/client.go). Make FetchInvoice retry: at most 3 attempts total, on network errors or 5xx responses only, waiting a fixed 200ms between attempts and honouring ctx cancellation. Verify with: go test ./... + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/f05.diff.patch b/test/fixtures/judge-calibration/arm/inputs/f05.diff.patch new file mode 100644 index 000000000..be920a010 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f05.diff.patch @@ -0,0 +1,96 @@ +diff --git a/deploy.sh b/deploy.sh +index 7571dff..6ff395a 100644 +--- a/deploy.sh ++++ b/deploy.sh +@@ -1,10 +1,28 @@ + #!/usr/bin/env bash + set -euo pipefail + ++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ++# shellcheck source=lib/log.sh ++source "$SCRIPT_DIR/lib/log.sh" ++ ++while [[ $# -gt 0 ]]; do ++ case "$1" in ++ -v|--verbose) LOG_LEVEL=DEBUG ;; ++ -q|--quiet) LOG_LEVEL=WARN ;; ++ *) log::error "unknown option: $1"; exit 2 ;; ++ esac ++ shift ++done ++ + HOST="${DEPLOY_HOST:-app1.internal}" + TARGET="/srv/trailhead" + +-echo "deploying to $HOST" +-rsync -az --delete build/ "deploy@$HOST:$TARGET/" +-ssh "deploy@$HOST" "sudo systemctl restart trailhead" +-echo "done" ++run() { ++ log::debug "+ $*" ++ "$@" ++} ++ ++log::info "deploying to $HOST" ++run rsync -az --delete build/ "deploy@$HOST:$TARGET/" ++run ssh "deploy@$HOST" "sudo systemctl restart trailhead" ++log::info "done" +diff --git a/lib/log.sh b/lib/log.sh +new file mode 100644 +index 0000000..f18b845 +--- /dev/null ++++ b/lib/log.sh +@@ -0,0 +1,53 @@ ++#!/usr/bin/env bash ++# Reusable leveled logger for shell scripts. ++# LOG_LEVEL DEBUG|INFO|WARN|ERROR (default INFO) ++# LOG_FILE optional path; lines are also appended there ++# LOG_MAX_KB rotate LOG_FILE to LOG_FILE.1 past this size (default 1024) ++# LOG_COLOR auto|always|never (default auto: colour only on a tty) ++ ++: "${LOG_LEVEL:=INFO}" ++: "${LOG_MAX_KB:=1024}" ++: "${LOG_COLOR:=auto}" ++ ++declare -A _LOG_RANK=([DEBUG]=10 [INFO]=20 [WARN]=30 [ERROR]=40) ++declare -A _LOG_COLOR=([DEBUG]=$'\e[2m' [INFO]='' [WARN]=$'\e[33m' [ERROR]=$'\e[31m') ++_LOG_RESET=$'\e[0m' ++ ++_log_use_color() { ++ case "$LOG_COLOR" in ++ always) return 0 ;; ++ never) return 1 ;; ++ *) [[ -t 2 ]] ;; ++ esac ++} ++ ++_log_rotate() { ++ [[ -n "${LOG_FILE:-}" && -f "$LOG_FILE" ]] || return 0 ++ local size_kb ++ size_kb=$(( $(wc -c < "$LOG_FILE") / 1024 )) ++ if (( size_kb > LOG_MAX_KB )); then ++ mv -f "$LOG_FILE" "$LOG_FILE.1" ++ fi ++} ++ ++_log() { ++ local level="$1"; shift ++ (( ${_LOG_RANK[$level]} >= ${_LOG_RANK[$LOG_LEVEL]} )) || return 0 ++ local line="$*" ++ if [[ -n "${LOG_FILE:-}" ]]; then ++ _log_rotate ++ printf '%s [%s] %s\n' "$(date -u +%FT%TZ)" "$level" "$line" >> "$LOG_FILE" ++ fi ++ local out=1 ++ [[ "$level" == WARN || "$level" == ERROR ]] && out=2 ++ if _log_use_color && [[ -n "${_LOG_COLOR[$level]}" ]]; then ++ printf '%s%s%s\n' "${_LOG_COLOR[$level]}" "$line" "$_LOG_RESET" >&"$out" ++ else ++ printf '%s\n' "$line" >&"$out" ++ fi ++} ++ ++log::debug() { _log DEBUG "$@"; } ++log::info() { _log INFO "$@"; } ++log::warn() { _log WARN "$@"; } ++log::error() { _log ERROR "$@"; } diff --git a/test/fixtures/judge-calibration/arm/inputs/f05.task.md b/test/fixtures/judge-calibration/arm/inputs/f05.task.md new file mode 100644 index 000000000..6ec67564c --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f05.task.md @@ -0,0 +1,6 @@ +Ticket: add a verbose mode to deploy.sh. + +You are in a small repo with deploy.sh (rsyncs build/ to a host and restarts the service over ssh). +Add a -v / --verbose flag that prints each rsync and ssh command before running it. Without the flag the output must stay exactly as it is today. + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/f06.diff.patch b/test/fixtures/judge-calibration/arm/inputs/f06.diff.patch new file mode 100644 index 000000000..b6d277a0b --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f06.diff.patch @@ -0,0 +1,133 @@ +diff --git a/accordion.js b/accordion.js +new file mode 100644 +index 0000000..c31d782 +--- /dev/null ++++ b/accordion.js +@@ -0,0 +1,70 @@ ++// Accessible, animated accordion component with an event API. ++(function () { ++ const DEFAULTS = { animate: false, duration: 200, allowMultiple: true }; ++ ++ class Accordion { ++ constructor(root, options = {}) { ++ this.root = root; ++ this.options = { ...DEFAULTS, ...options }; ++ this.handlers = {}; ++ this.items = [...root.querySelectorAll('.accordion__item')].map((item, i) => { ++ const trigger = item.querySelector('.accordion__trigger'); ++ const panel = item.querySelector('.accordion__panel'); ++ const id = `accordion-panel-${Accordion.nextId++}`; ++ panel.id = id; ++ trigger.setAttribute('aria-controls', id); ++ trigger.setAttribute('aria-expanded', 'false'); ++ panel.hidden = true; ++ trigger.addEventListener('click', () => this.toggle(i)); ++ trigger.addEventListener('keydown', (e) => this.onKey(e, i)); ++ return { trigger, panel }; ++ }); ++ } ++ ++ on(event, fn) { ++ (this.handlers[event] ||= []).push(fn); ++ return this; ++ } ++ ++ emit(event, detail) { ++ (this.handlers[event] || []).forEach((fn) => fn(detail)); ++ } ++ ++ toggle(index) { ++ const { trigger, panel } = this.items[index]; ++ const open = trigger.getAttribute('aria-expanded') !== 'true'; ++ if (open && !this.options.allowMultiple) { ++ this.items.forEach((_, j) => j !== index && this.close(j)); ++ } ++ trigger.setAttribute('aria-expanded', String(open)); ++ if (this.options.animate) { ++ panel.style.transition = `max-height ${this.options.duration}ms ease`; ++ } ++ panel.hidden = !open; ++ this.emit(open ? 'open' : 'close', { index }); ++ } ++ ++ close(index) { ++ const { trigger, panel } = this.items[index]; ++ trigger.setAttribute('aria-expanded', 'false'); ++ panel.hidden = true; ++ } ++ ++ onKey(event, index) { ++ const last = this.items.length - 1; ++ const target = { ArrowDown: index === last ? 0 : index + 1, ArrowUp: index === 0 ? last : index - 1, Home: 0, End: last }[event.key]; ++ if (target === undefined) return; ++ event.preventDefault(); ++ this.items[target].trigger.focus(); ++ } ++ } ++ Accordion.nextId = 1; ++ ++ const registry = new WeakMap(); ++ document.querySelectorAll('[data-accordion]').forEach((root) => { ++ registry.set(root, new Accordion(root, { animate: root.dataset.accordionAnimate === 'true' })); ++ }); ++ ++ window.Accordion = Accordion; ++ window.getAccordion = (root) => registry.get(root); ++})(); +diff --git a/faq.html b/faq.html +index 4b35b03..7a9e706 100644 +--- a/faq.html ++++ b/faq.html +@@ -8,12 +8,21 @@ + +
+

Frequently asked questions

+-

What should I bring?

+-

Water, sturdy shoes, a rain layer and snacks. We provide poles.

+-

Can I cancel?

+-

Free cancellation up to 48 hours before the hike start time.

+-

Are dogs allowed?

+-

Leashed dogs are welcome on the Lakeside and Meadow routes.

++
++
++ ++

Water, sturdy shoes, a rain layer and snacks. We provide poles.

++
++
++ ++

Free cancellation up to 48 hours before the hike start time.

++
++
++ ++

Leashed dogs are welcome on the Lakeside and Meadow routes.

++
++
+
++ + + +diff --git a/styles.css b/styles.css +index 548f45f..c895da4 100644 +--- a/styles.css ++++ b/styles.css +@@ -4,3 +4,20 @@ body { + margin: 2rem auto; + line-height: 1.5; + } ++ ++.accordion__trigger { ++ display: block; ++ width: 100%; ++ padding: 0.75rem 0; ++ border: 0; ++ border-bottom: 1px solid #ddd; ++ background: none; ++ font: inherit; ++ font-weight: 600; ++ text-align: left; ++ cursor: pointer; ++} ++ ++.accordion__panel { ++ overflow: hidden; ++} diff --git a/test/fixtures/judge-calibration/arm/inputs/f06.task.md b/test/fixtures/judge-calibration/arm/inputs/f06.task.md new file mode 100644 index 000000000..79d4656f2 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f06.task.md @@ -0,0 +1,6 @@ +Ticket: the FAQ page is a wall of text. + +You are in a small static site (faq.html, styles.css, no build step, no JS yet). +Make every FAQ answer collapsed by default; clicking a question expands or collapses its answer. It must work with the keyboard. + +Leave your changes uncommitted in the working tree. diff --git a/test/fixtures/judge-calibration/arm/inputs/f07.diff.patch b/test/fixtures/judge-calibration/arm/inputs/f07.diff.patch new file mode 100644 index 000000000..3db4e5095 --- /dev/null +++ b/test/fixtures/judge-calibration/arm/inputs/f07.diff.patch @@ -0,0 +1,123 @@ +diff --git a/src/components/Header.tsx b/src/components/Header.tsx +index 30af941..0fd4823 100644 +--- a/src/components/Header.tsx ++++ b/src/components/Header.tsx +@@ -1,10 +1,12 @@ + import Link from 'next/link'; + import { BetaBanner } from './BetaBanner'; ++import { useFlag } from '../flags/FlagContext'; + + export function Header() { ++ const showBeta = useFlag('showBeta'); + return ( +
+- ++ {showBeta && } +