diff --git a/docs/evals/headroom-diagnosis-2026-10.md b/docs/evals/headroom-diagnosis-2026-10.md index 6b5083512..64977e41c 100644 --- a/docs/evals/headroom-diagnosis-2026-10.md +++ b/docs/evals/headroom-diagnosis-2026-10.md @@ -83,3 +83,16 @@ None is near its own budget (plan-review-report peaks at 46% of 600 s). The fix is a refreshed duration seed so the planner packs slice 11 within the 540 s slice budget (lane F / integrator). review-army-perf and plan-design-review-plan-mode need no work. + +## plan-eng-multi-finding-batching: 975 s in census 37228573062 (GSTA-23 branch) + +The red trial ran 975 s of its 1,500 s armed session (65%), against 313-409 s +in the nine earlier censuses (37162480720 … 37198445662). The wall is a +consequence of the detector miss, not slower work: the runner stops +collecting once the counter credits FLOOR = 3 review questions (normally +by D3, about 5-7 minutes). The counter credited none, because the report +declared its target mid-line ("… by the user. Review target (fixed): +`PLAN.md` …"), so collection ran on through D10 and the completed report. +With the target-field fix the same capture is credited at D5 (521 s). +Replay: `bun test test/detector-corpus-plan-eng-multi-finding-batching.test.ts`. +No budget or kind change. diff --git a/test/detector-corpus-plan-eng-multi-finding-batching.test.ts b/test/detector-corpus-plan-eng-multi-finding-batching.test.ts new file mode 100644 index 000000000..5f3c1cb5b --- /dev/null +++ b/test/detector-corpus-plan-eng-multi-finding-batching.test.ts @@ -0,0 +1,72 @@ +/** + * Replays stored plan-eng-multi-finding-batching census captures through the + * case's real review-question counter (createEngBatchingIssueCounter), with + * the saved plan rebuilt from the session's own Write/Edit calls as it stood + * when each question was answered. The case passes at FLOOR = 3 credited + * review questions; the floor is the paid test's and is not changed here. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { engSetupAUQ } from './helpers/claude-pty-runner'; +import { createEngBatchingIssueCounter } from './helpers/eng-seeded-coverage'; +import { nativePlanCallFingerprint } from './helpers/pty/auq'; +import type { NativePlanQuestionCall } from './helpers/plan-count-transcript'; + +const DIR = path.join(import.meta.dir, 'fixtures/detector-corpora/plan-eng-multi-finding-batching'); +const FLOOR = 3; +const files = fs.readdirSync(DIR).filter(name => name !== 'inventory.json' && name.endsWith('.json')).sort(); +const load = (name: string) => JSON.parse(fs.readFileSync(path.join(DIR, name), 'utf8')); + +interface StoredCall { sessionId: string; toolUseId: string; answeredAt: string; answer: string; + question: NativePlanQuestionCall['questions'][number] } +type PlanEvent = { at: string; write?: string; old?: string; new?: string; all?: boolean }; + +function planAt(events: PlanEvent[], at: number): string { + let plan = ''; + for (const e of events) { + if (Date.parse(e.at) > at) continue; + if (e.write !== undefined) plan = e.write; + else if (plan.includes(e.old!)) plan = e.all ? plan.split(e.old!).join(e.new!) : plan.replace(e.old!, () => e.new!); + } + return plan; +} + +/** Credited review questions, as the paid runner asks the counter once per answered call. */ +function credited(input: { calls: StoredCall[]; planEvents: PlanEvent[] }): number { + let at = 0; + const counter = createEngBatchingIssueCounter(() => planAt(input.planEvents, at), engSetupAUQ); + const calls: NativePlanQuestionCall[] = input.calls.map(c => ({ sessionId: c.sessionId, toolUseId: c.toolUseId, + questions: [c.question], answered: true, failed: false, answers: { [c.question.question]: c.answer }, + unansweredQuestionIndices: [], answeredAt: c.answeredAt }) as NativePlanQuestionCall); + return calls.filter((call, i) => { + at = Date.parse(call.answeredAt!); + return counter.isReviewAUQ(nativePlanCallFingerprint(call, 0, false), calls.slice(0, i)); + }).length; +} + +describe('plan-eng-multi-finding-batching detector corpus', () => { + test('inventory names every census entry', () => { + const inventory: Array<{ entry: string | null }> = JSON.parse(fs.readFileSync(path.join(DIR, 'inventory.json'), 'utf8')); + expect(inventory.flatMap(row => row.entry ? [row.entry] : []).sort()) + .toEqual(files.filter(name => !load(name).derived)); + }); + + for (const name of files) { + const entry = load(name); + test(`${name}: the counter ${entry.expected === 'pass' ? 'credits' : 'refuses'} it (census ${entry.census_outcome})`, () => { + expect(credited(entry.input) >= FLOOR ? 'pass' : 'refuse').toBe(entry.expected); + }); + } + + test('37228573062: the target must open a sentence, and a plan identity is still required', () => { + const entry = load('37228573062-t1-fail.json'); + const declaration = 'Review target (fixed): `PLAN.md`'; + const edit = (mutate: (text: string) => string) => ({ ...entry.input, + planEvents: entry.input.planEvents.map((e: PlanEvent) => e.write === undefined ? e : { ...e, write: mutate(e.write) }) }); + expect(entry.input.planEvents[0].write).toContain(`Report destination requested by the user. ${declaration}`); + expect(credited(entry.input)).toBeGreaterThanOrEqual(FLOOR); + expect(credited(edit(t => t.replace(declaration, `the reviewer read the ${declaration}`)))).toBe(0); + expect(credited(edit(t => t.replace(declaration, 'Review target (fixed): `NOTES.md`')))).toBe(0); + }); +}); diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37162480720-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37162480720-t1-pass.json new file mode 100644 index 000000000..86b41776a --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37162480720-t1-pass.json @@ -0,0 +1,111 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "5553dbf327080983", + "source": { + "run": "37162480720", + "sha": "a818a991fb63699b6af142d5022d781fb418b2ef", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37162480720-1-eval-slices-1/plan-eng-review-1791070951503-gDx2l5" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "54bbb594-bd16-449c-b2ee-724def405c1b", + "toolUseId": "toolu_01AGyoSfTck7SrmtMJxU4Qzr", + "answeredAt": "2026-10-03T23:45:30.589Z", + "answer": "Library hooks + custom backoff function (recommended)", + "question": { + "question": "D1 — Retry mechanism: reuse the job library's retry hooks or build a custom inline scheduler?\nProject/branch/task: main, \"Add background job retry framework\" plan review; repo holds only PLAN.md, so this is judged on the plan text.\nELI10: The plan rebuilds retry scheduling inside every worker even though the job library already has retry hooks that do the same thing. The stated reason is wanting control over the backoff curve, but libraries in this class let you hand them your own backoff function, so you keep the curve and drop the rebuild. A custom scheduler also has to re-solve things the library already handles: remembering how many attempts happened, surviving a worker crash mid-wait, and showing up in the library's dashboards.\nStakes if we pick wrong: a hand-rolled scheduler means five places where attempt counting or delay math can drift, retries lost when a worker restarts, and a retry framework nobody else on the team recognizes. Picking the library when its hooks turn out to lack a needed feature costs a day of rework, which option C removes.\nRecommendation: A because the plan's only reason to leave the library (\"full control over the curve\") is met by a custom backoff function, so the rebuild buys nothing and costs persistence, observability and five maintenance sites.\nCompleteness: A=9/10, B=5/10, C=n/a (investigation only, approves nothing)\nPros / cons:\nA) Library hooks + custom backoff function (recommended)\n ✅ Attempt count, delay persistence and crash recovery stay library-owned; nothing to re-implement per worker\n ✅ The backoff curve is still fully yours: one function returns the delay for attempt N (human: ~1 day / CC: ~20 min)\n ✅ Finding 2 (five copy-pasted envelopes) disappears: the envelope is config, not worker code\n ❌ Depends on the library's hook API actually accepting a custom backoff function; not verified here because the library is not in this repo\nB) Custom inline scheduler per worker (as planned)\n ✅ Zero dependency on the library's hook surface; works even if its hooks are rigid\n ✅ Lets each worker tune its curve independently without touching queue config\n ❌ Each worker must own attempt tracking, timers and restart recovery; a crash mid-backoff silently drops the retry unless you also build persistence (human: ~3 days / CC: ~1 h)\n ❌ Keeps the five-copy duplication alive and puts the retry path outside the library's metrics and dashboards\nC) Investigate the hook API first, then decide\n ✅ Replaces the unverified assumption in A with a 5-minute read of the library's retry/backoff hook docs or source\n ✅ Costs almost nothing and makes the A/B call on evidence instead of plan text\n ❌ Defers the decision; nothing in this plan can be approved until the read-back is done\nNet: trade an unverified hook-API assumption (A) against a known rebuild of persistence, observability and five maintenance sites (B); C buys the evidence for a few minutes.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hooks + custom backoff function (recommended)", + "description": "Reuse the job library's built-in retry hooks; supply one custom backoff function for the curve. Library owns attempt count, delay persistence and crash recovery. Makes the five-copy envelope duplication moot. Human ~1 day / CC ~20 min. Risk: assumes the hook API accepts a custom backoff function (unverified, library not in repo). Completeness 9/10." + }, + { + "label": "Custom inline scheduler per worker (as planned)", + "description": "Build the exponential-backoff scheduler inline in each of ~5 workers as written in PLAN.md:7-9. Each worker owns attempt tracking, timers and restart recovery; retry path sits outside library metrics. Keeps the five copies pending a separate dedupe decision. Human ~3 days / CC ~1 h. Completeness 5/10." + }, + { + "label": "Investigate the hook API first", + "description": "Bounded read of the library's retry/backoff hook API (docs or source, human ~1 h / CC ~5 min) before choosing. Approves nothing; R1 and every dependent row stay pending until the read-back is reported." + } + ] + } + }, + { + "sessionId": "54bbb594-bd16-449c-b2ee-724def405c1b", + "toolUseId": "toolu_01HvPRvQBD8KTcgtMQpwKVU1", + "answeredAt": "2026-10-03T23:46:55.937Z", + "answer": "At-least-once with idempotency key (recommended)", + "question": { + "question": "D2 — Webhook delivery: what guarantee do receivers get once `processWebhookJob()` can retry?\nProject/branch/task: main, \"Add background job retry framework\"; R1 settled on the library's retry hooks, now the webhook flow's semantics.\nELI10: Today a webhook is sent at most once: if it fails, it is gone, but a receiver never sees the same event twice. The moment you add retries, a request that timed out after reaching the receiver gets sent again, and the receiver sees a duplicate. You have to pick on purpose: embrace duplicates and give receivers a key to spot them, only retry when you are sure the request never left, or keep webhooks out of the retry framework entirely.\nStakes if we pick wrong: silently flipping to at-least-once means customers' systems double-charge, double-notify or double-create records with no way to detect it. Keeping at-most-once means the common failure (a 5xx or a slow receiver) still loses the event.\nRecommendation: A because at-least-once with a documented idempotency key is the standard webhook contract and the only option that both recovers lost deliveries and lets receivers protect themselves.\nCompleteness: A=9/10, B=6/10, C=4/10\nPros / cons:\nA) At-least-once with idempotency key (recommended)\n ✅ Recovers deliveries lost to 5xx and read timeouts, which is most of what a retry framework exists for\n ✅ Receivers get a stable key per event and a documented \"may deliver more than once\" contract they can dedupe on (human: ~1.5 days / CC: ~30 min)\n ❌ Changes the receiver-facing contract; any receiver that does not dedupe will see duplicates until they do\nB) Retry only pre-send failures\n ✅ Preserves the current at-most-once guarantee exactly; no receiver sees a duplicate\n ✅ Still recovers the cheap failures (connection refused, DNS, TLS handshake) with no contract change (human: ~1 day / CC: ~20 min)\n ❌ The most common failures (5xx, slow receiver) stay lost, and classifying \"provably never sent\" per HTTP client is brittle\nC) Exclude the webhook flow from retries\n ✅ Zero contract risk and almost zero work; the other four workers still get retries (human: ~0 / CC: ~2 min)\n ✅ Buys time to decide the webhook contract with receivers separately\n ❌ The webhook flow gets no reliability gain from this change, and the plan's rewrite of `processWebhookJob()` would need to be dropped\nNet: trade a documented contract change that receivers must honor (A) against preserving at-most-once at the cost of still losing the common failures (B, C).", + "header": "Webhook delivery", + "multiSelect": false, + "options": [ + { + "label": "At-least-once with idempotency key (recommended)", + "description": "Webhook retries enabled through the library hooks; every delivery carries a stable idempotency key (job id + event id) header and each attempt is recorded (attempt number, response). Receiver-facing docs updated to \"at-least-once, dedupe on the key\". Human ~1.5 days / CC ~30 min. Completeness 9/10. R3 regression contract stays pending." + }, + { + "label": "Retry only pre-send failures", + "description": "Retry only when the request provably never left (connection refused, DNS, TLS handshake failure); read timeouts and 5xx are terminal. Preserves at-most-once. Each attempt recorded. Human ~1 day / CC ~20 min. Completeness 6/10: 5xx and slow receivers still lose the event. R3 stays pending." + }, + { + "label": "Exclude the webhook flow from retries", + "description": "Keep `processWebhookJob()` on its current no-retry path; the other four workers adopt the library hooks. No contract change, no reliability gain for webhooks. Human ~0 / CC ~2 min. Completeness 4/10. R3 stays pending." + } + ] + } + }, + { + "sessionId": "54bbb594-bd16-449c-b2ee-724def405c1b", + "toolUseId": "toolu_01H8or2ZvqSURzutGrFcrb2p", + "answeredAt": "2026-10-03T23:47:48.921Z", + "answer": "Full jitter (recommended)", + "question": { + "question": "D3 — Jitter: randomize retry delays, or keep the pure exponential curve?\nProject/branch/task: main, \"Add background job retry framework\"; the curve now lives in one custom backoff function supplied to the library hooks (D1).\nELI10: If a downstream service goes down for a minute, hundreds of jobs fail at the same moment. With a pure exponential curve they all wake up and retry at the same instants, hammering the service just as it recovers. Jitter adds randomness to each delay so the retries spread out instead of arriving as a wave.\nStakes if we pick wrong: without jitter the first retry wave after an outage can knock the dependency over again and extend the incident; with jitter the only cost is that a given retry may fire a little earlier or later than the textbook curve.\nRecommendation: A because full jitter is the documented safe default for background jobs, it is a one-line change inside the backoff function, and every real-world retry-storm bug found in research traced back to its absence.\nCompleteness: A=9/10, B=8/10, C=4/10\nPros / cons:\nA) Full jitter (recommended)\n ✅ Spreads retries across the whole window, the strongest protection against a lockstep wave after a shared outage\n ✅ One line inside the custom backoff function; no new state, no new dependency (human: ~1 h incl. test / CC: ~3 min)\n ❌ A retry can fire almost immediately after a failure (random lower bound is 0), which can look surprising in logs\nB) Equal jitter\n ✅ Guarantees a minimum wait of half the computed delay, so no retry fires \"too soon\"\n ✅ Still breaks up the lockstep wave, just over half the window (human: ~1 h incl. test / CC: ~3 min)\n ❌ Weaker spread than full jitter; only worth it if a hard minimum wait is a real requirement here, and none is stated\nC) No jitter (as planned)\n ✅ Delays are exactly predictable, which makes the curve easy to reason about and test\n ✅ Zero extra code beyond the curve itself\n ❌ Every job that failed in the same outage retries at the same instants; this is the classic retry-storm bug reported against real queue libraries\nNet: a few minutes of code (A or B) against a known outage amplifier (C); A unless a hard minimum wait is a stated requirement.", + "header": "Jitter", + "multiSelect": false, + "options": [ + { + "label": "Full jitter (recommended)", + "description": "Custom backoff function returns random(0, base * 2^attempt). One line plus a test asserting delays stay within [0, computed]. Human ~1 h / CC ~3 min. Completeness 9/10. R6 cap, R7 max attempts/dead-letter stay pending." + }, + { + "label": "Equal jitter", + "description": "Custom backoff function returns computed/2 + random(0, computed/2). Guaranteed minimum wait. One line plus a bounds test. Human ~1 h / CC ~3 min. Completeness 8/10. R6, R7 stay pending." + }, + { + "label": "No jitter (as planned)", + "description": "Custom backoff function returns base * 2^attempt exactly. Lockstep retries after a shared outage are accepted. Human ~0 / CC ~0. Completeness 4/10. R6, R7 stay pending." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-03T23:45:08.602Z", + "write": "# Eng review: Add background job retry framework\n\nReview target (fixed): `PLAN.md` in `` (branch `main`, commit c03143b), titled \"Plan: Add background job retry framework\".\nReport file: this file, at the path the user requested.\nReviewer: Claude (/plan-eng-review), session 689-1791070937-5a52bce9, 2026-10-03.\n\n## Original plan (unchanged copy of PLAN.md)\n\n```markdown\nProceed directly to the requested engineering review; skip the optional /office-hours prerequisite.\nPlease review this plan thoroughly. As you go, write your plan-mode plan to (use Edit/Write to that exact path).\n\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Review preparation record\n\n- Design doc: none found. /office-hours prerequisite skipped at the user's explicit request.\n- Repo evidence: the repository contains only `PLAN.md` and `CLAUDE.md`. No worker files, no `processWebhookJob()`, no job library config, no tests are checked in. Every code-level claim below is calibrated against the plan text only; runtime evidence is `unknown` throughout.\n- Retrospective history: `git log -- PLAN.md` shows one commit (c03143b \"Seed review plan\"); no reverts. Worker files and `processWebhookJob()` are named by the plan but have no path here: `not available`.\n- Test framework: unknown (no runtime markers, 0 test files).\n- TODOS.md: absent.\n- Prior learnings: cross-project learnings set to `false`; project-scoped search returned none.\n- Web research: Aside unavailable; one host WebSearch ran (read-only, untrusted). Takeaways: mainstream job libraries (BullMQ is the documented example) accept a custom backoff strategy function, so \"full control over the curve\" does not require leaving the library's retry hooks; pure exponential backoff without jitter produces lockstep retry storms during a provider outage (reported as real bugs in two queue projects); full or decorrelated jitter plus a delay cap and a dead-letter path are the baseline pattern.\n- Outside Voice: `codex_reviews` is `disabled`; the outside-voice step will be recorded as disabled, not replaced.\n\n## Scope Challenge\n\n### A. Assessment\n\n**What already solves each sub-problem?**\n- Retry scheduling, attempt counting, delay persistence: the plan itself says the existing job library has built-in retry hooks and that the custom scheduler is the \"same shape as the library version\" (PLAN.md:7-9). [Layer 1] reuse candidate. The library's hooks were not inspected (not in repo); label: plan-stated interface, unverified.\n- Backoff curve: libraries in this class take a custom backoff function. The plan's only stated reason to leave the library (\"full control over the curve\") is satisfiable inside the hooks. Assumption, not verified against the actual library.\n- Retry envelope (compute delay, log attempt, dispatch): if the library owns retries, the envelope ceases to exist in worker code. Otherwise it is one small helper, not five copies.\n\n**Minimum change to achieve the goal:** configure the library's retry hooks per queue (attempts, backoff function, jitter, cap, dead-letter), add attempt logging in one place, protect the webhook flow's delivery semantics with a test. The custom scheduler and the five copies are deferrable without blocking anything.\n\n**Complexity check (estimates, plan-stated):** ~5 worker files + the `processWebhookJob()` flow (likely one of the 5, counted once) = ~5-6 changed files; 0 new classes/services (scheduler is inline). Below the 8-file / 2-class gate, so B's selectors are skipped.\n\n**Search check:** done above (WebSearch). Recommendation labels: library hooks [Layer 1]; custom inline scheduler departs from standard practice without a stated reason the library cannot meet.\n\n**TODOS cross-reference:** no TODOS.md.\n\n**Completeness check:** the plan explicitly cuts three corners (duplication, regression test, graph recompute). Each saves CC minutes, not days. Completeness is cheap here.\n\n**Distribution check:** no new artifact; internal worker change only.\n\n### B. Complexity selectors\n\nSkipped: fewer than 8 files and fewer than 2 new classes/services.\n\n### C. Findings\n\n1. `[P1] (confidence: 8/10) PLAN.md:7-9 — Rebuilds retry scheduling the job library already provides (\"Same shape as the library version\"). Custom inline scheduler must re-own attempt persistence, delay timers, crash recovery and observability per worker. Reviewer: Claude.` Motivating lines: \"We'll roll a custom exponential-backoff scheduler inline in each worker / rather than use the existing job library's built-in retry hooks. Same / shape as the library version, but we want full control over the curve.\"\n2. `[P2] (confidence: 8/10) PLAN.md:12-14 — Retry envelope copy-pasted across 5 workers; a bug in delay math or attempt logging has to be fixed five times. Reviewer: Claude.` Motivating lines: \"The retry envelope (compute delay, log attempt, dispatch) is duplicated / across 5 worker files with copy-pasted bodies.\" Depends on finding 1: resolves itself if the library owns retries.\n\nFindings surfaced here but owned by later sections (not resolved in Scope Challenge): webhook at-most-once guarantee under retry (Architecture + Tests, PLAN.md:17-19); missing jitter / cap / max attempts / dead-letter (Architecture, PLAN.md:7-9); payload re-fetch and graph recompute per retry (Performance, PLAN.md:22-24).\n\nScope Challenge result: pending R1.\n\n## Decision ledger\n\n### R1: Retry mechanism — library built-in hooks vs custom inline scheduler\nFinding: Scope Challenge #1, P1, confidence 8/10, PLAN.md:7-9, reviewer Claude\nPlan baseline: original proposal — custom exponential-backoff scheduler inline in each of ~5 workers, bypassing the library's retry hooks (PLAN.md:7-9). Nothing approved yet.\nRuntime evidence: unknown. No worker source, library config or hook API is in the repo. The plan states the library has built-in retry hooks (plan-stated interface, unverified).\nComparison grid:\n\n| Choice | Current | A) Library hooks + custom backoff fn | B) Custom inline scheduler (as planned) | C) Investigate hook API first |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler per worker, proposed | library's retry hooks own scheduling, attempt count and persistence; one custom backoff function supplies the curve | custom scheduler inline in each worker, as written | pending; bounded read of the library's retry hook API (human ~1h / CC ~5 min), no implementation |\n| Backoff curve control (\"full control over the curve\") | required by plan | retained: the custom backoff function returns any delay | retained | pending |\n| Attempt state across worker crash/restart | unspecified | library-owned (plan-stated capability, unverified) | must be built per worker | pending |\n| R2 webhook delivery semantics under retry | unspecified, pending | pending | pending | pending |\n| R3 regression contract for at-most-once webhook flow | none planned, pending | pending | pending | pending |\n| R4 retry envelope deduplication (5 copies) | 5 copy-pasted bodies, pending | becomes moot: envelope lives in library config, not worker code | pending | pending |\n| R5 jitter | unspecified, pending | pending | pending | pending |\n| R6 delay cap / max attempts / dead-letter | unspecified, pending | pending | pending | pending |\n| R7 dependency-graph caching on retry | not planned, pending | pending | pending | pending |\n\nQuestion D1:\nD1 — Retry mechanism: reuse the job library's retry hooks or build a custom inline scheduler?\nProject/branch/task: main, \"Add background job retry framework\" plan review; repo holds only PLAN.md, so this is judged on the plan text.\nELI10: The plan rebuilds retry scheduling inside every worker even though the job library already has retry hooks that do the same thing. The stated reason is wanting control over the backoff curve, but libraries in this class let you hand them your own backoff function, so you keep the curve and drop the rebuild. A custom scheduler also has to re-solve things the library already handles: remembering how many attempts happened, surviving a worker crash mid-wait, and showing up in the library's dashboards.\nStakes if we pick wrong: a hand-rolled scheduler means five places where attempt counting or delay math can drift, retries lost when a worker restarts, and a retry framework nobody else on the team recognizes. Picking the library when its hooks turn out to lack a needed feature costs a day of rework, which option C removes.\nRecommendation: A because the plan's only reason to leave the library (\"full control over the curve\") is met by a custom backoff function, so the rebuild buys nothing and costs persistence, observability and five maintenance sites.\nCompleteness: A=9/10, B=5/10, C=n/a (investigation only, approves nothing)\nPros / cons:\nA) Library hooks + custom backoff function (recommended)\n ✅ Attempt count, delay persistence and crash recovery stay library-owned; nothing to re-implement per worker\n ✅ The backoff curve is still fully yours: one function returns the delay for attempt N (human: ~1 day / CC: ~20 min)\n ✅ Finding 2 (five copy-pasted envelopes) disappears: the envelope is config, not worker code\n ❌ Depends on the library's hook API actually accepting a custom backoff function; not verified here because the library is not in this repo\nB) Custom inline scheduler per worker (as planned)\n ✅ Zero dependency on the library's hook surface; works even if its hooks are rigid\n ✅ Lets each worker tune its curve independently without touching queue config\n ❌ Each worker must own attempt tracking, timers and restart recovery; a crash mid-backoff silently drops the retry unless you also build persistence (human: ~3 days / CC: ~1 h)\n ❌ Keeps the five-copy duplication alive and puts the retry path outside the library's metrics and dashboards\nC) Investigate the hook API first, then decide\n ✅ Replaces the unverified assumption in A with a 5-minute read of the library's retry/backoff hook docs or source\n ✅ Costs almost nothing and makes the A/B call on evidence instead of plan text\n ❌ Defers the decision; nothing in this plan can be approved until the read-back is done\nNet: trade an unverified hook-API assumption (A) against a known rebuild of persistence, observability and five maintenance sites (B); C buys the evidence for a few minutes.\nHeader: Retry mechanism\nOptions:\nA) Library hooks + custom backoff function (recommended)\nReuse the job library's built-in retry hooks; supply one custom backoff function for the curve. Library owns attempt count, delay persistence and crash recovery. Makes the five-copy envelope duplication moot. Human ~1 day / CC ~20 min. Risk: assumes the hook API accepts a custom backoff function (unverified, library not in repo). Completeness 9/10.\nB) Custom inline scheduler per worker (as planned)\nBuild the exponential-backoff scheduler inline in each of ~5 workers as written in PLAN.md:7-9. Each worker owns attempt tracking, timers and restart recovery; retry path sits outside library metrics. Keeps the five copies pending a separate dedupe decision. Human ~3 days / CC ~1 h. Completeness 5/10.\nC) Investigate the hook API first\nBounded read of the library's retry/backoff hook API (docs or source, human ~1 h / CC ~5 min) before choosing. Approves nothing; R1 and every dependent row stay pending until the read-back is reported.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-03T23:46:32.676Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hooks + custom backoff function — D1 answer, 2026-10-03\nAccepted scope: Replace the custom inline per-worker scheduler (PLAN.md:7-9) with the job library's built-in retry hooks; supply one custom backoff function for the curve; library owns attempt count, delay persistence and crash recovery. R4 (five-copy envelope dedupe) is moot under this answer: the envelope becomes queue/hook configuration, not worker code. Carried pending: R2, R3, R5, R6, R7 and later rows. Assumption to confirm at implementation: the library's hook API accepts a custom backoff function.\nHistory: none\n\n### R2: Webhook delivery semantics once `processWebhookJob()` gains retries\nFinding: Architecture #1, P1, confidence 8/10, PLAN.md:17-19, reviewer Claude. Motivating lines: \"The existing `processWebhookJob()` flow gets rewritten as part of this / change. No regression test for the prior at-most-once delivery guarantee / is planned.\"\nPlan baseline: original proposal — the webhook flow is rewritten into the retry framework; its current at-most-once guarantee is named but no decision is made about what replaces it. R1 approved (library hooks) governs the mechanism; semantics still unspecified.\nRuntime evidence: unknown. `processWebhookJob()` is not in the repo; whether it records a send before the HTTP call, or what receivers rely on, cannot be checked here.\nComparison grid:\n\n| Choice | Current | A) At-least-once with idempotency key | B) Retry only pre-send failures | C) Exclude webhook flow from retries |\n|---|---|---|---|---|\n| R2 delivery guarantee receivers get | at-most-once (plan-stated), under rewrite with no replacement guarantee named | at-least-once; each delivery carries a stable idempotency key (job id + event id) so receivers can dedupe | at-most-once preserved: retry only when the request provably never left (connection refused, DNS, TLS handshake); post-send ambiguity (read timeout, 5xx) is terminal | at-most-once preserved by keeping `processWebhookJob()` on the current no-retry path |\n| Duplicate deliveries to receivers | none today | possible; deduplicable via key | none | none |\n| Lost deliveries on 5xx / read timeout | lost (no retry today) | retried | still lost | still lost |\n| Receiver-facing contract change | none | yes: documented key header + \"may deliver more than once\" | none | none |\n| Per-attempt delivery record (attempt no., response) | unspecified | yes, part of this behavior | yes, part of this behavior | no change |\n| R1 mechanism | approved D1 | library hooks, unchanged | library hooks, unchanged | library hooks for the other 4 workers, unchanged |\n| R3 regression contract for the webhook flow | pending | pending | pending | pending |\n| R5 jitter, R6 delay cap, R7 max attempts/dead-letter, R8 graph caching | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — Webhook delivery: what guarantee do receivers get once `processWebhookJob()` can retry?\nProject/branch/task: main, \"Add background job retry framework\"; R1 settled on the library's retry hooks, now the webhook flow's semantics.\nELI10: Today a webhook is sent at most once: if it fails, it is gone, but a receiver never sees the same event twice. The moment you add retries, a request that timed out after reaching the receiver gets sent again, and the receiver sees a duplicate. You have to pick on purpose: embrace duplicates and give receivers a key to spot them, only retry when you are sure the request never left, or keep webhooks out of the retry framework entirely.\nStakes if we pick wrong: silently flipping to at-least-once means customers' systems double-charge, double-notify or double-create records with no way to detect it. Keeping at-most-once means the common failure (a 5xx or a slow receiver) still loses the event.\nRecommendation: A because at-least-once with a documented idempotency key is the standard webhook contract and the only option that both recovers lost deliveries and lets receivers protect themselves.\nCompleteness: A=9/10, B=6/10, C=4/10\nPros / cons:\nA) At-least-once with idempotency key (recommended)\n ✅ Recovers deliveries lost to 5xx and read timeouts, which is most of what a retry framework exists for\n ✅ Receivers get a stable key per event and a documented \"may deliver more than once\" contract they can dedupe on (human: ~1.5 days / CC: ~30 min)\n ❌ Changes the receiver-facing contract; any receiver that does not dedupe will see duplicates until they do\nB) Retry only pre-send failures\n ✅ Preserves the current at-most-once guarantee exactly; no receiver sees a duplicate\n ✅ Still recovers the cheap failures (connection refused, DNS, TLS handshake) with no contract change (human: ~1 day / CC: ~20 min)\n ❌ The most common failures (5xx, slow receiver) stay lost, and classifying \"provably never sent\" per HTTP client is brittle\nC) Exclude the webhook flow from retries\n ✅ Zero contract risk and almost zero work; the other four workers still get retries (human: ~0 / CC: ~2 min)\n ✅ Buys time to decide the webhook contract with receivers separately\n ❌ The webhook flow gets no reliability gain from this change, and the plan's rewrite of `processWebhookJob()` would need to be dropped\nNet: trade a documented contract change that receivers must honor (A) against preserving at-most-once at the cost of still losing the common failures (B, C).\nHeader: Webhook delivery\nOptions:\nA) At-least-once with idempotency key (recommended)\nWebhook retries enabled through the library hooks; every delivery carries a stable idempotency key (job id + event id) header and each attempt is recorded (attempt number, response). Receiver-facing docs updated to \"at-least-once, dedupe on the key\". Human ~1.5 days / CC ~30 min. Completeness 9/10. R3 regression contract stays pending.\nB) Retry only pre-send failures\nRetry only when the request provably never left (connection refused, DNS, TLS handshake failure); read timeouts and 5xx are terminal. Preserves at-most-once. Each attempt recorded. Human ~1 day / CC ~20 min. Completeness 6/10: 5xx and slow receivers still lose the event. R3 stays pending.\nC) Exclude the webhook flow from retries\nKeep `processWebhookJob()` on its current no-retry path; the other four workers adopt the library hooks. No contract change, no reliability gain for webhooks. Human ~0 / CC ~2 min. Completeness 4/10. R3 stays pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-03T23:47:31.675Z", + "old": "Keep `processWebhookJob()` on its current no-retry path; the other four workers adopt the library hooks. No contract change, no reliability gain for webhooks. Human ~0 / CC ~2 min. Completeness 4/10. R3 stays pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "Keep `processWebhookJob()` on its current no-retry path; the other four workers adopt the library hooks. No contract change, no reliability gain for webhooks. Human ~0 / CC ~2 min. Completeness 4/10. R3 stays pending.\n\nState: approved\nActual answer: A) At-least-once with idempotency key — D2 answer, 2026-10-03\nAccepted scope: `processWebhookJob()` retries through the library hooks (R1). Every delivery carries a stable idempotency key (job id + event id) header; each attempt is recorded (attempt number, response). Receiver-facing documentation updated to \"at-least-once, dedupe on the key\". Intentional difference from today: at-most-once is replaced by at-least-once. Carried pending: R3 (regression contract for the webhook flow), R5, R6, R7, R8.\nHistory: none\n\n### R5: Jitter on retry delays\nFinding: Architecture #2, P2, confidence 8/10, PLAN.md:7-9, reviewer Claude. Motivating lines: \"We'll roll a custom exponential-backoff scheduler ... we want full control over the curve.\" No jitter is mentioned anywhere in the plan.\nPlan baseline: original proposal — pure exponential curve, jitter unspecified. R1 approved: the curve is supplied by one custom backoff function in the library hooks.\nRuntime evidence: unknown (no backoff code in repo). Web research: two queue projects filed real bugs for lockstep retries after a provider outage because their exponential backoff had no jitter.\nComparison grid:\n\n| Choice | Current | A) Full jitter | B) Equal jitter | C) No jitter |\n|---|---|---|---|---|\n| R5 jitter | unspecified, pending | delay = random(0, base * 2^attempt) | delay = half fixed + half random of base * 2^attempt | delay = base * 2^attempt exactly, as planned |\n| Lockstep retry storm after a shared outage | unmitigated | spread across the full window | spread across half the window, guaranteed minimum wait | all failed jobs retry at the same instants |\n| Where it lives | n/a | inside the one custom backoff function (R1) | inside the one custom backoff function (R1) | inside the one custom backoff function (R1) |\n| R1 mechanism, R2 webhook semantics | approved D1, D2 | unchanged | unchanged | unchanged |\n| R3 regression contract, R6 delay cap, R7 max attempts/dead-letter, R8 graph caching | pending | pending | pending | pending |\n\nQuestion D3:\nD3 — Jitter: randomize retry delays, or keep the pure exponential curve?\nProject/branch/task: main, \"Add background job retry framework\"; the curve now lives in one custom backoff function supplied to the library hooks (D1).\nELI10: If a downstream service goes down for a minute, hundreds of jobs fail at the same moment. With a pure exponential curve they all wake up and retry at the same instants, hammering the service just as it recovers. Jitter adds randomness to each delay so the retries spread out instead of arriving as a wave.\nStakes if we pick wrong: without jitter the first retry wave after an outage can knock the dependency over again and extend the incident; with jitter the only cost is that a given retry may fire a little earlier or later than the textbook curve.\nRecommendation: A because full jitter is the documented safe default for background jobs, it is a one-line change inside the backoff function, and every real-world retry-storm bug found in research traced back to its absence.\nCompleteness: A=9/10, B=8/10, C=4/10\nPros / cons:\nA) Full jitter (recommended)\n ✅ Spreads retries across the whole window, the strongest protection against a lockstep wave after a shared outage\n ✅ One line inside the custom backoff function; no new state, no new dependency (human: ~1 h incl. test / CC: ~3 min)\n ❌ A retry can fire almost immediately after a failure (random lower bound is 0), which can look surprising in logs\nB) Equal jitter\n ✅ Guarantees a minimum wait of half the computed delay, so no retry fires \"too soon\"\n ✅ Still breaks up the lockstep wave, just over half the window (human: ~1 h incl. test / CC: ~3 min)\n ❌ Weaker spread than full jitter; only worth it if a hard minimum wait is a real requirement here, and none is stated\nC) No jitter (as planned)\n ✅ Delays are exactly predictable, which makes the curve easy to reason about and test\n ✅ Zero extra code beyond the curve itself\n ❌ Every job that failed in the same outage retries at the same instants; this is the classic retry-storm bug reported against real queue libraries\nNet: a few minutes of code (A or B) against a known outage amplifier (C); A unless a hard minimum wait is a stated requirement.\nHeader: Jitter\nOptions:\nA) Full jitter (recommended)\nCustom backoff function returns random(0, base * 2^attempt). One line plus a test asserting delays stay within [0, computed]. Human ~1 h / CC ~3 min. Completeness 9/10. R6 cap, R7 max attempts/dead-letter stay pending.\nB) Equal jitter\nCustom backoff function returns computed/2 + random(0, computed/2). Guaranteed minimum wait. One line plus a bounds test. Human ~1 h / CC ~3 min. Completeness 8/10. R6, R7 stay pending.\nC) No jitter (as planned)\nCustom backoff function returns base * 2^attempt exactly. Lockstep retries after a shared outage are accepted. Human ~0 / CC ~0. Completeness 4/10. R6, R7 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37174266054-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37174266054-t1-pass.json new file mode 100644 index 000000000..487f1ae16 --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37174266054-t1-pass.json @@ -0,0 +1,116 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "57a825b033078a7b", + "source": { + "run": "37174266054", + "sha": "a8ace39030dd92745b45ac16e5d733e63c936c0f", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37174266054-1-eval-slices-1/plan-eng-review-1791084772101-4G6gCj" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "dd4cbb83-123f-42d0-b789-0c217651632d", + "toolUseId": "toolu_01Jy4XCdUXYNhyhFhgDtRCTe", + "answeredAt": "2026-10-04T03:35:21.281Z", + "answer": "Library hook + custom backoff (recommended)", + "question": { + "question": "D1 — Retry mechanism: library retry hooks or a custom inline scheduler?\nProject/branch/task: main — plan \"Add background job retry framework\" (PLAN.md).\nELI10: Your job library already knows how to retry a failed job later. The plan rebuilds that machinery by hand inside each of 5 workers because you want to control the delay curve. Almost every job library lets you hand it your own backoff function, so you can own the curve without owning the scheduler. Hand-rolled schedulers are where lost jobs, double-runs and 3am retry storms come from.\nStakes if we pick wrong: a bug in the delay math or dispatch gets copied into 5 places, and failed jobs either vanish silently or hammer a downstream service in lockstep.\nRecommendation: A because the library hook gives you the curve you want and keeps attempt tracking, persistence and dispatch in code that already runs in production.\nCompleteness: A=9/10, B=6/10, C=4/10\nPros / cons:\nA) Library hook + custom backoff function (recommended) [Layer 1] (human: ~1 day / CC: ~20 min)\n ✅ Attempt counting, delay persistence and dispatch stay in battle-tested library code used by many teams\n ✅ The curve becomes one small pure function that is trivially unit-testable and shared by all 5 workers\n ❌ If the hook truly cannot express the desired curve you need a thin wrapper or an upstream patch; verify first\nB) Custom inline scheduler as planned (human: ~3-4 days / CC: ~1 h)\n ✅ Total control over delay, logging and dispatch timing with no library constraints at all\n ✅ Curve changes never wait on a library release or its configuration surface\n ❌ Re-implements attempt persistence and dispatch five times; every scheduling bug has a 5x blast radius and needs its own scheduler tests\nC) Investigate: 1-hour spike to confirm the hook can express the curve; R1 stays pending (human: ~1 h / CC: ~5 min)\n ✅ Removes the single unknown (hook expressiveness) before any code is written\n ✅ Cheap: read the library's retry docs and run one failing test job through its hook\n ❌ Nothing else in the plan can be locked until this returns; approves no implementation\nNet: owning the curve does not require owning the scheduler; the trade is library constraints versus a 5x hand-rolled blast radius.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hook + custom backoff (recommended)", + "description": "Plug one backoff function into the job library's existing retry hook; delete the planned inline scheduler. Attempt counting, persistence and dispatch stay in the library. Human ~1 day / CC ~20 min. Completeness 9/10. Risk: hook may not express the exact curve; verify in docs first. [Layer 1]" + }, + { + "label": "Custom inline scheduler", + "description": "Build the exponential-backoff scheduler by hand inside each worker as the plan states. Full control over curve, logging and dispatch. Human ~3-4 days / CC ~1 h. Completeness 6/10. Risk: re-implements persistence and dispatch five times; scheduler bugs have a 5x blast radius and need dedicated tests." + }, + { + "label": "Investigate hook first", + "description": "Bounded 1-hour spike: read the library's retry docs and push one failing job through its hook to confirm the curve is expressible. R1 stays pending; approves no implementation. Human ~1 h / CC ~5 min. Completeness 4/10." + } + ] + } + }, + { + "sessionId": "dd4cbb83-123f-42d0-b789-0c217651632d", + "toolUseId": "toolu_01Xh21vyj1n4igxYL95Smfwn", + "answeredAt": "2026-10-04T03:37:02.866Z", + "answer": "At-least-once + idempotency key (recommended)", + "question": { + "question": "D2 — Webhook retries: at-least-once with idempotency key, or keep at-most-once?\nProject/branch/task: main — plan \"Add background job retry framework\" (PLAN.md), webhook path.\nELI10: Today the webhook job promises \"at most once\": a delivery can be lost but is never duplicated. Retries flip that promise. A webhook that timed out after the receiver already handled it will be sent again. Receivers that charge a card or send an email on each webhook will do it twice unless every event carries a stable idempotency key they can dedupe on. You have to pick which promise you keep and write it down; Section 3 then protects it with a test.\nStakes if we pick wrong: silent duplicate webhooks to customers' endpoints (double charges, double emails), or no retry benefit on the path most likely to fail.\nRecommendation: A because retries without an idempotency key are duplicates by design, and at-least-once plus a stable event id is the contract every major webhook provider ships.\nCompleteness: A=9/10, B=7/10, C=3/10\nPros / cons:\nA) At-least-once + idempotency key per event (recommended) (human: ~1-2 days / CC: ~30 min)\n ✅ Failed deliveries are actually recovered, which is the whole point of the retry framework\n ✅ A stable event id persisted before the first attempt lets receivers dedupe; matches Stripe/GitHub webhook contracts\n ❌ Changes the documented guarantee; receivers must be told to dedupe, and the id must survive the payload re-fetch\nB) Keep at-most-once: webhook job opts out of retries (human: ~2 h / CC: ~10 min)\n ✅ Zero duplicate risk; existing receiver behavior is untouched\n ✅ Small change: max attempts 1 for this job type plus a recorded-failure row for manual replay\n ❌ Webhook failures stay unrecovered; operators replay by hand\nC) Retry webhooks with no idempotency key, as the plan reads (human: ~0 / CC: ~0)\n ✅ No extra work beyond the framework itself\n ✅ No receiver coordination needed\n ❌ Guaranteed duplicates on every timeout-after-success; the at-most-once promise breaks silently with nothing telling you\nNet: you are choosing between recovering failed webhooks (and asking receivers to dedupe) or keeping them un-duplicated (and un-recovered).", + "header": "Webhook guarantee", + "multiSelect": false, + "options": [ + { + "label": "At-least-once + idempotency key (recommended)", + "description": "Webhook retries enabled under the library hook. Each webhook event gets a stable idempotency id persisted before the first attempt and sent as a header; the guarantee is documented as at-least-once. Human ~1-2 days / CC ~30 min. Completeness 9/10. Risk: receivers must dedupe; id must survive the payload re-fetch." + }, + { + "label": "Keep at-most-once (opt out)", + "description": "Webhook job type is excluded from retries (max attempts 1). Failures are recorded for manual replay. Existing guarantee untouched. Human ~2 h / CC ~10 min. Completeness 7/10. Risk: webhook failures remain unrecovered." + }, + { + "label": "Retry with no idempotency key", + "description": "Plan as written: webhook retried like any job, no event id, guarantee undocumented. Human ~0 / CC ~0. Completeness 3/10. Risk: silent duplicates on every timeout-after-success." + } + ] + } + }, + { + "sessionId": "dd4cbb83-123f-42d0-b789-0c217651632d", + "toolUseId": "toolu_01WE9ouMGiiYz1BFJJmxfZg8", + "answeredAt": "2026-10-04T03:37:56.630Z", + "answer": "Cap + dead-letter + alert (recommended)", + "question": { + "question": "D3 — Bounded retries: cap attempts and dead-letter exhausted jobs, or leave it to the library default?\nProject/branch/task: main — plan \"Add background job retry framework\" (PLAN.md), all 5 workers.\nELI10: Some jobs will never succeed: a bad payload, a customer endpoint that is gone for good. Without an explicit cap they retry until the curve stretches into days, and without a parking spot they are either dropped silently or sit in the queue forever. The fix is boring: a per-job-type attempt cap, and a dead-letter place where exhausted jobs land with an alert so a human sees them.\nStakes if we pick wrong: poison jobs burn worker capacity for days, or real customer jobs vanish with no record and no alert.\nRecommendation: A because the library hook (D1) makes the cap one config value and the dead-letter one handler, and it turns a silent loss into a visible, replayable one.\nCompleteness: A=9/10, B=6/10, C=3/10\nPros / cons:\nA) Explicit cap (default 5, per job type) + dead-letter with alert (recommended) (human: ~half day / CC: ~15 min)\n ✅ Exhausted jobs are preserved with their payload and error for inspection and manual replay\n ✅ An alert on dead-letter depth turns a silent failure into an operator signal within minutes\n ❌ One more queue/set to monitor and to drain; needs a replay path or it fills up quietly\nB) Explicit cap (default 5), exhausted jobs logged as failed only (human: ~1 h / CC: ~5 min)\n ✅ Bounds the retry storm with a single config value per job type\n ✅ No new queue or alert plumbing to maintain\n ❌ Exhausted jobs are gone after the log line; no replay and nobody is paged\nC) Library default cap, no explicit policy, as the plan reads (human: ~0 / CC: ~0)\n ✅ Zero work; the library's default behavior applies\n ✅ Fewer decisions to document now\n ❌ You do not know today what the default is or where exhausted jobs go; the first poison job teaches you in production\nNet: a cap without a dead-letter bounds the damage but still loses the job; the dead-letter is what makes the loss recoverable.", + "header": "Bounded retries", + "multiSelect": false, + "options": [ + { + "label": "Cap + dead-letter + alert (recommended)", + "description": "Per-job-type max attempts (default 5) in the library config; exhausted jobs land in a dead-letter queue/failed set with payload and last error; alert when depth > 0; documented replay command. Human ~half day / CC ~15 min. Completeness 9/10. Risk: DLQ needs draining discipline." + }, + { + "label": "Cap only, log failures", + "description": "Per-job-type max attempts (default 5); exhausted jobs emit a structured failure log and are removed. No DLQ, no alert. Human ~1 h / CC ~5 min. Completeness 6/10. Risk: no replay, nobody paged." + }, + { + "label": "Library default, no policy", + "description": "Plan as written: no explicit cap or destination; library defaults apply. Human ~0 / CC ~0. Completeness 3/10. Risk: unknown default discovered in production." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T03:34:47.260Z", + "write": "# Eng Review — Plan: Add background job retry framework\n\nReview target (fixed): `PLAN.md` in repo `gstack-plan-count-PVw5NO`, branch `main`, commit 21338e5.\nReviewer: /plan-eng-review (Claude, native). Session 691-1791084764-0bd1c81e. Date 2026-10-04.\nEvidence limits: the repository contains only `PLAN.md` and `CLAUDE.md`. The 5 worker files, the job library and `processWebhookJob()` named in the plan are not present, so all runtime evidence below is **unknown** and findings are calibrated from the plan text itself.\n\n## Original plan (unchanged copy)\n\n```markdown\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Scope Challenge\n\n### A. Assessment\n\n- **What already solves each sub-problem:** The plan itself states the job library has built-in retry hooks (PLAN.md:7-9). Library not named in repo; its hook contract is unverified. Most mainstream job libraries (BullMQ, Sidekiq, Celery, Oban, Faktory, Resque, Rails ActiveJob) accept a custom backoff function or `retry_in`/`backoffStrategy` callback, so \"full control over the curve\" does not require owning the scheduler. **[Layer 1]**\n- **Minimum change to the goal:** a backoff function plugged into the library hook, plus idempotency for the webhook path, plus a regression test for the webhook rewrite. The hand-rolled scheduler and the 5x copy-paste are deferrable work that adds risk rather than reducing it.\n- **Complexity count (estimates):** 5 worker files + the webhook job path = ~5-6 changed files; 0 new classes/services (scheduler is inline). Below the 8-file / 2-service gate, so B's structure question is skipped.\n- **Search check:** Aside unavailable; host WebSearch used for one sanitized query on backoff/jitter/at-most-once practice (results in Section 1).\n- **TODOS cross-reference:** no `TODOS.md` in repo.\n- **Completeness check:** every shortcut in the plan (\"refactor later\", \"no regression test\", \"not planned\" cache) saves CC minutes, not days. Boil the ocean.\n- **Distribution check:** no new artifacts; not applicable.\n\n### B. Complexity selectors\n\nSkipped: fewer than 8 files and fewer than 2 new classes/services.\n\n### C. Findings\n\n1. **[P1] (confidence: 8/10) PLAN.md:7-9** — \"roll a custom exponential-backoff scheduler inline in each worker rather than use the existing job library's built-in retry hooks. Same shape as the library version\". Rebuilds attempt persistence, delay scheduling and dispatch that the library already provides, in 5 places, to gain a curve the hook almost certainly already lets you supply. Reviewer: native. Remedy: R1 below.\n\nScope Challenge result: pending D1.\n\n## Decision ledger\n\n### R1: Retry mechanism — library retry hooks or custom inline scheduler\nFinding: Scope Challenge #1, P1, confidence 8/10, PLAN.md:7-9, reviewer native\nPlan baseline: original proposal, custom exponential-backoff scheduler inline in each of 5 workers (PLAN.md:7-9); nothing approved yet\nRuntime evidence: unknown. Library not named and not in repo; hook contract unverified.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline per worker (PLAN.md:7-9), pending | library retry hook + one custom backoff function | custom inline scheduler as planned | pending; bounded 1h spike on hook expressiveness only |\n| R2 webhook delivery guarantee under retries | unspecified, pending | pending | pending | pending |\n| R3 shared retry envelope (5 workers) | duplicated, pending | pending (largely moot if A) | pending | pending |\n| R4 bounded retries + dead-letter | unspecified, pending | pending | pending | pending |\n| R5 jitter on backoff | unspecified, pending | pending | pending | pending |\n| R6 payload/graph caching across attempts | re-fetch every attempt, pending | pending | pending | pending |\n\nQuestion D1:\nD1 — Retry mechanism: library retry hooks or a custom inline scheduler?\nProject/branch/task: main — plan \"Add background job retry framework\" (PLAN.md).\nELI10: Your job library already knows how to retry a failed job later. The plan rebuilds that machinery by hand inside each of 5 workers because you want to control the delay curve. Almost every job library lets you hand it your own backoff function, so you can own the curve without owning the scheduler. Hand-rolled schedulers are where lost jobs, double-runs and 3am retry storms come from.\nStakes if we pick wrong: a bug in the delay math or dispatch gets copied into 5 places, and failed jobs either vanish silently or hammer a downstream service in lockstep.\nRecommendation: A because the library hook gives you the curve you want and keeps attempt tracking, persistence and dispatch in code that already runs in production.\nCompleteness: A=9/10, B=6/10, C=4/10\nPros / cons:\nA) Library hook + custom backoff function (recommended) [Layer 1] (human: ~1 day / CC: ~20 min)\n ✅ Attempt counting, delay persistence and dispatch stay in battle-tested library code used by many teams\n ✅ The curve becomes one small pure function that is trivially unit-testable and shared by all 5 workers\n ❌ If the hook truly cannot express the desired curve you need a thin wrapper or an upstream patch; verify first\nB) Custom inline scheduler as planned (human: ~3-4 days / CC: ~1 h)\n ✅ Total control over delay, logging and dispatch timing with no library constraints at all\n ✅ Curve changes never wait on a library release or its configuration surface\n ❌ Re-implements attempt persistence and dispatch five times; every scheduling bug has a 5x blast radius and needs its own scheduler tests\nC) Investigate: 1-hour spike to confirm the hook can express the curve; R1 stays pending (human: ~1 h / CC: ~5 min)\n ✅ Removes the single unknown (hook expressiveness) before any code is written\n ✅ Cheap: read the library's retry docs and run one failing test job through its hook\n ❌ Nothing else in the plan can be locked until this returns; approves no implementation\nNet: owning the curve does not require owning the scheduler; the trade is library constraints versus a 5x hand-rolled blast radius.\nHeader: Retry mechanism\nOptions:\nA) Library hook + custom backoff (recommended)\nPlug one backoff function into the job library's existing retry hook; delete the planned inline scheduler. Attempt counting, persistence and dispatch stay in the library. Human ~1 day / CC ~20 min. Completeness 9/10. Risk: hook may not express the exact curve; verify in docs first. [Layer 1]\nB) Custom inline scheduler\nBuild the exponential-backoff scheduler by hand inside each worker as the plan states. Full control over curve, logging and dispatch. Human ~3-4 days / CC ~1 h. Completeness 6/10. Risk: re-implements persistence and dispatch five times; scheduler bugs have a 5x blast radius and need dedicated tests.\nC) Investigate hook first\nBounded 1-hour spike: read the library's retry docs and push one failing job through its hook to confirm the curve is expressible. R1 stays pending; approves no implementation. Human ~1 h / CC ~5 min. Completeness 4/10.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T03:36:24.932Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hook + custom backoff (recommended) — D1, user answer\nAccepted scope: Replace the planned inline per-worker scheduler with the job library's existing retry hook plus one shared pure backoff function (the \"curve\"). Attempt counting, delay persistence and dispatch remain in the library. Includes: verify in the library docs that the hook accepts a custom delay callback before coding; unit tests for the backoff function (attempt → delay table, cap, boundary at attempt 0 and at max). R2–R6 remain pending.\nHistory: none\n\n## Section 1 — Architecture review\n\n### Retry flow after D1 (library hook + custom curve)\n\n```\njob fails ──► library retry hook ──► backoff(attempt, err) ──► delay ──► re-enqueue\n │ │ │\n │ └─ shared pure fn (R1 approved) └─ library persistence\n ├─ attempt >= max? ──► dead-letter / alert (R4 pending)\n └─ webhook job? ──► same payload re-sent ──► receiver sees duplicate? (R2 pending)\n```\n\n### Findings\n\n1. **[P1] (confidence: 8/10) PLAN.md:17-19** — \"`processWebhookJob()` flow gets rewritten ... No regression test for the prior at-most-once delivery guarantee\". Adding retries to a path that promised at-most-once converts it to at-least-once. A webhook that timed out after the receiver already processed it is re-sent; receivers that act on webhooks (charges, emails) double-act unless each event carries a stable idempotency key. The plan does not name which guarantee it keeps. Reviewer: native. Remedy: R2.\n2. **[P2] (confidence: 7/10) PLAN.md:7-8** — \"custom exponential-backoff scheduler\" names no max-attempts bound and no dead-letter destination. A poison job (bad payload, permanently 4xx endpoint) retries until the curve exceeds any useful horizon, or is dropped silently when the library's default cap fires. Reviewer: native. Remedy: R4. Library hook (D1) makes this a config + alert task, not a build.\n3. **[P2] (confidence: 7/10) PLAN.md:7-9** — \"exponential-backoff ... full control over the curve\" with no jitter named. After a downstream outage every failed job across 5 workers retries on the same schedule: a synchronized retry storm against the recovering service. Full jitter (random in [0, min(cap, base·2^n)]) is the standard fix. Reviewer: native. Remedy: R5.\n\nSecurity: no new auth or data-access boundaries; webhook payload re-sends reuse the existing signing path (unverified, not in repo). Distribution: no new artifacts.\n\n### Failure modes (per new path)\n\n| Path | Realistic production failure | Error handling in plan | Test in plan | User-visible? |\n|---|---|---|---|---|\n| Webhook retry | Receiver processed then timed out; retry re-delivers | none | none | Silent duplicate to customer endpoint — **critical gap** until R2 lands |\n| Backoff curve | Off-by-one makes attempt 0 delay 0 ms → hot loop | library cap only | unit test (D1 scope) | Silent CPU burn |\n| Max attempts | Poison job retries indefinitely / dropped at library default | none | none | Silent loss — **critical gap** until R4 lands |\n| Outage recovery | All jobs retry in lockstep | none | none | Downstream re-fails; users see delayed jobs |\n\n### R2: Webhook delivery guarantee under retries\nFinding: Section 1 #1, P1, confidence 8/10, PLAN.md:17-19, reviewer native\nPlan baseline: original proposal rewrites `processWebhookJob()` under the retry framework with no stated guarantee and no regression test (PLAN.md:17-19); nothing approved for R2\nRuntime evidence: unknown. `processWebhookJob()` and its current at-most-once mechanism are not in the repo; the receiver contract (idempotency header, event id) is unverified.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | approved A (D1): library hook + shared backoff fn | same | same | same |\n| R2 webhook delivery guarantee | unspecified; at-most-once implied by existing code, pending | at-least-once + stable idempotency key per webhook event, documented | keep at-most-once: webhook job opted out of retries (max attempts 1), failures recorded for manual replay | retry webhooks with no idempotency key (plan as written) |\n| R3 shared retry envelope | pending | pending | pending | pending |\n| R4 bounded retries + dead-letter | pending | pending | pending | pending |\n| R5 jitter | pending | pending | pending | pending |\n| R6 payload/graph caching | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — Webhook retries: at-least-once with idempotency key, or keep at-most-once?\nProject/branch/task: main — plan \"Add background job retry framework\" (PLAN.md), webhook path.\nELI10: Today the webhook job promises \"at most once\": a delivery can be lost but is never duplicated. Retries flip that promise. A webhook that timed out after the receiver already handled it will be sent again. Receivers that charge a card or send an email on each webhook will do it twice unless every event carries a stable idempotency key they can dedupe on. You have to pick which promise you keep and write it down; Section 3 then protects it with a test.\nStakes if we pick wrong: silent duplicate webhooks to customers' endpoints (double charges, double emails), or no retry benefit on the path most likely to fail.\nRecommendation: A because retries without an idempotency key are duplicates by design, and at-least-once plus a stable event id is the contract every major webhook provider ships.\nCompleteness: A=9/10, B=7/10, C=3/10\nPros / cons:\nA) At-least-once + idempotency key per event (recommended) (human: ~1-2 days / CC: ~30 min)\n ✅ Failed deliveries are actually recovered, which is the whole point of the retry framework\n ✅ A stable event id persisted before the first attempt lets receivers dedupe; matches Stripe/GitHub webhook contracts\n ❌ Changes the documented guarantee; receivers must be told to dedupe, and the id must survive the payload re-fetch\nB) Keep at-most-once: webhook job opts out of retries (human: ~2 h / CC: ~10 min)\n ✅ Zero duplicate risk; existing receiver behavior is untouched\n ✅ Small change: max attempts 1 for this job type plus a recorded-failure row for manual replay\n ❌ Webhook failures stay unrecovered; operators replay by hand\nC) Retry webhooks with no idempotency key, as the plan reads (human: ~0 / CC: ~0)\n ✅ No extra work beyond the framework itself\n ✅ No receiver coordination needed\n ❌ Guaranteed duplicates on every timeout-after-success; the at-most-once promise breaks silently with nothing telling you\nNet: you are choosing between recovering failed webhooks (and asking receivers to dedupe) or keeping them un-duplicated (and un-recovered).\nHeader: Webhook guarantee\nOptions:\nA) At-least-once + idempotency key (recommended)\nWebhook retries enabled under the library hook. Each webhook event gets a stable idempotency id persisted before the first attempt and sent as a header; the guarantee is documented as at-least-once. Human ~1-2 days / CC ~30 min. Completeness 9/10. Risk: receivers must dedupe; id must survive the payload re-fetch.\nB) Keep at-most-once (opt out)\nWebhook job type is excluded from retries (max attempts 1). Failures are recorded for manual replay. Existing guarantee untouched. Human ~2 h / CC ~10 min. Completeness 7/10. Risk: webhook failures remain unrecovered.\nC) Retry with no idempotency key\nPlan as written: webhook retried like any job, no event id, guarantee undocumented. Human ~0 / CC ~0. Completeness 3/10. Risk: silent duplicates on every timeout-after-success.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T03:36:27.173Z", + "old": "Scope Challenge result: pending D1.", + "new": "Dispositions: #1 accepted — D1 answer A (library hook + shared backoff function); the custom inline scheduler is cut.\n\nScope Challenge result: **scope reduced per recommendation** (custom scheduler removed; library hook retained). MODE = SCOPE_REDUCED." + }, + { + "at": "2026-10-04T03:37:41.143Z", + "old": "Plan as written: webhook retried like any job, no event id, guarantee undocumented. Human ~0 / CC ~0. Completeness 3/10. Risk: silent duplicates on every timeout-after-success.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "Plan as written: webhook retried like any job, no event id, guarantee undocumented. Human ~0 / CC ~0. Completeness 3/10. Risk: silent duplicates on every timeout-after-success.\n\nState: approved\nActual answer: A) At-least-once + idempotency key (recommended) — D2, user answer\nAccepted scope: Webhook job retries run under the library hook (R1). Each webhook event gets a stable idempotency id generated and persisted with the job before the first attempt, sent on every attempt as an idempotency header; the delivery guarantee is documented as at-least-once and the receiver-dedupe expectation is written into the webhook docs. Necessary proof carried forward to Section 3: regression test asserting the same event id on attempt 1 and attempt N, and a test that a timeout-after-success re-sends with the identical id. R3–R6 remain pending.\nHistory: none\n\n### R4: Bounded retries and exhausted-job destination\nFinding: Section 1 #2, P2, confidence 7/10, PLAN.md:7-8, reviewer native\nPlan baseline: original proposal names no max-attempts bound and no destination for exhausted jobs (PLAN.md:7-8); nothing approved for R4\nRuntime evidence: unknown. Library default attempt cap and whether a dead-letter/failed set exists are unverified (library not in repo).\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | approved A (D1) | same | same | same |\n| R2 webhook guarantee | approved A (D2) | same | same | same |\n| R3 shared retry envelope | pending | pending | pending | pending |\n| R4a max attempts | unspecified, pending | explicit per-job-type cap, default 5 attempts | explicit per-job-type cap, default 5 attempts | library default, unspecified |\n| R4b exhausted-job destination | unspecified, pending | dead-letter queue/failed set + alert on depth > 0 | logged as failed only, no DLQ, no alert | whatever the library does by default |\n| R5 jitter | pending | pending | pending | pending |\n| R6 payload/graph caching | pending | pending | pending | pending |\n\nQuestion D3:\nD3 — Bounded retries: cap attempts and dead-letter exhausted jobs, or leave it to the library default?\nProject/branch/task: main — plan \"Add background job retry framework\" (PLAN.md), all 5 workers.\nELI10: Some jobs will never succeed: a bad payload, a customer endpoint that is gone for good. Without an explicit cap they retry until the curve stretches into days, and without a parking spot they are either dropped silently or sit in the queue forever. The fix is boring: a per-job-type attempt cap, and a dead-letter place where exhausted jobs land with an alert so a human sees them.\nStakes if we pick wrong: poison jobs burn worker capacity for days, or real customer jobs vanish with no record and no alert.\nRecommendation: A because the library hook (D1) makes the cap one config value and the dead-letter one handler, and it turns a silent loss into a visible, replayable one.\nCompleteness: A=9/10, B=6/10, C=3/10\nPros / cons:\nA) Explicit cap (default 5, per job type) + dead-letter with alert (recommended) (human: ~half day / CC: ~15 min)\n ✅ Exhausted jobs are preserved with their payload and error for inspection and manual replay\n ✅ An alert on dead-letter depth turns a silent failure into an operator signal within minutes\n ❌ One more queue/set to monitor and to drain; needs a replay path or it fills up quietly\nB) Explicit cap (default 5), exhausted jobs logged as failed only (human: ~1 h / CC: ~5 min)\n ✅ Bounds the retry storm with a single config value per job type\n ✅ No new queue or alert plumbing to maintain\n ❌ Exhausted jobs are gone after the log line; no replay and nobody is paged\nC) Library default cap, no explicit policy, as the plan reads (human: ~0 / CC: ~0)\n ✅ Zero work; the library's default behavior applies\n ✅ Fewer decisions to document now\n ❌ You do not know today what the default is or where exhausted jobs go; the first poison job teaches you in production\nNet: a cap without a dead-letter bounds the damage but still loses the job; the dead-letter is what makes the loss recoverable.\nHeader: Bounded retries\nOptions:\nA) Cap + dead-letter + alert (recommended)\nPer-job-type max attempts (default 5) in the library config; exhausted jobs land in a dead-letter queue/failed set with payload and last error; alert when depth > 0; documented replay command. Human ~half day / CC ~15 min. Completeness 9/10. Risk: DLQ needs draining discipline.\nB) Cap only, log failures\nPer-job-type max attempts (default 5); exhausted jobs emit a structured failure log and are removed. No DLQ, no alert. Human ~1 h / CC ~5 min. Completeness 6/10. Risk: no replay, nobody paged.\nC) Library default, no policy\nPlan as written: no explicit cap or destination; library defaults apply. Human ~0 / CC ~0. Completeness 3/10. Risk: unknown default discovered in production.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37176837432-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37176837432-t1-pass.json new file mode 100644 index 000000000..d337debb7 --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37176837432-t1-pass.json @@ -0,0 +1,111 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "57a825b033078a7b", + "source": { + "run": "37176837432", + "sha": "7249f014648202840af294b730ac244e03ce514c", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37176837432-1-eval-slices-1/plan-eng-review-1791087925106-l3TgIg" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "f639c2ad-c78e-40de-be08-40b0ac614f36", + "toolUseId": "toolu_01UMCDB7ntEy9gZoZPgr2P6t", + "answeredAt": "2026-10-04T04:28:00.234Z", + "answer": "Library hooks + custom curve (recommended)", + "question": { + "question": "D1 — Use the job library's retry hooks, or build a custom inline scheduler?\nProject/branch/task: gstack-plan-count-jikZRT on main, plan \"Add background job retry framework\".\nELI10: The plan writes its own retry timer inside each of 5 workers even though the job library already ships retry hooks that do the same job. The only stated reason is control over the delay curve, and retry hooks normally let you plug in your own delay function. Writing your own scheduler means you also own attempt counting, persistence of retry state across restarts, and every bug in those.\nStakes if we pick wrong: a hand-rolled scheduler that loses retry state on a worker restart drops or double-runs jobs silently, and you maintain five copies of it.\nRecommendation: A because the library hook almost certainly accepts a custom delay function, which gives the curve control the plan wants with none of the scheduling code [Layer 1: proven built-in over reinvention].\nCompleteness: A=10/10, B=6/10, C=n/a (investigation only)\nPros / cons:\nA) Library retry hooks + custom backoff function (recommended)\n ✅ The library already persists attempt counts and schedules the next run, so restarts do not lose retry state\n ✅ The curve lives in one small pure function that is trivial to unit test and reuse across all 5 workers\n ❌ If the hook does not expose a per-job delay function, you need a shim or a library upgrade before this works\nB) Custom inline scheduler in each worker (as planned)\n ✅ Complete control over timing with no dependency on the library's hook semantics\n ✅ No need to learn or verify the library's retry API before starting\n ❌ You own attempt counting, persistence across restarts and the dispatch loop, in five places (human: ~3 days / CC: ~40 min, plus ongoing maintenance)\nC) Investigate the hook API first (bounded, mechanism stays pending)\n ✅ Settles the one open fact (does the hook take a delay function) before committing to either design\n ✅ Cheap: read one library signature and one existing hook usage (human: ~1h / CC: ~5 min)\n ❌ Delays the decision; the review continues with the mechanism unresolved\nNet: you are trading a few minutes of reading the library's hook API against owning a scheduler forever.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hooks + custom curve (recommended)", + "description": "Use the existing job library's retry hook and pass the exponential-backoff curve in as one small function. The library keeps attempt counts and schedules the retry; workers only supply the curve. Completeness 10/10. Human ~1 day / CC ~10 min. Risk: hook may not accept a per-job delay function; verify at the first worker. Maintenance: one function." + }, + { + "label": "Custom inline scheduler", + "description": "Build the exponential-backoff scheduler inline in each of the 5 workers as the plan proposes. Completeness 6/10. Human ~3 days / CC ~40 min. Risk: retry state lost on restart unless you also persist attempts; five copies to maintain. Maintenance: high." + }, + { + "label": "Investigate hook API first", + "description": "Bounded investigation only: read the library's retry hook signature and one existing usage, confirm whether it accepts a custom delay function. Approves no implementation; R1 stays pending and all other rows stay pending. Human ~1h / CC ~5 min." + } + ] + } + }, + { + "sessionId": "f639c2ad-c78e-40de-be08-40b0ac614f36", + "toolUseId": "toolu_01JeD5JExNnckpJZk87yproz", + "answeredAt": "2026-10-04T04:29:18.756Z", + "answer": "Idempotent delivery + retry (recommended)", + "question": { + "question": "D2 — What delivery guarantee does processWebhookJob keep once it can retry?\nProject/branch/task: gstack-plan-count-jikZRT on main, plan \"Add background job retry framework\".\nELI10: Today a webhook is sent at most once: if the send fails, it is dropped, never duplicated. Adding retries flips that. If the request reaches the receiver but the response times out, the retry sends the same webhook again, and the receiver sees two. Receivers that charge a card or create a record on each webhook will do it twice.\nStakes if we pick wrong: duplicate side effects at every customer receiving your webhooks, with no way for them to tell a retry from a new event.\nRecommendation: A because an idempotency key plus a persisted delivery record keeps retries safe for receivers and costs one small table or column and one header.\nCompleteness: A=10/10, B=7/10, C=3/10\nPros / cons:\nA) Idempotent delivery with per-job delivery record (recommended)\n ✅ Retries recover from transient receiver outages without ever recording two successful deliveries for one job\n ✅ The `Idempotency-Key` header is the standard way webhook receivers dedupe, so well-built receivers drop the duplicate automatically\n ❌ Needs a persisted delivery-state record (pending / delivered / ambiguous) written before each send, plus a migration (human: ~1 day / CC: ~15 min)\nB) Exclude processWebhookJob from the retry framework\n ✅ Preserves today's at-most-once behavior exactly, so the existing contract and its regression test stay simple\n ✅ Smallest change: this worker just does not wire the retry hook\n ❌ Webhooks still vanish on any transient failure, which is the pain this plan exists to fix\nC) At-least-once with no dedup\n ✅ Simplest retry wiring; no delivery table and no header\n ✅ Every job eventually delivers or dead-letters\n ❌ Every ambiguous send produces a visible duplicate at the receiver and pushes the dedup burden onto every customer\nNet: one header and one state record versus either keeping the drops or shipping duplicates to customers.", + "header": "Webhook semantics", + "multiSelect": false, + "options": [ + { + "label": "Idempotent delivery + retry (recommended)", + "description": "Persist a per-job delivery record (pending / delivered / ambiguous) written before each send; send `Idempotency-Key: ` on every attempt; on retry, skip the send if the record is already `delivered`, otherwise resend with the same key so the receiver can dedupe. Completeness 10/10. Human ~1 day / CC ~15 min. Risk: needs a migration; receivers that ignore the header may still see a duplicate after an ambiguous send, which is the documented limit." + }, + { + "label": "Exclude webhooks from retries", + "description": "Keep processWebhookJob on its current at-most-once path and do not wire the retry hook for it. The other 4 workers retry. Completeness 7/10. Human ~0 / CC ~0. Risk: webhook drops on transient failure continue." + }, + { + "label": "At-least-once, no dedup", + "description": "Wire the retry hook for processWebhookJob with no delivery record and no idempotency header; document to receivers that delivery is at-least-once. Completeness 3/10. Human ~1h / CC ~5 min. Risk: duplicate side effects at receivers." + } + ] + } + }, + { + "sessionId": "f639c2ad-c78e-40de-be08-40b0ac614f36", + "toolUseId": "toolu_01RKj7piBifaror2JZpqmmKJ", + "answeredAt": "2026-10-04T04:30:12.398Z", + "answer": "Max attempts + dead-letter (recommended)", + "question": { + "question": "D3 — What happens when a job keeps failing: cap attempts and dead-letter it, cap and drop, or retry forever?\nProject/branch/task: gstack-plan-count-jikZRT on main, plan \"Add background job retry framework\".\nELI10: Some jobs will never succeed: a receiver that returns 410 Gone, a payload that fails validation. The plan never says when to stop retrying or where the job goes when you do. Retry forever and those poison jobs sit in the queue burning worker time. Drop them and you lose the evidence of what went wrong. Dead-letter them and someone at 3am can see the job, the last error, and replay it.\nStakes if we pick wrong: either an ever-growing queue of jobs that will never complete, or silently lost work with no record.\nRecommendation: A because a finite cap plus a dead-letter set is the standard pattern and most job libraries already ship the dead set, so the cost is one setting and one alert.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\nA) Finite max attempts + dead-letter set with alert (recommended)\n ✅ Poison jobs leave the live queue after a bounded number of attempts, so healthy jobs are not starved\n ✅ The dead set keeps the job and its last error, so an operator can inspect and replay instead of guessing\n ❌ Needs a dead set (library-provided or one table) and one alert wired to it (human: ~half day / CC: ~10 min)\nB) Finite max attempts, discard on exhaustion\n ✅ Bounded retries with the least code: one setting and one error log line\n ✅ No dead-letter storage to operate or prune\n ❌ Exhausted jobs are gone; the only trace is a log line, so replay means re-creating the job by hand\nC) Unbounded retries (plan as written)\n ✅ No job is ever declared failed, so nothing needs an operator decision\n ✅ Zero additional code beyond the curve\n ❌ A permanently failing job retries forever, and once the delay cap is hit it consumes a worker slot on every cycle\nNet: a few lines of configuration and one alert versus either an unbounded poison queue or silently discarded work.", + "header": "Terminal policy", + "multiSelect": false, + "options": [ + { + "label": "Max attempts + dead-letter (recommended)", + "description": "Finite max attempts (default 5, per-worker override) on the library retry hook. On exhaustion, move the job and its last error to the library's dead set if it has one, otherwise to a dead-letter table; emit one metric and alert on dead-letter growth. Includes tests for the exhaustion path. Completeness 10/10. Human ~half day / CC ~10 min." + }, + { + "label": "Max attempts, discard", + "description": "Finite max attempts (default 5, per-worker override). On exhaustion, log the job id and last error at error level and drop the job. Includes a test for the exhaustion path. Completeness 6/10. Human ~1h / CC ~5 min. Risk: no replay path." + }, + { + "label": "Unbounded retries", + "description": "No attempt limit; the job retries on the curve until it succeeds. Completeness 2/10. Human ~0 / CC ~0. Risk: poison jobs never leave the queue." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T04:27:24.084Z", + "write": "# Eng review report: Add background job retry framework\n\nReview target (fixed): `PLAN.md` in `gstack-plan-count-jikZRT` (branch `main`, commit e9dce01), plan title \"Add background job retry framework\".\nReport file: this document (destination requested by the user).\nReviewer: /plan-eng-review (gstack), session 692-1791087913-3b742802, 2026-10-04.\n\n## Original plan (unchanged copy)\n\n```markdown\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Review preparation notes\n\n- Repository contents at review time: `PLAN.md`, `CLAUDE.md` only. No worker source, job library, tests or `TODOS.md` are present, so every code-level claim below is calibrated against the plan text and marked unverified where the code would be needed.\n- Test framework: unknown (0 test files detected, no `## Testing` section in CLAUDE.md). The plan proposes no framework, so no framework question is asked.\n- Prior learnings: none found (project-scoped search).\n- Design doc: none. User asked to skip the /office-hours prerequisite.\n- Outside voice: `codex_reviews` is disabled in config.\n- Complexity count (proposed work): 5 worker files changed, 0 new classes/services (scheduler is inline per the plan). Below the 8-file / 2-class gate, so the structure question is skipped.\n\n## Decision ledger\n\n### R1: Retry mechanism — library retry hooks vs custom inline scheduler\nFinding: #1, P1, confidence 8/10, PLAN.md:7-9, reviewer: plan-eng-review (Scope Challenge)\nMotivating lines (PLAN.md:7-9): \"We'll roll a custom exponential-backoff scheduler inline in each worker rather than use the existing job library's built-in retry hooks. Same shape as the library version, but we want full control over the curve.\"\nPlan baseline: original proposal, custom inline exponential-backoff scheduler in each of 5 workers; no prior approval.\nRuntime evidence: unknown. The job library and worker files are not in this repository, so the hook API (whether it accepts a custom delay function) could not be read. The plan itself states the custom version has the \"same shape\" as the library's.\nComparison grid:\n\n| Choice | Current | A) Library hooks + custom curve | B) Custom inline scheduler | C) Investigate hook API first |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler per worker (proposed) | library's retry hook, with the backoff curve supplied as one small function; library owns scheduling, persistence and attempt counting | custom scheduler written inline in each worker, as proposed | bounded: read the library's retry hook signature and confirm it accepts a per-job delay function; mechanism stays pending until confirmed |\n| R2 webhook delivery semantics | unspecified, pending | pending | pending | pending |\n| R3 terminal failure policy (max attempts, dead-letter) | unspecified, pending | pending | pending | pending |\n| R4 jitter | unspecified, pending | pending | pending | pending |\n| R5 delay cap | unspecified, pending | pending | pending | pending |\n| R6 shared retry envelope | duplicated across 5 workers (proposed) | pending | pending | pending |\n| R7 regression coverage for processWebhookJob | none planned | pending | pending | pending |\n| R8 dependency-graph caching | re-fetch and recompute every retry (proposed) | pending | pending | pending |\n\nQuestion D1:\nD1 — Use the job library's retry hooks, or build a custom inline scheduler?\nProject/branch/task: gstack-plan-count-jikZRT on main, plan \"Add background job retry framework\".\nELI10: The plan writes its own retry timer inside each of 5 workers even though the job library already ships retry hooks that do the same job. The only stated reason is control over the delay curve, and retry hooks normally let you plug in your own delay function. Writing your own scheduler means you also own attempt counting, persistence of retry state across restarts, and every bug in those.\nStakes if we pick wrong: a hand-rolled scheduler that loses retry state on a worker restart drops or double-runs jobs silently, and you maintain five copies of it.\nRecommendation: A because the library hook almost certainly accepts a custom delay function, which gives the curve control the plan wants with none of the scheduling code [Layer 1: proven built-in over reinvention].\nCompleteness: A=10/10, B=6/10, C=n/a (investigation only)\nPros / cons:\nA) Library retry hooks + custom backoff function (recommended)\n ✅ The library already persists attempt counts and schedules the next run, so restarts do not lose retry state\n ✅ The curve lives in one small pure function that is trivial to unit test and reuse across all 5 workers\n ❌ If the hook does not expose a per-job delay function, you need a shim or a library upgrade before this works\nB) Custom inline scheduler in each worker (as planned)\n ✅ Complete control over timing with no dependency on the library's hook semantics\n ✅ No need to learn or verify the library's retry API before starting\n ❌ You own attempt counting, persistence across restarts and the dispatch loop, in five places (human: ~3 days / CC: ~40 min, plus ongoing maintenance)\nC) Investigate the hook API first (bounded, mechanism stays pending)\n ✅ Settles the one open fact (does the hook take a delay function) before committing to either design\n ✅ Cheap: read one library signature and one existing hook usage (human: ~1h / CC: ~5 min)\n ❌ Delays the decision; the review continues with the mechanism unresolved\nNet: you are trading a few minutes of reading the library's hook API against owning a scheduler forever.\nHeader: Retry mechanism\nOptions:\nA) Library hooks + custom curve\nUse the existing job library's retry hook and pass the exponential-backoff curve in as one small function. The library keeps attempt counts and schedules the retry; workers only supply the curve. Completeness 10/10. Human ~1 day / CC ~10 min. Risk: hook may not accept a per-job delay function; verify at the first worker. Maintenance: one function.\nB) Custom inline scheduler\nBuild the exponential-backoff scheduler inline in each of the 5 workers as the plan proposes. Completeness 6/10. Human ~3 days / CC ~40 min. Risk: retry state lost on restart unless you also persist attempts; five copies to maintain. Maintenance: high.\nC) Investigate hook API first\nBounded investigation only: read the library's retry hook signature and one existing usage, confirm whether it accepts a custom delay function. Approves no implementation; R1 stays pending and all other rows stay pending. Human ~1h / CC ~5 min.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T04:28:51.035Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hooks + custom curve (D1 answer)\nAccepted scope: replace the proposed inline scheduler with the job library's retry hook in all 5 workers; the exponential-backoff curve is one small pure function passed to the hook; the library owns attempt counting, scheduling and persistence. Includes the unit test for the curve function and the hook wiring per worker. Verify at the first worker that the hook accepts a per-job delay function; if not, stop and reopen R1.\nHistory: none\n\nScope Challenge result: scope accepted as-is (feature list unchanged; D1 changes the mechanism, not the scope). MODE = FULL_REVIEW.\n\n### R2: processWebhookJob delivery semantics under retry\nFinding: #2, P1, confidence 9/10, PLAN.md:17-19, reviewer: plan-eng-review (Architecture)\nMotivating lines (PLAN.md:17-19): \"The existing `processWebhookJob()` flow gets rewritten as part of this change. No regression test for the prior at-most-once delivery guarantee is planned.\"\nPlan baseline: original proposal; the rewrite adds retries to a flow that today guarantees at-most-once delivery, with no statement of the new guarantee. No prior approval.\nRuntime evidence: unknown. `processWebhookJob()` is not in this repository; the at-most-once guarantee is taken from the plan's own statement. The semantic shift is definitional: any retry after an ambiguous send (timeout after the request left the box) can deliver the same webhook twice.\nComparison grid:\n\n| Choice | Current | A) Idempotent delivery + retry | B) Exclude webhooks from retries | C) At-least-once, no dedup |\n|---|---|---|---|---|\n| R1 retry mechanism | approved: library hooks + curve (D1) | unchanged (D1) | unchanged (D1) | unchanged (D1) |\n| R2 webhook delivery semantics | at-most-once today; rewrite adds retries with no stated guarantee | retries allowed; a persisted per-job delivery record plus an `Idempotency-Key` header (job id) guarantees at most one successful delivery is recorded and lets receivers drop duplicates from ambiguous sends | processWebhookJob keeps today's at-most-once path; retry hook not wired for this one worker | retries allowed; duplicates possible; receivers are told delivery is at-least-once |\n| R3 terminal failure policy | unspecified, pending | pending | pending | pending |\n| R4 jitter | unspecified, pending | pending | pending | pending |\n| R5 delay cap | unspecified, pending | pending | pending | pending |\n| R6 shared retry envelope | duplicated across 5 workers (proposed) | pending | pending | pending |\n| R7 regression coverage for processWebhookJob | none planned | pending (contract to protect becomes \"at most one successful delivery per job\") | pending (contract to protect stays \"at most one send per job\") | pending (contract becomes \"every job eventually delivered or dead-lettered\") |\n| R8 dependency-graph caching | re-fetch and recompute every retry (proposed) | pending | pending | pending |\n\nQuestion D2:\nD2 — What delivery guarantee does processWebhookJob keep once it can retry?\nProject/branch/task: gstack-plan-count-jikZRT on main, plan \"Add background job retry framework\".\nELI10: Today a webhook is sent at most once: if the send fails, it is dropped, never duplicated. Adding retries flips that. If the request reaches the receiver but the response times out, the retry sends the same webhook again, and the receiver sees two. Receivers that charge a card or create a record on each webhook will do it twice.\nStakes if we pick wrong: duplicate side effects at every customer receiving your webhooks, with no way for them to tell a retry from a new event.\nRecommendation: A because an idempotency key plus a persisted delivery record keeps retries safe for receivers and costs one small table or column and one header.\nCompleteness: A=10/10, B=7/10, C=3/10\nPros / cons:\nA) Idempotent delivery with per-job delivery record (recommended)\n ✅ Retries recover from transient receiver outages without ever recording two successful deliveries for one job\n ✅ The `Idempotency-Key` header is the standard way webhook receivers dedupe, so well-built receivers drop the duplicate automatically\n ❌ Needs a persisted delivery-state record (pending / delivered / ambiguous) written before each send, plus a migration (human: ~1 day / CC: ~15 min)\nB) Exclude processWebhookJob from the retry framework\n ✅ Preserves today's at-most-once behavior exactly, so the existing contract and its regression test stay simple\n ✅ Smallest change: this worker just does not wire the retry hook\n ❌ Webhooks still vanish on any transient failure, which is the pain this plan exists to fix\nC) At-least-once with no dedup\n ✅ Simplest retry wiring; no delivery table and no header\n ✅ Every job eventually delivers or dead-letters\n ❌ Every ambiguous send produces a visible duplicate at the receiver and pushes the dedup burden onto every customer\nNet: one header and one state record versus either keeping the drops or shipping duplicates to customers.\nHeader: Webhook semantics\nOptions:\nA) Idempotent delivery + retry\nPersist a per-job delivery record (pending / delivered / ambiguous) written before each send; send `Idempotency-Key: ` on every attempt; on retry, skip the send if the record is already `delivered`, otherwise resend with the same key so the receiver can dedupe. Completeness 10/10. Human ~1 day / CC ~15 min. Risk: needs a migration; receivers that ignore the header may still see a duplicate after an ambiguous send, which is the documented limit.\nB) Exclude webhooks from retries\nKeep processWebhookJob on its current at-most-once path and do not wire the retry hook for it. The other 4 workers retry. Completeness 7/10. Human ~0 / CC ~0. Risk: webhook drops on transient failure continue.\nC) At-least-once, no dedup\nWire the retry hook for processWebhookJob with no delivery record and no idempotency header; document to receivers that delivery is at-least-once. Completeness 3/10. Human ~1h / CC ~5 min. Risk: duplicate side effects at receivers.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T04:29:54.828Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Idempotent delivery + retry (D2 answer)\nAccepted scope: processWebhookJob is wired to the retry hook (per D1). Add a persisted per-job delivery record with states pending / delivered / ambiguous, written before each send and updated after; send `Idempotency-Key: ` on every attempt; on retry, skip the send when the record is `delivered`, otherwise resend with the same key. Includes the migration for the record and the tests that prove the contract \"at most one successful delivery recorded per job\" (R7 carries the regression contract). Documented limit: receivers that ignore the header may see one duplicate after an ambiguous send.\nHistory: none\n\n### R3: Terminal failure policy — max attempts and dead-letter handling\nFinding: #3 (Architecture list item 2), P1, confidence 8/10, PLAN.md:7-9, reviewer: plan-eng-review (Architecture)\nMotivating lines (PLAN.md:7-9): \"We'll roll a custom exponential-backoff scheduler inline in each worker ... we want full control over the curve.\" No attempt limit or exhausted-job handling appears anywhere in the plan.\nPlan baseline: original proposal; retries with no stated attempt limit and no stated behavior when retries run out. No prior approval.\nRuntime evidence: unknown. The job library is not in this repository; most job libraries' retry hooks include a max-attempts setting and a dead/failed set, but this one's defaults are unverified.\nComparison grid:\n\n| Choice | Current | A) Max attempts + dead-letter | B) Max attempts, discard | C) Unbounded retries |\n|---|---|---|---|---|\n| R1 retry mechanism | approved (D1) | unchanged (D1) | unchanged (D1) | unchanged (D1) |\n| R2 webhook semantics | approved (D2) | unchanged (D2) | unchanged (D2) | unchanged (D2) |\n| R3 terminal failure policy | unspecified | finite max attempts (default 5, per-worker override, measured in attempts); on exhaustion move the job and its last error to the library's dead set or a dead-letter table and emit one metric/alert | finite max attempts (default 5); on exhaustion log the error and drop the job | no limit; job retries until it succeeds |\n| R4 jitter | unspecified, pending | pending | pending | pending |\n| R5 delay cap | unspecified, pending | pending | pending | pending |\n| R6 shared retry envelope | duplicated across 5 workers (proposed) | pending | pending | pending |\n| R7 regression coverage | pending (contract fixed by D2) | pending | pending | pending |\n| R8 dependency-graph caching | re-fetch and recompute every retry (proposed) | pending | pending | pending |\n\nQuestion D3:\nD3 — What happens when a job keeps failing: cap attempts and dead-letter it, cap and drop, or retry forever?\nProject/branch/task: gstack-plan-count-jikZRT on main, plan \"Add background job retry framework\".\nELI10: Some jobs will never succeed: a receiver that returns 410 Gone, a payload that fails validation. The plan never says when to stop retrying or where the job goes when you do. Retry forever and those poison jobs sit in the queue burning worker time. Drop them and you lose the evidence of what went wrong. Dead-letter them and someone at 3am can see the job, the last error, and replay it.\nStakes if we pick wrong: either an ever-growing queue of jobs that will never complete, or silently lost work with no record.\nRecommendation: A because a finite cap plus a dead-letter set is the standard pattern and most job libraries already ship the dead set, so the cost is one setting and one alert.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\nA) Finite max attempts + dead-letter set with alert (recommended)\n ✅ Poison jobs leave the live queue after a bounded number of attempts, so healthy jobs are not starved\n ✅ The dead set keeps the job and its last error, so an operator can inspect and replay instead of guessing\n ❌ Needs a dead set (library-provided or one table) and one alert wired to it (human: ~half day / CC: ~10 min)\nB) Finite max attempts, discard on exhaustion\n ✅ Bounded retries with the least code: one setting and one error log line\n ✅ No dead-letter storage to operate or prune\n ❌ Exhausted jobs are gone; the only trace is a log line, so replay means re-creating the job by hand\nC) Unbounded retries (plan as written)\n ✅ No job is ever declared failed, so nothing needs an operator decision\n ✅ Zero additional code beyond the curve\n ❌ A permanently failing job retries forever, and once the delay cap is hit it consumes a worker slot on every cycle\nNet: a few lines of configuration and one alert versus either an unbounded poison queue or silently discarded work.\nHeader: Terminal policy\nOptions:\nA) Max attempts + dead-letter\nFinite max attempts (default 5, per-worker override) on the library retry hook. On exhaustion, move the job and its last error to the library's dead set if it has one, otherwise to a dead-letter table; emit one metric and alert on dead-letter growth. Includes tests for the exhaustion path. Completeness 10/10. Human ~half day / CC ~10 min.\nB) Max attempts, discard\nFinite max attempts (default 5, per-worker override). On exhaustion, log the job id and last error at error level and drop the job. Includes a test for the exhaustion path. Completeness 6/10. Human ~1h / CC ~5 min. Risk: no replay path.\nC) Unbounded retries\nNo attempt limit; the job retries on the curve until it succeeds. Completeness 2/10. Human ~0 / CC ~0. Risk: poison jobs never leave the queue.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37179171083-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37179171083-t1-pass.json new file mode 100644 index 000000000..d3978cc8a --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37179171083-t1-pass.json @@ -0,0 +1,122 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "57a825b033078a7b", + "source": { + "run": "37179171083", + "sha": "db91607a7fd9419592eb63e036149920304e32b5", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37179171083-1-eval-slices-1/plan-eng-review-1791090853086-Li4aER" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "fe0156ca-72d4-4134-9411-fa163af1fec9", + "toolUseId": "toolu_01F5N5AsP7A3K1QHxnzZNetz", + "answeredAt": "2026-10-04T05:16:20.889Z", + "answer": "Library hook + custom curve (recommended)", + "question": { + "question": "D1 — Rebuild retry scheduling, or drive the library's retry hook with a custom curve?\nProject/branch/task: main — plan \"Add background job retry framework\", PLAN.md:7-9.\nELI10: The job library you already run has a retry hook. The plan rewrites that machinery by hand inside every worker, admitting it is the \"same shape as the library version\", purely to own the delay curve. Most job libraries let you plug in your own delay function, so you can usually get full control over the curve without rebuilding attempt counting, max-attempt cutoffs, dead-lettering and crash recovery.\nStakes if we pick wrong: a hand-rolled scheduler that loses the attempt counter on a crash either retries forever or drops the job silently, and five copies means five places to get it wrong.\nRecommendation: A because the plan states no behavioral difference from the library version, and a custom backoff function inside the library hook gives the same curve control with the library's persistence and cutoffs intact. [Layer 1: reuse the installed dependency]\nCompleteness: A=10/10, B=4/10, C=n/a (investigation only, approves nothing)\nPros / cons:\nA) Library retry hook + custom backoff function (recommended) (human: ~1 day / CC: ~20 min)\n ✅ Keeps the library's attempt persistence, max-attempt cutoff and dead-letter handling, which the plan would otherwise have to reimplement\n ✅ Full control of the delay curve lives in one small pure function that is trivial to unit test\n ❌ Must confirm the installed library version exposes a custom delay callback; if it does not, the curve may need an upgrade or a thin adapter\nB) Custom inline scheduler per worker (as proposed) (human: ~1 week + ongoing / CC: ~2 h + ongoing)\n ✅ Zero dependence on the library's hook semantics or version\n ✅ Curve, logging and dispatch can be tuned per worker independently\n ❌ Reimplements attempt counting, cutoffs, dead-lettering, jitter and crash-mid-retry handling that the library already provides, and does so five times\nC) Investigate first: bounded probe of the library's hook API (human: ~1 h / CC: ~5 min)\n ✅ Settles the one open fact (does the hook accept a custom delay function?) before committing\n ✅ Costs almost nothing and leaves every other choice untouched\n ❌ Approves no implementation; R1 stays pending until the probe result comes back\nNet: you are trading a small compatibility check against owning (and testing) retry-state machinery five times over.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hook + custom curve (recommended)", + "description": "Use the existing job library's retry hook and supply one custom exponential-backoff function for the delay curve. Keeps the library's attempt persistence, max-attempt cutoff and dead-letter path. Completeness 10/10. Effort human ~1 day / CC ~20 min. Risk: the installed version must expose a custom delay callback. R2-R5 stay pending." + }, + { + "label": "Custom inline scheduler (as planned)", + "description": "Keep the plan as written: a hand-rolled exponential-backoff scheduler inline in each of the 5 workers. Must reimplement attempt counting, cutoffs, dead-lettering, jitter and crash-mid-retry recovery. Completeness 4/10. Effort human ~1 week plus ongoing maintenance / CC ~2 h plus ongoing. R2-R5 stay pending." + }, + { + "label": "Investigate the library hook first", + "description": "Bounded probe only: read the installed job library's retry hook API and confirm whether it accepts a caller-supplied delay function. Approves no implementation. R1 stays pending until the probe reports back; R2-R5 stay pending." + } + ] + } + }, + { + "sessionId": "fe0156ca-72d4-4134-9411-fa163af1fec9", + "toolUseId": "toolu_01PXLYa3xPTYq262TeeqQ3Sb", + "answeredAt": "2026-10-04T05:17:55.867Z", + "answer": "At-least-once + idempotency key (recommended)", + "question": { + "question": "D2 — When webhooks start retrying, what delivery guarantee do receivers get?\nProject/branch/task: main — plan \"Add background job retry framework\", PLAN.md:17-19.\nELI10: Today a webhook is sent at most once: if it fails, it is gone, but the receiver never sees a duplicate. Adding retries flips that. A send that timed out may have actually landed, so the retry delivers the same event twice. Receivers that charge a card or create a record on each webhook will do it twice. The standard fix is an idempotency key: a stable id per delivery that goes out with every attempt, plus a record on our side so a retry never re-sends something already confirmed.\nStakes if we pick wrong: duplicate webhooks cause double side effects at every customer endpoint, and nobody on our side sees an error.\nRecommendation: A because retries on webhooks are only safe when receivers can dedupe, and the key plus delivery record is a small addition that turns the silent-duplicate failure into a defined contract. [Layer 1: standard webhook practice]\nCompleteness: A=10/10, B=7/10, C=3/10\nPros / cons:\nA) At-least-once with idempotency key + delivery record (recommended) (human: ~2 days / CC: ~30 min)\n ✅ Receivers can dedupe on a stable id; the contract becomes explicit and documentable\n ✅ A persisted delivery record stops a retry from re-sending an attempt that already succeeded\n ❌ Adds a small write per attempt and a schema change for the delivery record\nB) Keep retries off for processWebhookJob (human: ~1 h / CC: ~5 min)\n ✅ Preserves today's at-most-once guarantee with no receiver-facing change\n ✅ Smallest possible change to the webhook path\n ❌ Failed webhooks are still dropped silently, which is the pain this plan exists to fix\nC) Retries on, no idempotency key (as proposed) (human: ~0 / CC: ~0)\n ✅ No extra schema or header work\n ✅ Ships fastest\n ❌ Receivers get duplicate deliveries with no way to detect them; double side effects at customer endpoints\nNet: a small id plus one table row per delivery versus silently duplicating customer-visible side effects.", + "header": "Webhook guarantee", + "multiSelect": false, + "options": [ + { + "label": "At-least-once + idempotency key (recommended)", + "description": "Enable retries on processWebhookJob through the library hook (R1). Send a stable per-delivery idempotency id on every attempt, persist a delivery record and check it before each send so a confirmed attempt is never re-sent. Completeness 10/10. Effort human ~2 days / CC ~30 min. R2-R5 stay pending; R3's regression contract will target this guarantee." + }, + { + "label": "Retries off for webhooks only", + "description": "Keep processWebhookJob at-most-once with no retries; the other 4 workers get retries via R1. Failed webhooks remain dropped. Completeness 7/10. Effort human ~1 h / CC ~5 min. R2-R5 stay pending; R3's regression contract will target at-most-once." + }, + { + "label": "Retries on, no idempotency (as planned)", + "description": "Enable retries on processWebhookJob with no idempotency key or delivery record. Receivers may receive duplicates. Completeness 3/10. Effort none beyond R1. R2-R5 stay pending; R3's regression contract will have to document the duplicate-delivery behavior." + } + ] + } + }, + { + "sessionId": "fe0156ca-72d4-4134-9411-fa163af1fec9", + "toolUseId": "toolu_018ey1K9ffDscnzBtDJ24dZ5", + "answeredAt": "2026-10-04T05:18:53.539Z", + "answer": "Full jitter (recommended)", + "question": { + "question": "D3 — Should the backoff curve add random jitter?\nProject/branch/task: main — plan \"Add background job retry framework\", PLAN.md:7-9.\nELI10: When a downstream service goes down, every job that touches it fails in the same second. With a pure exponential curve they all wake up again in the same second, hit the service together and knock it over again. Jitter spreads each retry randomly across its window so the retries trickle in instead of arriving as a wave. It is one line in the delay function.\nStakes if we pick wrong: a recovering dependency gets re-flattened by our own synchronized retry storm, and the outage lasts longer than it needed to.\nRecommendation: A because full jitter is the standard remedy for synchronized retries and costs one random call inside a function that is already being written. [Layer 1]\nCompleteness: A=10/10, B=7/10\nPros / cons:\nA) Full jitter (recommended) (human: ~30 min incl. test / CC: ~3 min)\n ✅ Spreads retries uniformly across the window, so a recovering dependency sees a trickle, not a wave\n ✅ One expression in the delay function; the unit test seeds the RNG and asserts the range\n ❌ Individual retry timing is no longer exactly predictable from the attempt number alone\nB) No jitter, deterministic exponential (human: ~0 / CC: ~0)\n ✅ Retry timing is exactly predictable, which makes log reading trivial\n ✅ Nothing extra to write or test\n ❌ Every job that failed together retries together, recreating the overload that caused the failure\nNet: predictable timestamps in logs versus not DDoSing your own dependencies after an outage.", + "header": "Jitter", + "multiSelect": false, + "options": [ + { + "label": "Full jitter (recommended)", + "description": "Delay = random(0, min(cap, base * 2^attempt)). Unit test seeds the RNG and asserts the delay stays within [0, bound]. Completeness 10/10. Effort human ~30 min / CC ~3 min. R5b (cap) and R5c (exhaustion) stay pending and are asked next." + }, + { + "label": "No jitter", + "description": "Delay = base * 2^attempt, deterministic. Completeness 7/10. Effort none. R5b and R5c stay pending and are asked next." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T05:15:44.250Z", + "write": "# Eng Review — PLAN.md (\"Add background job retry framework\")\n\nTarget (fixed): \nReport file: this file (user-requested destination)\nReview: /plan-eng-review, session 693-1791090838-01c198b5, branch main, commit 2f3995b\nDate: 2026-10-04\nRuntime evidence note: the repository contains only CLAUDE.md and PLAN.md. The worker files, job library and `processWebhookJob()` named in the plan are not present in this checkout, so all runtime evidence below is `unknown` and findings are calibrated against the plan text.\n\n## Original plan (unchanged copy)\n\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n\n## Decision ledger\n\n### R1: Retry mechanism — library retry hooks vs custom inline scheduler\nFinding: S1, P1, confidence 8/10, PLAN.md:7-9, reviewer: plan-eng-review (native)\nPlan baseline: original proposal — \"roll a custom exponential-backoff scheduler inline in each worker rather than use the existing job library's built-in retry hooks. Same shape as the library version, but we want full control over the curve.\" No prior approval.\nRuntime evidence: unknown. The job library and worker files are not in this checkout; whether the library's retry hook accepts a caller-supplied delay function is unverified.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler per worker (proposed) | library retry hook + one custom backoff function | custom inline scheduler per worker (as proposed) | unchanged, pending; bounded probe of the library hook API only |\n| R2 shared retry envelope vs 5 copies | 5 copy-pasted bodies (proposed) | pending | pending | pending |\n| R3 at-most-once regression test for processWebhookJob | none planned (proposed) | pending | pending | pending |\n| R4 dependency-graph cache across retries | none planned (proposed) | pending | pending | pending |\n| R5 backoff curve details (jitter, cap, max attempts) | unspecified | pending | pending | pending |\n\nQuestion D1:\nD1 — Rebuild retry scheduling, or drive the library's retry hook with a custom curve?\nProject/branch/task: main — plan \"Add background job retry framework\", PLAN.md:7-9.\nELI10: The job library you already run has a retry hook. The plan rewrites that machinery by hand inside every worker, admitting it is the \"same shape as the library version\", purely to own the delay curve. Most job libraries let you plug in your own delay function, so you can usually get full control over the curve without rebuilding attempt counting, max-attempt cutoffs, dead-lettering and crash recovery.\nStakes if we pick wrong: a hand-rolled scheduler that loses the attempt counter on a crash either retries forever or drops the job silently, and five copies means five places to get it wrong.\nRecommendation: A because the plan states no behavioral difference from the library version, and a custom backoff function inside the library hook gives the same curve control with the library's persistence and cutoffs intact. [Layer 1: reuse the installed dependency]\nCompleteness: A=10/10, B=4/10, C=n/a (investigation only, approves nothing)\nPros / cons:\nA) Library retry hook + custom backoff function (recommended) (human: ~1 day / CC: ~20 min)\n ✅ Keeps the library's attempt persistence, max-attempt cutoff and dead-letter handling, which the plan would otherwise have to reimplement\n ✅ Full control of the delay curve lives in one small pure function that is trivial to unit test\n ❌ Must confirm the installed library version exposes a custom delay callback; if it does not, the curve may need an upgrade or a thin adapter\nB) Custom inline scheduler per worker (as proposed) (human: ~1 week + ongoing / CC: ~2 h + ongoing)\n ✅ Zero dependence on the library's hook semantics or version\n ✅ Curve, logging and dispatch can be tuned per worker independently\n ❌ Reimplements attempt counting, cutoffs, dead-lettering, jitter and crash-mid-retry handling that the library already provides, and does so five times\nC) Investigate first: bounded probe of the library's hook API (human: ~1 h / CC: ~5 min)\n ✅ Settles the one open fact (does the hook accept a custom delay function?) before committing\n ✅ Costs almost nothing and leaves every other choice untouched\n ❌ Approves no implementation; R1 stays pending until the probe result comes back\nNet: you are trading a small compatibility check against owning (and testing) retry-state machinery five times over.\nHeader: Retry mechanism\nOptions:\nA) Library hook + custom curve (recommended)\nUse the existing job library's retry hook and supply one custom exponential-backoff function for the delay curve. Keeps the library's attempt persistence, max-attempt cutoff and dead-letter path. Completeness 10/10. Effort human ~1 day / CC ~20 min. Risk: the installed version must expose a custom delay callback. R2-R5 stay pending.\nB) Custom inline scheduler (as planned)\nKeep the plan as written: a hand-rolled exponential-backoff scheduler inline in each of the 5 workers. Must reimplement attempt counting, cutoffs, dead-lettering, jitter and crash-mid-retry recovery. Completeness 4/10. Effort human ~1 week plus ongoing maintenance / CC ~2 h plus ongoing. R2-R5 stay pending.\nC) Investigate the library hook first\nBounded probe only: read the installed job library's retry hook API and confirm whether it accepts a caller-supplied delay function. Approves no implementation. R1 stays pending until the probe reports back; R2-R5 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T05:16:59.272Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hook + custom curve — D1 answer\nAccepted scope: Drop the custom inline scheduler from the plan. All 5 workers use the job library's built-in retry hook; the delay curve is one custom exponential-backoff function supplied to that hook. First implementation step: confirm the installed library version exposes a custom delay callback (if not, upgrade or add a thin adapter). Unit tests for the delay function are required proof of this approved behavior and are carried forward without a further question. R2-R5 remain pending.\nHistory: none\n\n### Scope Challenge record\nfeature answers: none proposed (feature list unchanged: retry framework across 5 workers incl. processWebhookJob); structure: complexity gate not tripped (5 files, 0 new classes/services), structure question skipped; accepted scope: R1 approved per D1 (library hook + custom curve replaces custom inline scheduler); pending remedies: R2, R3, R4, R5.\nScope Challenge result: scope reduced per recommendation (custom scheduler build removed; features unchanged).\n" + }, + { + "at": "2026-10-04T05:17:03.002Z", + "old": "## Decision ledger\n", + "new": "## Working plan (current, with approved changes applied)\n\n### Architecture (R1 approved, D1)\nUse the existing job library's built-in retry hooks in all 5 workers. Supply one custom exponential-backoff delay function to the hook for full control of the curve. The custom inline scheduler is removed. First step: confirm the installed library exposes a custom delay callback. Curve details (jitter, cap, max attempts, dead-letter): R5 pending.\n\n### Code quality\nR2 pending (shared envelope vs 5 copies).\n\n### Tests\nUnit tests for the custom delay function: required proof of R1. R3 pending (at-most-once regression contract for `processWebhookJob()`).\n\n### Performance\nR4 pending (dependency-graph cache across retries).\n\n## Decision ledger\n" + }, + { + "at": "2026-10-04T05:17:33.381Z", + "old": "Scope Challenge result: scope reduced per recommendation (custom scheduler build removed; features unchanged).\n", + "new": "Scope Challenge result: scope reduced per recommendation (custom scheduler build removed; features unchanged).\n\n### R6: Webhook delivery guarantee once retries are enabled on processWebhookJob\nFinding: A1, P1, confidence 8/10, PLAN.md:17-19, reviewer: plan-eng-review (native), Section 1 Architecture\nPlan baseline: original proposal — \"The existing `processWebhookJob()` flow gets rewritten as part of this change. No regression test for the prior at-most-once delivery guarantee is planned.\" Retries are added to this job; nothing in the plan preserves or replaces the at-most-once guarantee. R1 approved (D1): retries run through the library hook. No prior approval for R6.\nRuntime evidence: unknown. `processWebhookJob()` and its callers are not in this checkout; whether webhook receivers tolerate duplicate deliveries is unverified.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R6 webhook delivery guarantee | at-most-once today (plan text); proposed: retried with no dedupe, i.e. at-least-once with duplicates | at-least-once with an idempotency key: stable per-delivery id sent with every attempt and a persisted delivery record checked before each send | retries stay off for processWebhookJob; at-most-once preserved; the other 4 workers get retries | retries on with no idempotency key (as proposed); receivers may see duplicates |\n| R1 retry mechanism | approved (D1): library hook + custom curve | unchanged | unchanged | unchanged |\n| R2 shared envelope | pending | pending | pending | pending |\n| R3 regression contract for processWebhookJob | pending | pending (contract shape follows R6) | pending (contract shape follows R6) | pending (contract shape follows R6) |\n| R4 graph cache | pending | pending | pending | pending |\n| R5 curve details | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — When webhooks start retrying, what delivery guarantee do receivers get?\nProject/branch/task: main — plan \"Add background job retry framework\", PLAN.md:17-19.\nELI10: Today a webhook is sent at most once: if it fails, it is gone, but the receiver never sees a duplicate. Adding retries flips that. A send that timed out may have actually landed, so the retry delivers the same event twice. Receivers that charge a card or create a record on each webhook will do it twice. The standard fix is an idempotency key: a stable id per delivery that goes out with every attempt, plus a record on our side so a retry never re-sends something already confirmed.\nStakes if we pick wrong: duplicate webhooks cause double side effects at every customer endpoint, and nobody on our side sees an error.\nRecommendation: A because retries on webhooks are only safe when receivers can dedupe, and the key plus delivery record is a small addition that turns the silent-duplicate failure into a defined contract. [Layer 1: standard webhook practice]\nCompleteness: A=10/10, B=7/10, C=3/10\nPros / cons:\nA) At-least-once with idempotency key + delivery record (recommended) (human: ~2 days / CC: ~30 min)\n ✅ Receivers can dedupe on a stable id; the contract becomes explicit and documentable\n ✅ A persisted delivery record stops a retry from re-sending an attempt that already succeeded\n ❌ Adds a small write per attempt and a schema change for the delivery record\nB) Keep retries off for processWebhookJob (human: ~1 h / CC: ~5 min)\n ✅ Preserves today's at-most-once guarantee with no receiver-facing change\n ✅ Smallest possible change to the webhook path\n ❌ Failed webhooks are still dropped silently, which is the pain this plan exists to fix\nC) Retries on, no idempotency key (as proposed) (human: ~0 / CC: ~0)\n ✅ No extra schema or header work\n ✅ Ships fastest\n ❌ Receivers get duplicate deliveries with no way to detect them; double side effects at customer endpoints\nNet: a small id plus one table row per delivery versus silently duplicating customer-visible side effects.\nHeader: Webhook guarantee\nOptions:\nA) At-least-once + idempotency key (recommended)\nEnable retries on processWebhookJob through the library hook (R1). Send a stable per-delivery idempotency id on every attempt, persist a delivery record and check it before each send so a confirmed attempt is never re-sent. Completeness 10/10. Effort human ~2 days / CC ~30 min. R2-R5 stay pending; R3's regression contract will target this guarantee.\nB) Retries off for webhooks only\nKeep processWebhookJob at-most-once with no retries; the other 4 workers get retries via R1. Failed webhooks remain dropped. Completeness 7/10. Effort human ~1 h / CC ~5 min. R2-R5 stay pending; R3's regression contract will target at-most-once.\nC) Retries on, no idempotency (as planned)\nEnable retries on processWebhookJob with no idempotency key or delivery record. Receivers may receive duplicates. Completeness 3/10. Effort none beyond R1. R2-R5 stay pending; R3's regression contract will have to document the duplicate-delivery behavior.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T05:18:32.600Z", + "old": "R2-R5 stay pending; R3's regression contract will have to document the duplicate-delivery behavior.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "R2-R5 stay pending; R3's regression contract will have to document the duplicate-delivery behavior.\n\nState: approved\nActual answer: A) At-least-once + idempotency key — D2 answer\nAccepted scope: processWebhookJob retries through the library hook (R1). Every attempt carries a stable per-delivery idempotency id (header). A persisted delivery record is written per delivery and checked before each send so a confirmed attempt is never re-sent. Schema change for the delivery record is in scope. R3's regression contract targets this at-least-once-with-idempotency guarantee. R2-R5 remain pending.\nHistory: none\n\n### R5a: Jitter on the backoff curve\nFinding: A2, P2, confidence 7/10, PLAN.md:7-9, reviewer: plan-eng-review (native), Section 1 Architecture\nPlan baseline: original proposal — \"exponential-backoff scheduler ... we want full control over the curve.\" Curve details are unspecified. R1 approved (D1): the curve is one custom delay function supplied to the library hook. No prior approval for jitter.\nRuntime evidence: unknown. No existing curve or worker code in this checkout.\nComparison grid:\n\n| Choice | Current | A | B |\n|---|---|---|---|\n| R5a jitter | unspecified | full jitter: delay = random(0, min(cap, base * 2^attempt)) | none: delay = base * 2^attempt (deterministic) |\n| R5b delay cap | unspecified, pending | pending | pending |\n| R5c exhaustion handling (max attempts, dead-letter) | unspecified, pending | pending | pending |\n| R1 retry mechanism | approved (D1) | unchanged | unchanged |\n| R6 webhook guarantee | approved (D2) | unchanged | unchanged |\n| R2, R3, R4 | pending | pending | pending |\n\nQuestion D3:\nD3 — Should the backoff curve add random jitter?\nProject/branch/task: main — plan \"Add background job retry framework\", PLAN.md:7-9.\nELI10: When a downstream service goes down, every job that touches it fails in the same second. With a pure exponential curve they all wake up again in the same second, hit the service together and knock it over again. Jitter spreads each retry randomly across its window so the retries trickle in instead of arriving as a wave. It is one line in the delay function.\nStakes if we pick wrong: a recovering dependency gets re-flattened by our own synchronized retry storm, and the outage lasts longer than it needed to.\nRecommendation: A because full jitter is the standard remedy for synchronized retries and costs one random call inside a function that is already being written. [Layer 1]\nCompleteness: A=10/10, B=7/10\nPros / cons:\nA) Full jitter (recommended) (human: ~30 min incl. test / CC: ~3 min)\n ✅ Spreads retries uniformly across the window, so a recovering dependency sees a trickle, not a wave\n ✅ One expression in the delay function; the unit test seeds the RNG and asserts the range\n ❌ Individual retry timing is no longer exactly predictable from the attempt number alone\nB) No jitter, deterministic exponential (human: ~0 / CC: ~0)\n ✅ Retry timing is exactly predictable, which makes log reading trivial\n ✅ Nothing extra to write or test\n ❌ Every job that failed together retries together, recreating the overload that caused the failure\nNet: predictable timestamps in logs versus not DDoSing your own dependencies after an outage.\nHeader: Jitter\nOptions:\nA) Full jitter (recommended)\nDelay = random(0, min(cap, base * 2^attempt)). Unit test seeds the RNG and asserts the delay stays within [0, bound]. Completeness 10/10. Effort human ~30 min / CC ~3 min. R5b (cap) and R5c (exhaustion) stay pending and are asked next.\nB) No jitter\nDelay = base * 2^attempt, deterministic. Completeness 7/10. Effort none. R5b and R5c stay pending and are asked next.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T05:18:39.019Z", + "old": "Use the existing job library's built-in retry hooks in all 5 workers. Supply one custom exponential-backoff delay function to the hook for full control of the curve. The custom inline scheduler is removed. First step: confirm the installed library exposes a custom delay callback. Curve details (jitter, cap, max attempts, dead-letter): R5 pending.\n\n### Code quality\nR2 pending (shared envelope vs 5 copies).\n\n### Tests\nUnit tests for the custom delay function: required proof of R1. R3 pending (at-most-once regression contract for `processWebhookJob()`).", + "new": "Use the existing job library's built-in retry hooks in all 5 workers. Supply one custom exponential-backoff delay function to the hook for full control of the curve. The custom inline scheduler is removed. First step: confirm the installed library exposes a custom delay callback. Curve details: R5a jitter, R5b cap, R5c exhaustion handling pending.\n\nWebhook delivery (R6 approved, D2): `processWebhookJob()` retries through the hook. Every attempt carries a stable per-delivery idempotency id; a persisted delivery record is checked before each send so a confirmed attempt is never re-sent. Delivery-record schema change is in scope. Guarantee becomes at-least-once with receiver-side dedupe on the id.\n\n### Code quality\nR2 pending (shared envelope vs 5 copies).\n\n### Tests\nUnit tests for the custom delay function: required proof of R1. Tests for the idempotency id and delivery-record check: required proof of R6. R3 pending (regression contract for `processWebhookJob()` under the new guarantee)." + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37182865432-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37182865432-t1-pass.json new file mode 100644 index 000000000..79a63cbf0 --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37182865432-t1-pass.json @@ -0,0 +1,116 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "c620ebc29ce470c6", + "source": { + "run": "37182865432", + "sha": "44ef18cd0ee8bfb33b3812fbc7f012a2d52f6d9e", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37182865432-1-eval-slices-1/plan-eng-review-1791095412268-YjAH7t" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "d6fa6bb9-d929-41cb-84b0-fdd0c5dc3713", + "toolUseId": "toolu_01GyZu62twEwvYDuxP5KMtVM", + "answeredAt": "2026-10-04T06:33:26.049Z", + "answer": "Library hook + custom curve (recommended)", + "question": { + "question": "D1 — Retry scheduling: library hooks or a custom inline scheduler?\nProject/branch/task: main branch, plan \"Add background job retry framework\" (PLAN.md), touching 5 worker files.\nELI10: Your job library already knows how to retry a failed job later. The plan says: ignore that and write our own timer logic inside each of the 5 workers, because we want to pick the exact wait-time curve. Almost every job library lets you plug in your own curve through a hook, so you get the control without rebuilding scheduling, persistence of attempt counts, and the failed-job UI. Building it ourselves means five copies of code that has to survive worker crashes and restarts.\nStakes if we pick wrong: a hand-rolled scheduler that loses in-flight retries on worker restart or double-fires them, invisible in the library's dashboard; or, if the hook truly cannot express the curve, a library-bound design that needs a hack later.\nRecommendation: A because the stated goal (a custom curve) is exactly what the backoff hook exists for [Layer 1], and the library already handles attempt persistence, dead-lettering and tooling.\nCompleteness: A=9/10, B=5/10, C=n/a (investigation only, approves nothing)\nPros / cons:\nA) Library hook + custom curve (recommended) (human: ~0.5 day / CC: ~10 min)\n ✅ Attempt count, next-run time and failed state persist in the library's store, so retries survive worker restarts\n ✅ The curve lives in one small function; dashboard, dead-letter and metrics come from the library for free\n ❌ If the installed library's hook cannot express the curve (unverified: library not in this repo), we fall back to B after a short delay\nB) Custom inline scheduler (human: ~3 days / CC: ~45 min, plus ongoing maintenance)\n ✅ Total control over timing with no dependency on the library's hook semantics\n ✅ Works the same even if the library's retry feature is weak or absent\n ❌ Re-implements attempt persistence, restart safety and visibility; five copies to keep correct at 3am\nC) Spike the hook first (human: ~30 min / CC: ~5 min)\n ✅ Removes the one real unknown (hook capability) before committing either way\n ✅ Cheap, and produces a concrete snippet to review\n ❌ Approves nothing and delays every downstream choice that depends on the mechanism\nNet: trading a small chance of hook limitations against owning scheduling correctness in five places.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hook + custom curve (recommended)", + "description": "Use the job library's retry hook with one custom backoff function that produces the desired curve. Remove the inline scheduler from the plan; workers only mark which errors are retryable. Attempt count, next-run time and failed state persist in the library's store, so retries survive worker restarts. Dashboard, dead-letter and metrics come from the library for free. Con: if the installed library's hook cannot express the curve (unverified, library not in this repo), we fall back to B after a short delay. Effort human ~0.5 day / CC ~10 min. R2-R7 stay pending." + }, + { + "label": "Custom inline scheduler", + "description": "Keep the plan as written: a hand-rolled exponential-backoff scheduler inline in each of the 5 workers. Total control over timing, no dependency on the library's hook semantics, and works even if the library's retry feature is weak. Con: re-implements attempt persistence, restart safety and visibility, and leaves five copies to keep correct at 3am. Effort human ~3 days / CC ~45 min plus ongoing maintenance. R2-R7 stay pending." + }, + { + "label": "Spike the hook first", + "description": "Decide nothing now. Run a bounded 30-minute spike that confirms whether the library's retry hook can express the desired curve, then re-ask this question with the snippet in hand. Plan text unchanged meanwhile. Cheap and removes the one real unknown, but approves nothing and delays every downstream choice that depends on the mechanism. Effort human ~30 min / CC ~5 min. R2-R7 stay pending." + } + ] + } + }, + { + "sessionId": "d6fa6bb9-d929-41cb-84b0-fdd0c5dc3713", + "toolUseId": "toolu_01DcYAX5rQjxzNSR7VQpxKjq", + "answeredAt": "2026-10-04T06:34:45.901Z", + "answer": "At-least-once + idempotency key (recommended)", + "question": { + "question": "D2 — Webhook delivery: what guarantee do receivers get once we retry?\nProject/branch/task: main branch, plan \"Add background job retry framework\" (PLAN.md), the `processWebhookJob()` rewrite.\nELI10: Today a webhook is sent at most once: if the job dies, the receiver gets nothing, but never two copies. Once we retry, a job that sent the webhook and then crashed before recording success will send it again. Receivers now have to cope with duplicates. The standard fix is to stamp every send with the same idempotency key across attempts so receivers can drop the second copy. The plan is silent on this.\nStakes if we pick wrong: duplicate webhooks trigger double side effects at receivers (double emails, double charges, double syncs), and the first support ticket arrives before anyone knows the guarantee changed.\nRecommendation: A because at-least-once with a stable idempotency key is the standard contract for retried webhooks and costs one header plus a doc line.\nCompleteness: A=9/10, B=7/10, C=3/10\nPros / cons:\nA) At-least-once + idempotency key (recommended) (human: ~0.5 day / CC: ~10 min)\n ✅ Webhook failures are actually retried, and receivers have a documented way to drop duplicates\n ✅ Key derived from the job id, so it is stable across attempts with no new storage\n ❌ Receivers that ignore the header still see duplicates; the contract change must be communicated\nB) Keep webhook at-most-once (opt out) (human: ~1 hour / CC: ~5 min)\n ✅ Zero behavior change for receivers; no contract to communicate\n ✅ Smallest diff for the webhook path; other 4 workers still gain retries\n ❌ Transient webhook failures stay lost, which is the problem the plan set out to fix\nC) At-least-once, no key (as planned) (human: ~0 / CC: ~0)\n ✅ No extra work on the send path\n ✅ Receivers already deduping by payload are unaffected\n ❌ Silent contract change; receivers without dedup get duplicates and nothing tells them\nNet: trading one header and a doc line against either lost webhooks or silent duplicates.", + "header": "Webhook semantics", + "multiSelect": false, + "options": [ + { + "label": "At-least-once + idempotency key (recommended)", + "description": "Retry `processWebhookJob()` and stamp every send with a stable idempotency key derived from the job id (same value on every attempt) in a documented header. Document the at-least-once contract for receivers. Webhook failures are actually retried and receivers have a standard way to drop duplicates; no new storage needed. Con: receivers that ignore the header still see duplicates, so the contract change must be communicated. Effort human ~0.5 day / CC ~10 min. R1 fixed (D1); R3-R8 stay pending." + }, + { + "label": "Keep webhook at-most-once (opt out)", + "description": "Exclude `processWebhookJob()` from the retry framework so its at-most-once guarantee is preserved exactly; the other 4 workers gain retries. Zero behavior change for receivers and the smallest diff on the webhook path. Con: transient webhook failures stay lost, which is part of what the plan set out to fix. Effort human ~1 hour / CC ~5 min. R1 fixed (D1); R3-R8 stay pending." + }, + { + "label": "At-least-once, no key (as planned)", + "description": "Retry `processWebhookJob()` with no idempotency key and no contract change communicated, as the plan currently reads. No extra work on the send path; receivers that already dedup by payload are unaffected. Con: silent contract change; receivers without dedup get duplicates and nothing tells them. Effort none. R1 fixed (D1); R3-R8 stay pending." + } + ] + } + }, + { + "sessionId": "d6fa6bb9-d929-41cb-84b0-fdd0c5dc3713", + "toolUseId": "toolu_013V7Ujf6tnhVQizkCyNfoKw", + "answeredAt": "2026-10-04T06:35:41.544Z", + "answer": "Cap 5 attempts + dead-letter + alert (recommended)", + "question": { + "question": "D3 — How many times do we retry, and what happens when we give up?\nProject/branch/task: main branch, plan \"Add background job retry framework\" (PLAN.md), all 5 workers.\nELI10: Exponential backoff says how long to wait between tries, but not when to stop. Without a cap, a job whose dependency is permanently broken retries forever, and nobody is told. The usual answer is a maximum number of attempts, after which the job is parked in a failed set (dead-letter) and an alert fires so a human can look at it in the morning.\nStakes if we pick wrong: either zombie jobs that retry for days and quietly eat worker capacity, or jobs that silently vanish into a failed set nobody watches.\nRecommendation: A because a cap plus an exhaustion alert is the only option where a permanently failing job becomes a visible, bounded event.\nCompleteness: A=9/10, B=7/10, C=3/10\nPros / cons:\nA) Cap 5 attempts + dead-letter + alert (recommended) (human: ~0.5 day / CC: ~10 min)\n ✅ Every job's retry cost is bounded and exhaustion becomes a visible metric per worker\n ✅ Dead-lettered jobs are inspectable and replayable from the library's failed set\n ❌ One more metric/alert to wire into the existing monitoring stack\nB) Cap 5 attempts, library default on exhaustion (human: ~1 hour / CC: ~5 min)\n ✅ Retry cost is bounded with almost no extra code beyond a config value\n ✅ Failed jobs still land in the library's failed set for manual inspection\n ❌ Nobody is told when jobs are exhausted; the failed set grows unwatched\nC) No cap, as planned (human: ~0 / CC: ~0)\n ✅ No config or monitoring work at all\n ✅ A job eventually succeeds if the dependency ever recovers\n ❌ Permanently broken jobs retry forever, consuming worker slots with no signal\nNet: trading one metric and a config value against unbounded or invisible failure.", + "header": "Retry bounds", + "multiSelect": false, + "options": [ + { + "label": "Cap 5 attempts + dead-letter + alert (recommended)", + "description": "Set a maximum attempt count per worker type, default 5, configurable. When a job exhausts its attempts it lands in the library's failed/dead-letter set and a `job_retries_exhausted` metric/alert fires carrying the worker name and job id. Every job's retry cost is bounded, exhaustion is visible per worker, and dead-lettered jobs are inspectable and replayable. Con: one more metric/alert to wire into the existing monitoring stack. Effort human ~0.5 day / CC ~10 min. R1, R2 fixed (D1, D2); R4-R8 stay pending." + }, + { + "label": "Cap 5 attempts, library default on exhaustion", + "description": "Set the same maximum attempt count (default 5, configurable) and rely on the library's default behavior when attempts are exhausted: the job sits in the failed set with no metric or alert. Retry cost is bounded with almost no code beyond a config value, and failed jobs remain inspectable. Con: nobody is told when jobs are exhausted, so the failed set grows unwatched. Effort human ~1 hour / CC ~5 min. R1, R2 fixed (D1, D2); R4-R8 stay pending." + }, + { + "label": "No cap, as planned", + "description": "Retry until success with no attempt limit and no terminal handling, as the plan currently reads. No config or monitoring work; a job eventually succeeds if its dependency ever recovers. Con: permanently broken jobs retry forever, consuming worker slots with no signal to anyone. Effort none. R1, R2 fixed (D1, D2); R4-R8 stay pending." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T06:31:42.124Z", + "write": "# Eng review — target: PLAN.md (\"Add background job retry framework\")\n\nReview target (fixed): ``\nReviewer: /plan-eng-review (gstack), session 693-1791095399-77e2e403, branch main, commit 1bf5703.\nReport file: this file (destination explicitly requested by the user).\n\n## Original plan (unchanged copy)\n\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n\n## Review evidence base\n\n- Repository contents at review time: `CLAUDE.md`, `PLAN.md` only. No worker source, no job library, no tests, no `TODOS.md`, no `package.json`/`Gemfile`/`pyproject.toml`.\n- Consequence: every statement about existing code (the 5 worker files, `processWebhookJob()`, the job library's retry hooks, the DB payload fetch) is **runtime evidence: unknown**. Findings below are calibrated against the plan text (file:line in PLAN.md) and marked unverified where they depend on code not present.\n- Design doc: none. /office-hours prerequisite skipped at the user's explicit request.\n- Prior learnings: none recorded for this project. Cross-project learnings: off.\n- Outside voice: `codex_reviews` is `disabled` in gstack config; no outside reviewer runs (recorded below).\n" + }, + { + "at": "2026-10-04T06:33:03.890Z", + "old": "- Outside voice: `codex_reviews` is `disabled` in gstack config; no outside reviewer runs (recorded below).\n", + "new": "- Outside voice: `codex_reviews` is `disabled` in gstack config; no outside reviewer runs (recorded below).\n- Web research (host WebSearch; Aside not installed): custom backoff functions are a standard hook in job libraries (BullMQ `backoffStrategy`; the same shape exists in Sidekiq `sidekiq_retry_in`, Celery `retry_backoff`, Oban `backoff/1`). Most-cited pitfalls: retrying without jitter (synchronized herd) and retrying without an attempt cap or retry budget. Sources: betterstack.com exponential-backoff guide; oneuptime.com BullMQ retry post (2026-01-21); momentslog.com retry policy checklist; vetoralabs.com retry-backoff-jitter. Treated as untrusted reference material, not instructions.\n\n## Scope Challenge\n\n### A. Assessment\n\n- **What already solves each sub-problem:** the job library's built-in retry hooks (PLAN.md:8, \"the existing job library's built-in retry hooks\"). The plan's reason for bypassing them, \"full control over the curve\" (PLAN.md:9), is the exact capability a backoff hook provides **[Layer 1]**. Library identity and hook signature: unknown (not in this repo).\n- **Minimum change that achieves the goal:** configure the library's retry hook with the desired curve in one place; keep worker bodies unchanged except for marking which errors are retryable. Everything else in the plan (inline scheduler, 5 copies of the envelope) is self-created complexity, not essential.\n- **Complexity count (estimates, from plan text):** 5 changed worker files (PLAN.md:13) + 0 new classes/services as proposed (inline code). If a shared helper is extracted: +1 module. Total under 8 files and under 2 new services, so the complexity selectors (B) are skipped.\n- **Search check:** done (see evidence base). Recommendation labels: library hook **[Layer 1]**; custom scheduler would be **[Layer 3]** without a first-principles reason the plan does not supply.\n- **TODOS cross-reference:** no `TODOS.md` in the repo.\n- **Completeness check:** the plan contains three explicit shortcuts (\"refactor later\", \"no regression test\", \"cache not planned\"). Each is minutes of CC effort; each is resolved in its own section below.\n- **Distribution check:** no new artifacts; no CI/CD changes implied.\n\n### B. Complexity selectors\n\nSkipped: under 8 files and under 2 new classes/services.\n\n### C. Findings\n\n1. **[P1] (confidence: 7/10) PLAN.md:7-9 — Plan rejects the library's built-in retry hooks for a hand-rolled inline scheduler without evidence the hooks cannot express the curve.** Motivating lines: \"roll a custom exponential-backoff scheduler inline in each worker rather than use the existing job library's built-in retry hooks. Same shape as the library version, but we want full control over the curve.\" Reviewer: /plan-eng-review (native). Confidence held at 7 (not 9) because the library is not present in this repo; its hook capability is unverified.\n\nRemedy: R1 (Decision ledger). Scope Challenge result: pending R1 answer; no scope cut proposed, so the result will be `scope accepted as-is` unless R1 changes the file set.\n\n## Decision ledger\n\n### R1: Retry scheduling mechanism (library hook vs custom inline scheduler)\nFinding: SC-1, P1, confidence 7/10, PLAN.md:7-9, reviewer /plan-eng-review (native)\nPlan baseline: original proposal, no prior approval: custom exponential-backoff scheduler inline in each of the 5 workers (PLAN.md:7-9)\nRuntime evidence: unknown. No job library, worker source or config present in this repository. Web research confirms custom backoff hooks are standard across major job libraries, but the installed library is unidentified.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry scheduling mechanism | custom inline scheduler in each worker (PLAN.md:7-9, original proposal) | library retry hook + one custom backoff function; inline scheduler removed from plan | custom inline scheduler, as proposed | undecided; bounded 30-minute spike confirming the hook can express the curve; plan unchanged |\n| R2 webhook delivery semantics / idempotency | unspecified, pending | pending | pending | pending |\n| R3 attempt cap + terminal (dead-letter) handling | unspecified, pending | pending | pending | pending |\n| R4 jitter | unspecified, pending | pending | pending | pending |\n| R5 shared retry envelope | duplicated across 5 files, \"refactor later\" (PLAN.md:12-14), pending | pending | pending | pending |\n| R6 regression contract for processWebhookJob | none planned (PLAN.md:17-19), pending | pending | pending | pending |\n| R7 dependency-graph cache on retry | not planned (PLAN.md:22-24), pending | pending | pending | pending |\n\nQuestion D1:\nD1 — Retry scheduling: library hooks or a custom inline scheduler?\nProject/branch/task: main branch, plan \"Add background job retry framework\" (PLAN.md), touching 5 worker files.\nELI10: Your job library already knows how to retry a failed job later. The plan says: ignore that and write our own timer logic inside each of the 5 workers, because we want to pick the exact wait-time curve. Almost every job library lets you plug in your own curve through a hook, so you get the control without rebuilding scheduling, persistence of attempt counts, and the failed-job UI. Building it ourselves means five copies of code that has to survive worker crashes and restarts.\nStakes if we pick wrong: a hand-rolled scheduler that loses in-flight retries on worker restart or double-fires them, invisible in the library's dashboard; or, if the hook truly cannot express the curve, a library-bound design that needs a hack later.\nRecommendation: A because the stated goal (a custom curve) is exactly what the backoff hook exists for [Layer 1], and the library already handles attempt persistence, dead-lettering and tooling.\nCompleteness: A=9/10, B=5/10, C=n/a (investigation only, approves nothing)\nPros / cons:\nA) Library hook + custom curve (recommended) (human: ~0.5 day / CC: ~10 min)\n ✅ Attempt count, next-run time and failed state persist in the library's store, so retries survive worker restarts\n ✅ The curve lives in one small function; dashboard, dead-letter and metrics come from the library for free\n ❌ If the installed library's hook cannot express the curve (unverified: library not in this repo), we fall back to B after a short delay\nB) Custom inline scheduler (human: ~3 days / CC: ~45 min, plus ongoing maintenance)\n ✅ Total control over timing with no dependency on the library's hook semantics\n ✅ Works the same even if the library's retry feature is weak or absent\n ❌ Re-implements attempt persistence, restart safety and visibility; five copies to keep correct at 3am\nC) Spike the hook first (human: ~30 min / CC: ~5 min)\n ✅ Removes the one real unknown (hook capability) before committing either way\n ✅ Cheap, and produces a concrete snippet to review\n ❌ Approves nothing and delays every downstream choice that depends on the mechanism\nNet: trading a small chance of hook limitations against owning scheduling correctness in five places.\nHeader: Retry mechanism\nOptions:\nA) Library hook + custom curve (recommended)\nUse the job library's retry hook with one custom backoff function that produces the desired curve. Remove the inline scheduler from the plan; workers only mark which errors are retryable. Attempt count, next-run time and failed state persist in the library's store, so retries survive worker restarts. Dashboard, dead-letter and metrics come from the library for free. Con: if the installed library's hook cannot express the curve (unverified, library not in this repo), we fall back to B after a short delay. Effort human ~0.5 day / CC ~10 min. R2-R7 stay pending.\nB) Custom inline scheduler\nKeep the plan as written: a hand-rolled exponential-backoff scheduler inline in each of the 5 workers. Total control over timing, no dependency on the library's hook semantics, and works even if the library's retry feature is weak. Con: re-implements attempt persistence, restart safety and visibility, and leaves five copies to keep correct at 3am. Effort human ~3 days / CC ~45 min plus ongoing maintenance. R2-R7 stay pending.\nC) Spike the hook first\nDecide nothing now. Run a bounded 30-minute spike that confirms whether the library's retry hook can express the desired curve, then re-ask this question with the snippet in hand. Plan text unchanged meanwhile. Cheap and removes the one real unknown, but approves nothing and delays every downstream choice that depends on the mechanism. Effort human ~30 min / CC ~5 min. R2-R7 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T06:34:27.837Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hook + custom curve (D1, user answer)\nAccepted scope: Replace the inline per-worker scheduler (PLAN.md:7-9) with the job library's retry hook plus one custom backoff function producing the desired curve. Workers only mark which errors are retryable. Attempt count, next-run time and failed state persist in the library's store. Includes the code, tests and plan text for this mechanism. R2-R7 remain pending.\nHistory: none\n\n**Scope Challenge result:** `scope accepted as-is` (no feature cut; D1 changes the arrangement, not the scope). MODE = FULL_REVIEW.\n\n## Section 1: Architecture review\n\nFindings (most severe first):\n\n1. **[P1] (confidence: 7/10) PLAN.md:17-19 — Retrying `processWebhookJob()` changes delivery from at-most-once to at-least-once; receivers will see duplicate webhooks unless each send carries an idempotency key.** Motivating lines: \"The existing `processWebhookJob()` flow gets rewritten as part of this change. No regression test for the prior at-most-once delivery guarantee is planned.\" The plan names neither the new guarantee nor a dedup key. Remedy: R2. Confidence 7: the plan states the current guarantee; the send path is not in this repo.\n2. **[P1] (confidence: 8/10) PLAN.md:7-9 — No maximum attempt count or terminal handling (dead-letter) is named.** Motivating line: \"roll a custom exponential-backoff scheduler\". An uncapped exponential curve retries forever and hides permanently broken jobs. Remedy: R3.\n3. **[P2] (confidence: 8/10) PLAN.md:7 — No jitter.** Motivating line: \"custom exponential-backoff scheduler\". After a shared dependency outage every failed job across five workers retries on the same tick. Remedy: R4.\n4. **[P2] (confidence: 6/10) PLAN.md:7-9 — No retryable vs permanent error classification.** Medium confidence, verify against the worker bodies. A 4xx from a webhook receiver or a validation error would consume the whole retry budget. Remedy: R8.\n\nSecurity/data boundaries: no new auth or data-access surface; the retry path re-reads the job payload from the DB (PLAN.md:22), so payload access is unchanged. Distribution: no new artifacts.\n\nRealistic production failure per new path: (a) webhook receiver times out after processing, job retries, receiver gets a duplicate charge/notification (finding 1); (b) downstream API hard-down for an hour, all jobs exhaust attempts simultaneously with no dead-letter (findings 2, 3).\n\n### R2: Webhook delivery semantics under retry (idempotency)\nFinding: ARCH-1, P1, confidence 7/10, PLAN.md:17-19, reviewer /plan-eng-review (native)\nPlan baseline: original proposal, no prior approval: `processWebhookJob()` rewritten to retry; current guarantee stated as at-most-once; no idempotency key or dedup named (PLAN.md:17-19)\nRuntime evidence: unknown. `processWebhookJob()` and its send path are not in this repository. Whether receivers already dedup is unknown.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry scheduling mechanism | approved A (D1): library hook + custom curve | unchanged (D1) | unchanged (D1) | unchanged (D1) |\n| R2 webhook delivery semantics / idempotency | at-most-once today; plan makes it at-least-once with no key (PLAN.md:17-19) | at-least-once; every send carries a stable idempotency key (derived from job id, same across attempts) in a documented header; documented for receivers | at-most-once preserved: `processWebhookJob()` opts out of retries; the other 4 workers retry | at-least-once with no idempotency key, as planned |\n| R3 attempt cap + terminal handling | unspecified, pending | pending | pending | pending |\n| R4 jitter | unspecified, pending | pending | pending | pending |\n| R5 shared retry envelope | pending | pending | pending | pending |\n| R6 regression contract for processWebhookJob | pending | pending | pending | pending |\n| R7 dependency-graph cache | pending | pending | pending | pending |\n| R8 retryable vs permanent error classification | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — Webhook delivery: what guarantee do receivers get once we retry?\nProject/branch/task: main branch, plan \"Add background job retry framework\" (PLAN.md), the `processWebhookJob()` rewrite.\nELI10: Today a webhook is sent at most once: if the job dies, the receiver gets nothing, but never two copies. Once we retry, a job that sent the webhook and then crashed before recording success will send it again. Receivers now have to cope with duplicates. The standard fix is to stamp every send with the same idempotency key across attempts so receivers can drop the second copy. The plan is silent on this.\nStakes if we pick wrong: duplicate webhooks trigger double side effects at receivers (double emails, double charges, double syncs), and the first support ticket arrives before anyone knows the guarantee changed.\nRecommendation: A because at-least-once with a stable idempotency key is the standard contract for retried webhooks and costs one header plus a doc line.\nCompleteness: A=9/10, B=7/10, C=3/10\nPros / cons:\nA) At-least-once + idempotency key (recommended) (human: ~0.5 day / CC: ~10 min)\n ✅ Webhook failures are actually retried, and receivers have a documented way to drop duplicates\n ✅ Key derived from the job id, so it is stable across attempts with no new storage\n ❌ Receivers that ignore the header still see duplicates; the contract change must be communicated\nB) Keep webhook at-most-once (opt out) (human: ~1 hour / CC: ~5 min)\n ✅ Zero behavior change for receivers; no contract to communicate\n ✅ Smallest diff for the webhook path; other 4 workers still gain retries\n ❌ Transient webhook failures stay lost, which is the problem the plan set out to fix\nC) At-least-once, no key (as planned) (human: ~0 / CC: ~0)\n ✅ No extra work on the send path\n ✅ Receivers already deduping by payload are unaffected\n ❌ Silent contract change; receivers without dedup get duplicates and nothing tells them\nNet: trading one header and a doc line against either lost webhooks or silent duplicates.\nHeader: Webhook semantics\nOptions:\nA) At-least-once + idempotency key (recommended)\nRetry `processWebhookJob()` and stamp every send with a stable idempotency key derived from the job id (same value on every attempt) in a documented header. Document the at-least-once contract for receivers. Webhook failures are actually retried and receivers have a standard way to drop duplicates; no new storage needed. Con: receivers that ignore the header still see duplicates, so the contract change must be communicated. Effort human ~0.5 day / CC ~10 min. R1 fixed (D1); R3-R8 stay pending.\nB) Keep webhook at-most-once (opt out)\nExclude `processWebhookJob()` from the retry framework so its at-most-once guarantee is preserved exactly; the other 4 workers gain retries. Zero behavior change for receivers and the smallest diff on the webhook path. Con: transient webhook failures stay lost, which is part of what the plan set out to fix. Effort human ~1 hour / CC ~5 min. R1 fixed (D1); R3-R8 stay pending.\nC) At-least-once, no key (as planned)\nRetry `processWebhookJob()` with no idempotency key and no contract change communicated, as the plan currently reads. No extra work on the send path; receivers that already dedup by payload are unaffected. Con: silent contract change; receivers without dedup get duplicates and nothing tells them. Effort none. R1 fixed (D1); R3-R8 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T06:35:23.510Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) At-least-once + idempotency key (D2, user answer)\nAccepted scope: `processWebhookJob()` is retried under the R1 mechanism. Every webhook send carries a stable idempotency key derived from the job id (identical on every attempt) in a documented header. The at-least-once contract is documented for receivers. Includes the send-path code, tests for key stability across attempts, and the doc line. R3-R8 remain pending.\nHistory: none\n\n### R3: Attempt cap and terminal (dead-letter) handling\nFinding: ARCH-2, P1, confidence 8/10, PLAN.md:7-9, reviewer /plan-eng-review (native)\nPlan baseline: original proposal, no prior approval: exponential backoff with no attempt cap, no terminal handling and no alert named (PLAN.md:7-9)\nRuntime evidence: unknown. The library's default attempt limit and failed-job handling are not observable here.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry scheduling mechanism | approved A (D1) | unchanged (D1) | unchanged (D1) | unchanged (D1) |\n| R2 webhook delivery semantics | approved A (D2) | unchanged (D2) | unchanged (D2) | unchanged (D2) |\n| R3 attempt cap + terminal handling | unspecified (PLAN.md:7-9) | max attempts per worker type, default 5 (configurable); on exhaustion the job lands in the library's failed/dead-letter set and a `job_retries_exhausted` metric/alert fires with worker name and job id | max attempts per worker type, default 5 (configurable); on exhaustion the library's default failed-set behavior, no metric or alert | no cap: retry until success, as the plan reads |\n| R4 jitter | unspecified, pending | pending | pending | pending |\n| R5 shared retry envelope | pending | pending | pending | pending |\n| R6 regression contract for processWebhookJob | pending | pending | pending | pending |\n| R7 dependency-graph cache | pending | pending | pending | pending |\n| R8 retryable vs permanent error classification | pending | pending | pending | pending |\n\nQuestion D3:\nD3 — How many times do we retry, and what happens when we give up?\nProject/branch/task: main branch, plan \"Add background job retry framework\" (PLAN.md), all 5 workers.\nELI10: Exponential backoff says how long to wait between tries, but not when to stop. Without a cap, a job whose dependency is permanently broken retries forever, and nobody is told. The usual answer is a maximum number of attempts, after which the job is parked in a failed set (dead-letter) and an alert fires so a human can look at it in the morning.\nStakes if we pick wrong: either zombie jobs that retry for days and quietly eat worker capacity, or jobs that silently vanish into a failed set nobody watches.\nRecommendation: A because a cap plus an exhaustion alert is the only option where a permanently failing job becomes a visible, bounded event.\nCompleteness: A=9/10, B=7/10, C=3/10\nPros / cons:\nA) Cap 5 attempts + dead-letter + alert (recommended) (human: ~0.5 day / CC: ~10 min)\n ✅ Every job's retry cost is bounded and exhaustion becomes a visible metric per worker\n ✅ Dead-lettered jobs are inspectable and replayable from the library's failed set\n ❌ One more metric/alert to wire into the existing monitoring stack\nB) Cap 5 attempts, library default on exhaustion (human: ~1 hour / CC: ~5 min)\n ✅ Retry cost is bounded with almost no extra code beyond a config value\n ✅ Failed jobs still land in the library's failed set for manual inspection\n ❌ Nobody is told when jobs are exhausted; the failed set grows unwatched\nC) No cap, as planned (human: ~0 / CC: ~0)\n ✅ No config or monitoring work at all\n ✅ A job eventually succeeds if the dependency ever recovers\n ❌ Permanently broken jobs retry forever, consuming worker slots with no signal\nNet: trading one metric and a config value against unbounded or invisible failure.\nHeader: Retry bounds\nOptions:\nA) Cap 5 attempts + dead-letter + alert (recommended)\nSet a maximum attempt count per worker type, default 5, configurable. When a job exhausts its attempts it lands in the library's failed/dead-letter set and a `job_retries_exhausted` metric/alert fires carrying the worker name and job id. Every job's retry cost is bounded, exhaustion is visible per worker, and dead-lettered jobs are inspectable and replayable. Con: one more metric/alert to wire into the existing monitoring stack. Effort human ~0.5 day / CC ~10 min. R1, R2 fixed (D1, D2); R4-R8 stay pending.\nB) Cap 5 attempts, library default on exhaustion\nSet the same maximum attempt count (default 5, configurable) and rely on the library's default behavior when attempts are exhausted: the job sits in the failed set with no metric or alert. Retry cost is bounded with almost no code beyond a config value, and failed jobs remain inspectable. Con: nobody is told when jobs are exhausted, so the failed set grows unwatched. Effort human ~1 hour / CC ~5 min. R1, R2 fixed (D1, D2); R4-R8 stay pending.\nC) No cap, as planned\nRetry until success with no attempt limit and no terminal handling, as the plan currently reads. No config or monitoring work; a job eventually succeeds if its dependency ever recovers. Con: permanently broken jobs retry forever, consuming worker slots with no signal to anyone. Effort none. R1, R2 fixed (D1, D2); R4-R8 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37186854666-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37186854666-t1-pass.json new file mode 100644 index 000000000..bc1d2577f --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37186854666-t1-pass.json @@ -0,0 +1,122 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "a3e4c3429b2cfef8", + "source": { + "run": "37186854666", + "sha": "b24b6d8fcf57db7e9c8ded236d31b8a5d0b4504a", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37186854666-1-eval-slices-1/plan-eng-review-1791100200678-EiJ8dB" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "4e4fc81f-c399-45c6-9e1c-9d6dde841f1f", + "toolUseId": "toolu_01AZdnykhPY3Sq6GXwv2uExQ", + "answeredAt": "2026-10-04T07:53:31.688Z", + "answer": "Library hooks + custom curve (recommended)", + "question": { + "question": "D1 — Retry mechanism: library built-in hooks or a custom inline scheduler?\nProject/branch/task: main — plan \"Add background job retry framework\", reviewing PLAN.md.\nELI10: The job library already knows how to retry a job later with a growing delay. The plan rebuilds that from scratch inside each of five workers because the team wants to pick the exact delay curve. Most libraries let you hand them your own delay function, so you keep the curve and drop the rebuild. The stakes: a homegrown scheduler must also remember attempt counts across restarts, stop after N tries, and report failures, or jobs retry forever or vanish silently.\nStakes if we pick wrong: Either we ship five slightly different retry engines that drift and lose attempt state on a worker restart, or we discover late that the library cannot express the curve and have to rip out the hooks.\nRecommendation: A because the plan admits the library has the same shape and the only stated reason to rebuild is curve control, which a backoff callback normally gives you.\nCompleteness: A=9/10, B=5/10, C=n/a (investigation, approves nothing)\nPros / cons:\nA) Library hooks + custom curve (recommended) (human: ~1 day / CC: ~10 min)\n ✅ Attempt counts, scheduling and dead-letter handling come from code already running in production for other users of the library\n ✅ The curve lives in one backoff function that unit-tests trivially; five workers cannot drift apart\n ❌ If the hook API cannot express the curve (unverified), we fall back to B after a short spike\nB) Custom inline scheduler (as planned) (human: ~3-4 days / CC: ~30 min plus ongoing upkeep)\n ✅ Total control over delay math, logging and dispatch with no library constraints\n ✅ No dependency on the library's hook contract or its future API changes\n ❌ Re-implements persistence of attempt state, max attempts and dead-lettering, each a new failure path with no production history\n ❌ Five copies of a scheduler is five places to fix the same bug\nC) Investigate hook API first (bounded) (human: ~30 min / CC: ~5 min)\n ✅ Settles the one unknown (custom backoff support) before committing either way\n ✅ Cheap; produces evidence instead of an assumption\n ❌ Approves nothing; the mechanism stays undecided and this question returns\nNet: Trading full control over code we must now own against reusing a scheduler the plan itself says has the same shape.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hooks + custom curve (recommended)", + "description": "Configure the job library's built-in retry hooks and pass the team's own exponential backoff function so the curve stays under our control. Attempt counts, scheduling and dead-letter handling come from the library. Human ~1 day / CC ~10 min. ✅ Reuses production-proven scheduling and persistence. ✅ Curve lives in one unit-testable function. ❌ If the hook API cannot express the curve (unverified), we fall back to B after a short spike. Completeness 9/10. R2 to R8 stay pending." + }, + { + "label": "Custom inline scheduler (as planned)", + "description": "Keep the plan: write an exponential-backoff scheduler inline in each worker and skip the library's hooks. Human ~3-4 days / CC ~30 min plus ongoing upkeep. ✅ Total control over delay math, logging and dispatch. ✅ No dependency on the library's hook contract. ❌ Re-implements attempt-state persistence, max attempts and dead-lettering with no production history. ❌ Five schedulers to keep in sync. Completeness 5/10. R2 to R8 stay pending." + }, + { + "label": "Investigate hook API first", + "description": "Spend a bounded 30 minutes reading the library's retry hook API to confirm it accepts a custom backoff function, then return to this question. Approves no implementation; mechanism stays undecided. ✅ Settles the one unknown before committing. ✅ Cheap evidence instead of assumption. ❌ This question comes back. R2 to R8 stay pending." + } + ] + } + }, + { + "sessionId": "4e4fc81f-c399-45c6-9e1c-9d6dde841f1f", + "toolUseId": "toolu_012nkq6ma1zJuvM3eZEmDMUv", + "answeredAt": "2026-10-04T07:54:58.563Z", + "answer": "Bounded + dead-letter + alert (recommended)", + "question": { + "question": "D2 — Retry budget: what happens when a job keeps failing?\nProject/branch/task: main — plan \"Add background job retry framework\", reviewing PLAN.md.\nELI10: Some failures are temporary (the database hiccupped) and some are permanent (the record was deleted). Retrying a permanent failure forever just burns workers and hides the bug. The plan says how long to wait between tries but never says when to stop or what to do with a job that never succeeds. A dead-letter set is a parking lot for those jobs so a human can look at them and replay them.\nStakes if we pick wrong: Unbounded retries keep poisoned jobs cycling and starve healthy ones; log-and-drop loses the job and whatever the user was waiting on with no way to replay it.\nRecommendation: A because exhausted jobs are information; parking them with an alert is the only option where a tired human at 3am can see what broke and replay it.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\nA) Bounded + dead-letter + alert (recommended) (human: ~half day / CC: ~10 min)\n ✅ Permanent failures stop consuming workers after a known number of tries and stay replayable\n ✅ An alert on the dead-letter set turns silent job loss into a visible incident\n ❌ Needs a dead-letter destination and an alert rule, both small but real plumbing\nB) Bounded, log and drop (human: ~1 hour / CC: ~5 min)\n ✅ Stops infinite retries with almost no new plumbing beyond a max-attempts number\n ✅ Error log still records which job died and why\n ❌ The job is gone; replay means finding the log line and re-enqueuing by hand, if the payload was even logged\nC) Unbounded (human: 0 / CC: 0)\n ✅ No job is ever dropped, so a dependency that recovers next week still gets its work done\n ✅ Zero configuration\n ❌ Poisoned jobs cycle forever, inflate the queue and mask real bugs; the usual production incident\nNet: Trading a little plumbing now against job loss or queue poisoning that only shows up during an incident.", + "header": "Retry budget", + "multiSelect": false, + "options": [ + { + "label": "Bounded + dead-letter + alert (recommended)", + "description": "Set max attempts per job type (default 5, configurable). On exhaustion the library moves the job to its dead-letter/failed set and an alert fires so a human can inspect and replay. Human ~half day / CC ~10 min. ✅ Permanent failures stop after N tries and stay replayable. ✅ Alert makes silent job loss visible. ❌ Needs a dead-letter destination and an alert rule. Completeness 10/10. R2, R3, R4, R6, R7, R8 stay pending." + }, + { + "label": "Bounded, log and drop", + "description": "Set max attempts per job type (default 5). On exhaustion log at error level and discard the job. Human ~1 hour / CC ~5 min. ✅ Stops infinite retries with minimal plumbing. ✅ Error log records which job died. ❌ Job is gone; replay means hand re-enqueuing from a log line. Completeness 6/10. R2, R3, R4, R6, R7, R8 stay pending." + }, + { + "label": "Unbounded", + "description": "No max attempts; failing jobs retry forever on the backoff curve. Human 0 / CC 0. ✅ No job is ever dropped. ✅ Zero configuration. ❌ Poisoned jobs cycle forever and mask bugs. Completeness 2/10. R2, R3, R4, R6, R7, R8 stay pending." + } + ] + } + }, + { + "sessionId": "4e4fc81f-c399-45c6-9e1c-9d6dde841f1f", + "toolUseId": "toolu_01Uc4fDZaZ59uotYrEVNTavf", + "answeredAt": "2026-10-04T07:55:42.400Z", + "answer": "Full jitter (recommended)", + "question": { + "question": "D3 — Jitter: randomize the retry delay or keep it exact?\nProject/branch/task: main — plan \"Add background job retry framework\", reviewing PLAN.md.\nELI10: When a shared dependency blips, hundreds of jobs fail in the same second. If they all wait exactly 2 s, then 4 s, then 8 s, they all come back at the same instant and knock the dependency over again. Jitter adds randomness to each wait so the retries spread out. It is one line in the backoff function.\nStakes if we pick wrong: Without jitter a 30-second outage turns into a rolling series of self-inflicted spikes; with jitter the curve is slightly harder to reason about in tests (you assert a range, not a value).\nRecommendation: A because full jitter is the standard remedy for retry storms and costs one random call; the test asserts a bounded range.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) Full jitter (recommended) (human: ~30 min incl. test / CC: ~3 min)\n ✅ Spreads retries after a shared failure so the dependency gets room to recover\n ✅ Lives inside the one backoff function approved in D1, so every worker gets it for free\n ❌ Unit tests assert a delay range instead of an exact value; slightly less legible curve\nB) No jitter (human: 0 / CC: 0)\n ✅ Exact, predictable delays that are trivial to assert and to read in logs\n ✅ Nothing to add\n ❌ Correlated failures retry in lockstep, the textbook thundering herd\nNet: Trading exact-value test assertions for protection against the most common retry incident.", + "header": "Jitter", + "multiSelect": false, + "options": [ + { + "label": "Full jitter (recommended)", + "description": "Backoff function returns random(0, base × 2^attempt). Applied once, inside the shared backoff function from D1, so all workers inherit it. Human ~30 min incl. test / CC ~3 min. ✅ Spreads correlated retries. ✅ One place, every worker. ❌ Tests assert a range, not a value. Completeness 10/10. R2, R3, R4, R7, R8 stay pending." + }, + { + "label": "No jitter", + "description": "Backoff function returns exactly base × 2^attempt. Human 0 / CC 0. ✅ Exact, predictable delays. ✅ Nothing to add. ❌ Correlated failures retry in lockstep. Completeness 6/10. R2, R3, R4, R7, R8 stay pending." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T07:52:06.224Z", + "write": "# Eng review: Add background job retry framework\n\nReview target (fixed): `PLAN.md` in `` (branch `main`, commit 93b9b0a), titled \"Plan: Add background job retry framework\".\nReport file: this file (user-requested destination).\nReviewer: /plan-eng-review, session 689-1791100191-adc98fae, 2026-10-04.\n\n## Original plan (verbatim copy of PLAN.md)\n\n```markdown\nProceed directly to the requested engineering review; skip the optional /office-hours prerequisite.\nPlease review this plan thoroughly. As you go, write your plan-mode plan to (use Edit/Write to that exact path).\n\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Working plan\n\nStarts as the original proposal. Amended only by approved decisions recorded in the Decision ledger below.\n\n### Architecture (current value: original proposal)\nCustom exponential-backoff scheduler written inline in each worker instead of the job library's built-in retry hooks.\n\n### Code quality (current value: original proposal)\nRetry envelope (compute delay, log attempt, dispatch) copy-pasted across 5 worker files; refactor deferred.\n\n### Tests (current value: original proposal)\n`processWebhookJob()` rewritten; no regression test for the prior at-most-once delivery guarantee.\n\n### Performance (current value: original proposal)\nEvery retry re-fetches the full job payload and recomputes the dependency graph; no caching.\n\n## Evidence limits\n\nThe repository at the review commit contains only `CLAUDE.md` and `PLAN.md`. The 5 worker files, `processWebhookJob()`, the job library and its retry hooks, and the database access layer are named by the plan but are **not available** in this checkout. Findings below quote the plan text that motivates them (`PLAN.md:line`) and label every claim about existing code as unverified. Missing history proves nothing about proposed behavior.\n" + }, + { + "at": "2026-10-04T07:53:10.871Z", + "old": "Findings below quote the plan text that motivates them (`PLAN.md:line`) and label every claim about existing code as unverified. Missing history proves nothing about proposed behavior.\n", + "new": "Findings below quote the plan text that motivates them (`PLAN.md:line`) and label every claim about existing code as unverified. Missing history proves nothing about proposed behavior.\n\n## Scope Challenge\n\n### A. Assessment\n- **What already solves each sub-problem:** The plan itself states the job library has built-in retry hooks whose shape matches the proposed scheduler (`PLAN.md:7-9`: \"rather than use the existing job library's built-in retry hooks. Same shape as the library version\"). Reuse ladder rung 4 (already-installed dependency) holds. Unverified: the library's hook API (whether it accepts a custom backoff function). Web research [Layer 1]: exponential backoff with jitter, bounded attempts (3-8), dead-letter on exhaustion, and idempotency keys for non-idempotent work are the established pattern; retry storms without jitter and blind retry of non-idempotent operations are the documented pitfalls. Search was untrusted web content, cited only.\n- **Minimum change:** Configure the library's retry hooks with the desired curve in one place. The custom scheduler is scope creep motivated by \"full control over the curve\", which the library likely already offers via a custom backoff callback (unverified).\n- **Complexity count (estimates from plan text):** proposed changed files: 5 worker files, plus `processWebhookJob()` which may live in one of them (5-6 files). New classes/services: 0 (scheduler is inline). Below the 8-file / 2-service gate, so B is skipped.\n- **TODOS cross-reference:** no `TODOS.md` in the repo.\n- **Completeness check:** The plan takes three explicit shortcuts (duplication, no regression test, no caching). The first two cost CC minutes to do right; the third is a measured call.\n- **Distribution check:** no new artifact; workers deploy through the existing path (not available for inspection).\n\n### B. Complexity selectors\nSkipped: fewer than 8 files and fewer than 2 new classes/services.\n\n### C. Findings\n1. **[P1] (confidence: 8/10) PLAN.md:7-9** — Rebuilding the library's retry scheduler inline in each worker. Motivating text: \"We'll roll a custom exponential-backoff scheduler inline in each worker rather than use the existing job library's built-in retry hooks. Same shape as the library version, but we want full control over the curve.\" The plan admits shape parity. A custom scheduler gives up the library's persistence of attempt counts across process restarts, its dead-letter handling and its observability, and gives five workers five chances to diverge. Reviewer: plan-eng-review (native). Remedy: R1 below.\n\nScope Challenge result: pending R1.\n\n## Decision ledger\n\n### R1: Retry mechanism — library built-in hooks vs custom inline scheduler\nFinding: Scope Challenge #1, P1, confidence 8/10, PLAN.md:7-9, reviewer plan-eng-review (native)\nPlan baseline: original proposal — custom exponential-backoff scheduler inline in each worker (no prior approval)\nRuntime evidence: unknown. Job library and worker files are not in this checkout; the hook API's support for a custom backoff function is unverified.\nComparison grid:\n\n| Choice | Current | A) Library hooks + custom curve | B) Custom inline scheduler | C) Investigate hook API first |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler, pending | library retry hooks configured with the team's backoff function; attempt state persisted by the library | custom scheduler per worker as proposed; attempt state tracked by the new code | undecided; bounded 30-minute read of the library's retry hook API, then return to this question |\n| R2 envelope location | duplicated in 5 workers, pending | pending | pending | pending |\n| R3 webhook delivery semantics + regression test | at-most-once, no test, pending | pending | pending | pending |\n| R4 graph recompute on retry | recompute each retry, pending | pending | pending | pending |\n| R5 retry budget / exhaustion | unspecified, pending | pending | pending | pending |\n| R6 jitter | unspecified, pending | pending | pending | pending |\n| R7 delay cap | unspecified, pending | pending | pending | pending |\n| R8 rollout flag | unspecified, pending | pending | pending | pending |\n\nQuestion D1:\nD1 — Retry mechanism: library built-in hooks or a custom inline scheduler?\nProject/branch/task: main — plan \"Add background job retry framework\", reviewing PLAN.md.\nELI10: The job library already knows how to retry a job later with a growing delay. The plan rebuilds that from scratch inside each of five workers because the team wants to pick the exact delay curve. Most libraries let you hand them your own delay function, so you keep the curve and drop the rebuild. The stakes: a homegrown scheduler must also remember attempt counts across restarts, stop after N tries, and report failures, or jobs retry forever or vanish silently.\nStakes if we pick wrong: Either we ship five slightly different retry engines that drift and lose attempt state on a worker restart, or we discover late that the library cannot express the curve and have to rip out the hooks.\nRecommendation: A because the plan admits the library has the same shape and the only stated reason to rebuild is curve control, which a backoff callback normally gives you.\nCompleteness: A=9/10, B=5/10, C=n/a (investigation, approves nothing)\nPros / cons:\nA) Library hooks + custom curve (recommended) (human: ~1 day / CC: ~10 min)\n ✅ Attempt counts, scheduling and dead-letter handling come from code already running in production for other users of the library\n ✅ The curve lives in one backoff function that unit-tests trivially; five workers cannot drift apart\n ❌ If the hook API cannot express the curve (unverified), we fall back to B after a short spike\nB) Custom inline scheduler (as planned) (human: ~3-4 days / CC: ~30 min plus ongoing upkeep)\n ✅ Total control over delay math, logging and dispatch with no library constraints\n ✅ No dependency on the library's hook contract or its future API changes\n ❌ Re-implements persistence of attempt state, max attempts and dead-lettering, each a new failure path with no production history\n ❌ Five copies of a scheduler is five places to fix the same bug\nC) Investigate hook API first (bounded) (human: ~30 min / CC: ~5 min)\n ✅ Settles the one unknown (custom backoff support) before committing either way\n ✅ Cheap; produces evidence instead of an assumption\n ❌ Approves nothing; the mechanism stays undecided and this question returns\nNet: Trading full control over code we must now own against reusing a scheduler the plan itself says has the same shape.\nHeader: Retry mechanism\nOptions:\nA) Library hooks + custom curve (recommended)\nConfigure the job library's built-in retry hooks and pass the team's own exponential backoff function so the curve stays under our control. Attempt counts, scheduling and dead-letter handling come from the library. Human ~1 day / CC ~10 min. ✅ Reuses production-proven scheduling and persistence. ✅ Curve lives in one unit-testable function. ❌ If the hook API cannot express the curve (unverified), we fall back to B after a short spike. Completeness 9/10. R2 to R8 stay pending.\nB) Custom inline scheduler (as planned)\nKeep the plan: write an exponential-backoff scheduler inline in each worker and skip the library's hooks. Human ~3-4 days / CC ~30 min plus ongoing upkeep. ✅ Total control over delay math, logging and dispatch. ✅ No dependency on the library's hook contract. ❌ Re-implements attempt-state persistence, max attempts and dead-lettering with no production history. ❌ Five schedulers to keep in sync. Completeness 5/10. R2 to R8 stay pending.\nC) Investigate hook API first\nSpend a bounded 30 minutes reading the library's retry hook API to confirm it accepts a custom backoff function, then return to this question. Approves no implementation; mechanism stays undecided. ✅ Settles the one unknown before committing. ✅ Cheap evidence instead of assumption. ❌ This question comes back. R2 to R8 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T07:53:56.631Z", + "old": "### Architecture (current value: original proposal)\nCustom exponential-backoff scheduler written inline in each worker instead of the job library's built-in retry hooks.", + "new": "### Architecture (current value: approved D1 = A)\nUse the job library's built-in retry hooks, configured with the team's own exponential backoff function so the curve stays under our control. The library persists attempt counts and schedules the retry. If the hook API turns out not to accept a custom backoff function, run a short spike and bring the decision back (R1 History). Retry budget, jitter, delay cap and rollout control: see R5 to R8." + }, + { + "at": "2026-10-04T07:54:14.946Z", + "old": "Scope Challenge result: pending R1.", + "new": "Dispositions: #1 accepted (D1 = A, library hooks + custom curve).\n\nScope Challenge result: **scope accepted as-is** (MODE: FULL_REVIEW). D1 replaces the mechanism with a smaller arrangement but keeps every feature; no scope was cut.\n\n## Section 1: Architecture review\n\nFlow under the approved mechanism (D1 = A):\n\n```\nenqueue(job) ──► library picks job ──► worker.handle(payload)\n │\n ┌───────────┴─────────────┐\n success throws\n │ │\n ack/done library retry hook: attempt n\n │\n backoffFn(n) ──► delay d (R6 jitter? R7 cap?)\n │\n n < maxAttempts? (R5) ── no ──► exhaustion: dead-letter? drop? alert?\n │ yes\n reschedule at now+d ──► (loop)\n\nprocessWebhookJob: HTTP POST to receiver ──► timeout/5xx ──► throws ──► RETRY ──► POST again\n ▲\n receiver may already have the first POST: duplicate delivery (R3)\n```\n\nFindings (most severe first):\n\n2. **[P1] (confidence: 8/10) PLAN.md:17-19** — Retrying `processWebhookJob()` silently changes the delivery contract. Motivating text: \"The existing `processWebhookJob()` flow gets rewritten as part of this change. No regression test for the prior at-most-once delivery guarantee is planned.\" A retry after a POST that was sent but not acknowledged (timeout, 5xx after the receiver committed) delivers the same webhook twice. At-most-once becomes at-least-once for every receiver, which is a contract change, not a test gap. Realistic production failure: receiver returns 504 under load, every webhook in flight is re-sent, receivers without dedupe double-process. Remedy: R3 (settled by one dedicated regression question in Test review, per the regression rule).\n3. **[P1] (confidence: 7/10) PLAN.md:7-9** — No retry budget or exhaustion behavior. The plan names an \"exponential-backoff scheduler\" and nothing about max attempts, dead-lettering or alerting. A job whose failure is permanent (bad payload, deleted record) retries until the heat death of the queue, or is dropped on the floor with no record, depending on what the library does by default (unverified). Remedy: R5.\n4. **[P2] (confidence: 7/10) PLAN.md:7-9** — No jitter. Five workers retrying after a shared dependency blip compute the same delays and hit it again in lockstep: a retry storm that extends the outage. Remedy: R6.\n5. **[P2] (confidence: 6/10) PLAN.md:7-9** — No delay cap. Pure exponential growth at base 1 s puts attempt 10 at ~17 min and attempt 20 at ~6 days; with a budget (R5) the cap matters less, but an explicit ceiling keeps the curve legible. Medium confidence, verify this is actually an issue once the backoff function is written. Remedy: R7.\n6. **[P2] (confidence: 6/10) PLAN.md:7-9** — No rollout control. All five workers switch from no-retry to retry in one deploy with no per-job-type switch to turn it back off if a receiver melts down. Medium confidence, verify this is actually an issue given the deploy tooling (not available). Remedy: R8.\n\nSecurity: no auth or data-access boundary changes; retried webhook POSTs re-send the same payload to the same receiver, so no new exposure beyond duplication (R3). Distribution: no new artifact. Single point of failure: the database both stores jobs and gets re-queried on each retry (Section 4).\n\nDispositions: #2 pending R3 (Test review); #3 pending R5 (D2); #4 pending R6; #5 pending R7; #6 pending R8." + }, + { + "at": "2026-10-04T07:54:39.677Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hooks + custom curve (D1 answer)\nAccepted scope: Use the job library's built-in retry hooks configured with the team's own exponential backoff function; attempt state, scheduling and dead-letter plumbing come from the library. Includes the hook wiring in each worker and a unit test of the backoff function (necessary proof of this behavior). Fallback to a custom scheduler only after a short spike proves the hook API cannot express the curve; that reopens R1. R2 to R8 stay pending.\nHistory: none\n\n### R5: Retry budget and exhaustion behavior\nFinding: Architecture #3, P1, confidence 7/10, PLAN.md:7-9, reviewer plan-eng-review (native)\nPlan baseline: original proposal is silent on max attempts and exhaustion (no prior approval)\nRuntime evidence: unknown. The library's default max attempts and dead-letter behavior are not inspectable in this checkout.\nComparison grid:\n\n| Choice | Current | A) Bounded + dead-letter + alert | B) Bounded, log and drop | C) Unbounded |\n|---|---|---|---|---|\n| R5 retry budget / exhaustion | unspecified, pending | max attempts per job type (default 5, configurable), exhausted jobs moved to the library's dead-letter/failed set and an alert fires | max attempts per job type (default 5), exhausted jobs logged at error and discarded | no max attempts; failing jobs retry forever on the curve |\n| R1 retry mechanism | approved D1 = A | unchanged | unchanged | unchanged |\n| R2 envelope location | pending | pending | pending | pending |\n| R3 webhook semantics + regression test | pending | pending | pending | pending |\n| R4 graph recompute | pending | pending | pending | pending |\n| R6 jitter | pending | pending | pending | pending |\n| R7 delay cap | pending | pending | pending | pending |\n| R8 rollout flag | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — Retry budget: what happens when a job keeps failing?\nProject/branch/task: main — plan \"Add background job retry framework\", reviewing PLAN.md.\nELI10: Some failures are temporary (the database hiccupped) and some are permanent (the record was deleted). Retrying a permanent failure forever just burns workers and hides the bug. The plan says how long to wait between tries but never says when to stop or what to do with a job that never succeeds. A dead-letter set is a parking lot for those jobs so a human can look at them and replay them.\nStakes if we pick wrong: Unbounded retries keep poisoned jobs cycling and starve healthy ones; log-and-drop loses the job and whatever the user was waiting on with no way to replay it.\nRecommendation: A because exhausted jobs are information; parking them with an alert is the only option where a tired human at 3am can see what broke and replay it.\nCompleteness: A=10/10, B=6/10, C=2/10\nPros / cons:\nA) Bounded + dead-letter + alert (recommended) (human: ~half day / CC: ~10 min)\n ✅ Permanent failures stop consuming workers after a known number of tries and stay replayable\n ✅ An alert on the dead-letter set turns silent job loss into a visible incident\n ❌ Needs a dead-letter destination and an alert rule, both small but real plumbing\nB) Bounded, log and drop (human: ~1 hour / CC: ~5 min)\n ✅ Stops infinite retries with almost no new plumbing beyond a max-attempts number\n ✅ Error log still records which job died and why\n ❌ The job is gone; replay means finding the log line and re-enqueuing by hand, if the payload was even logged\nC) Unbounded (human: 0 / CC: 0)\n ✅ No job is ever dropped, so a dependency that recovers next week still gets its work done\n ✅ Zero configuration\n ❌ Poisoned jobs cycle forever, inflate the queue and mask real bugs; the usual production incident\nNet: Trading a little plumbing now against job loss or queue poisoning that only shows up during an incident.\nHeader: Retry budget\nOptions:\nA) Bounded + dead-letter + alert (recommended)\nSet max attempts per job type (default 5, configurable). On exhaustion the library moves the job to its dead-letter/failed set and an alert fires so a human can inspect and replay. Human ~half day / CC ~10 min. ✅ Permanent failures stop after N tries and stay replayable. ✅ Alert makes silent job loss visible. ❌ Needs a dead-letter destination and an alert rule. Completeness 10/10. R2, R3, R4, R6, R7, R8 stay pending.\nB) Bounded, log and drop\nSet max attempts per job type (default 5). On exhaustion log at error level and discard the job. Human ~1 hour / CC ~5 min. ✅ Stops infinite retries with minimal plumbing. ✅ Error log records which job died. ❌ Job is gone; replay means hand re-enqueuing from a log line. Completeness 6/10. R2, R3, R4, R6, R7, R8 stay pending.\nC) Unbounded\nNo max attempts; failing jobs retry forever on the backoff curve. Human 0 / CC 0. ✅ No job is ever dropped. ✅ Zero configuration. ❌ Poisoned jobs cycle forever and mask bugs. Completeness 2/10. R2, R3, R4, R6, R7, R8 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T07:55:27.108Z", + "old": "No max attempts; failing jobs retry forever on the backoff curve. Human 0 / CC 0. ✅ No job is ever dropped. ✅ Zero configuration. ❌ Poisoned jobs cycle forever and mask bugs. Completeness 2/10. R2, R3, R4, R6, R7, R8 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "No max attempts; failing jobs retry forever on the backoff curve. Human 0 / CC 0. ✅ No job is ever dropped. ✅ Zero configuration. ❌ Poisoned jobs cycle forever and mask bugs. Completeness 2/10. R2, R3, R4, R6, R7, R8 stay pending.\n\nState: approved\nActual answer: A) Bounded + dead-letter + alert (D2 answer)\nAccepted scope: Max attempts per job type, default 5, configurable through the library's retry hook config. On exhaustion the job moves to the library's dead-letter/failed set and an alert fires. Necessary proof: a test that a job failing maxAttempts times lands in the dead-letter set and is not rescheduled, and that the alert hook is invoked once. R2, R3, R4, R6, R7, R8 stay pending.\nHistory: none\n\n### R6: Jitter on the backoff delay\nFinding: Architecture #4, P2, confidence 7/10, PLAN.md:7-9, reviewer plan-eng-review (native)\nPlan baseline: original proposal is silent on jitter (no prior approval)\nRuntime evidence: unknown. Whether the library's hook applies jitter by default is not inspectable here.\nComparison grid:\n\n| Choice | Current | A) Full jitter | B) No jitter |\n|---|---|---|---|\n| R6 jitter | unspecified, pending | delay = random(0, base × 2^n) applied inside the shared backoff function | delay = base × 2^n exactly, deterministic |\n| R1 retry mechanism | approved D1 = A | unchanged | unchanged |\n| R5 retry budget | approved D2 = A | unchanged | unchanged |\n| R2 envelope location | pending | pending | pending |\n| R3 webhook semantics + regression test | pending | pending | pending |\n| R4 graph recompute | pending | pending | pending |\n| R7 delay cap | pending | pending | pending |\n| R8 rollout flag | pending | pending | pending |\n\nQuestion D3:\nD3 — Jitter: randomize the retry delay or keep it exact?\nProject/branch/task: main — plan \"Add background job retry framework\", reviewing PLAN.md.\nELI10: When a shared dependency blips, hundreds of jobs fail in the same second. If they all wait exactly 2 s, then 4 s, then 8 s, they all come back at the same instant and knock the dependency over again. Jitter adds randomness to each wait so the retries spread out. It is one line in the backoff function.\nStakes if we pick wrong: Without jitter a 30-second outage turns into a rolling series of self-inflicted spikes; with jitter the curve is slightly harder to reason about in tests (you assert a range, not a value).\nRecommendation: A because full jitter is the standard remedy for retry storms and costs one random call; the test asserts a bounded range.\nCompleteness: A=10/10, B=6/10\nPros / cons:\nA) Full jitter (recommended) (human: ~30 min incl. test / CC: ~3 min)\n ✅ Spreads retries after a shared failure so the dependency gets room to recover\n ✅ Lives inside the one backoff function approved in D1, so every worker gets it for free\n ❌ Unit tests assert a delay range instead of an exact value; slightly less legible curve\nB) No jitter (human: 0 / CC: 0)\n ✅ Exact, predictable delays that are trivial to assert and to read in logs\n ✅ Nothing to add\n ❌ Correlated failures retry in lockstep, the textbook thundering herd\nNet: Trading exact-value test assertions for protection against the most common retry incident.\nHeader: Jitter\nOptions:\nA) Full jitter (recommended)\nBackoff function returns random(0, base × 2^attempt). Applied once, inside the shared backoff function from D1, so all workers inherit it. Human ~30 min incl. test / CC ~3 min. ✅ Spreads correlated retries. ✅ One place, every worker. ❌ Tests assert a range, not a value. Completeness 10/10. R2, R3, R4, R7, R8 stay pending.\nB) No jitter\nBackoff function returns exactly base × 2^attempt. Human 0 / CC 0. ✅ Exact, predictable delays. ✅ Nothing to add. ❌ Correlated failures retry in lockstep. Completeness 6/10. R2, R3, R4, R7, R8 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37193478719-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37193478719-t1-pass.json new file mode 100644 index 000000000..e4d46d49f --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37193478719-t1-pass.json @@ -0,0 +1,121 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "a3e4c3429b2cfef8", + "source": { + "run": "37193478719", + "sha": "9f357d9b9c75441b6a18e4f75174274954ba6f96", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37193478719-1-eval-slices-1/plan-eng-review-1791107629170-XPmpHH" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "29344c71-f9f0-40f3-b654-20ee9525ca09", + "toolUseId": "toolu_01CqY8j4zmo98BsVwYoFoZCF", + "answeredAt": "2026-10-04T09:56:42.587Z", + "answer": "Library hooks + custom curve (recommended)", + "question": { + "question": "D1 — Build the retry scheduler inline, or use the job library's retry hooks with a custom backoff curve?\nProject/branch/task: main — reviewing PLAN.md \"Add background job retry framework\"; the plan proposes a hand-rolled exponential-backoff scheduler inside each of 5 workers.\nELI10: The job library already knows how to retry a failed job: it stores the attempt count, waits, and re-runs it, and almost every mainstream library lets you plug in your own delay formula. The plan rebuilds that machinery inside each worker so \"we control the curve.\" That control is already available through the library's backoff callback, and the rebuilt version has to re-solve persistence, visibility and crash safety on its own.\nStakes if we pick wrong: an in-process retry timer dies with the worker, so a crash or deploy mid-backoff silently drops the job; retry counts and dead-letter state vanish from the library's dashboards and tooling.\nRecommendation: A because the library's hook gives the same curve control with persistence and tooling for free [Layer 1]; rebuilding it is accidental complexity in 5 places.\nCompleteness: A=9/10, B=4/10, C=n/a (bounded investigation, decides nothing)\nPros / cons:\nA) Library retry hooks + custom backoff callback (recommended)\n ✅ Retry state (attempt count, next-run time, dead-letter) persists in the queue store and survives worker crashes and deploys\n ✅ Existing library tooling (dashboards, admin retry, metrics) keeps working; one backoff function instead of five schedulers (human: ~1 day / CC: ~20 min)\n ❌ Curve is bounded by the hook's inputs (often attempt number only), so error-specific delays like Retry-After need a small wrapper\nB) Custom inline scheduler in each worker (plan as written)\n ✅ Unlimited control over delay math and per-error behavior without touching library config\n ✅ No dependency on the library's backoff API shape or version\n ❌ Retries live in worker memory unless you also build persistence; a crash during backoff loses the job silently and bypasses the library's retry visibility (human: ~1 week / CC: ~1 hr, plus 5 copies to maintain)\nC) Investigate first: bounded probe of the library's backoff hook\n ✅ Confirms whether the installed library's hook supports the exact curve before committing either way\n ✅ Cheap: read the library docs and one worker's current retry config (human: ~1 hr / CC: ~5 min)\n ❌ Decides nothing; R1 stays pending and Sections 1–4 proceed against the plan as written\nNet: trading a bounded callback signature for crash-safe, observable retries you don't have to write five times.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hooks + custom curve (recommended)", + "description": "Use the job library's built-in retry hooks and supply one custom backoff callback for the curve. ✅ Retry state (attempt count, next-run time, dead-letter) persists in the queue store and survives worker crashes and deploys. ✅ Existing library tooling (dashboards, admin retry, metrics) keeps working; one backoff function instead of five schedulers (human: ~1 day / CC: ~20 min). ❌ Curve is bounded by the hook's inputs (often attempt number only), so error-specific delays like Retry-After need a small wrapper. Completeness 9/10. R2–R5 stay pending." + }, + { + "label": "Custom inline scheduler", + "description": "Keep the plan as written: a hand-rolled exponential-backoff scheduler inside each of the 5 workers. ✅ Unlimited control over delay math and per-error behavior without touching library config. ✅ No dependency on the library's backoff API shape or version. ❌ Retries live in worker memory unless you also build persistence; a crash during backoff loses the job silently and bypasses the library's retry visibility (human: ~1 week / CC: ~1 hr, plus 5 copies to maintain). Completeness 4/10. R2–R5 stay pending." + }, + { + "label": "Investigate library hook first", + "description": "Bounded probe only: read the installed library's backoff hook docs and one worker's current retry config, then return to this question. ✅ Confirms whether the hook supports the exact curve before committing either way. ✅ Cheap (human: ~1 hr / CC: ~5 min). ❌ Decides nothing; R1 stays pending and Sections 1–4 proceed against the plan as written. No completeness score (investigation, not coverage). R2–R5 stay pending." + } + ] + } + }, + { + "sessionId": "29344c71-f9f0-40f3-b654-20ee9525ca09", + "toolUseId": "toolu_016iX2cRJrKmjtB3yJQCScmd", + "answeredAt": "2026-10-04T09:59:00.483Z", + "answer": "Idempotent retried delivery (recommended)", + "question": { + "question": "D2 — What delivery guarantee does the retried processWebhookJob keep, and what regression test proves it?\nProject/branch/task: main — reviewing PLAN.md \"Add background job retry framework\"; the webhook job is rewritten under the new retry path and currently guarantees at-most-once delivery.\nELI10: Today a webhook is sent once; if it fails, it is gone. Adding retries means the same event can be sent again. Sometimes the first send actually arrived but the response got lost, so the retry delivers a duplicate. Receivers that bill, email or create records on each webhook will double-act unless we give them a way to recognise a repeat.\nStakes if we pick wrong: downstream systems process duplicate events (double charges, double notifications) with no test telling us the guarantee changed, or webhook failures stay unrecovered forever.\nRecommendation: A because it recovers failed sends and keeps receivers safe from duplicates, and the four assertions make the guarantee executable rather than implied.\nCompleteness: A=9/10, B=5/10, C=6/10\nPros / cons:\nA) Idempotent retried delivery (recommended)\n ✅ Failed webhooks are retried, and a stable per-job idempotency-key header plus a persisted delivered marker let receivers and the worker dedup repeats\n ✅ Four concrete assertions cover first-try success, 5xx-then-2xx, lost response and lost ack, so the regression test fails on any semantic slip (human: ~2 days / CC: ~30 min)\n ❌ Receivers must honor the idempotency header to be fully safe; the marker adds one persisted field per job\nB) Explicit at-least-once\n ✅ Simplest code: library retry with no key or marker; one changelog note documents the new semantics\n ✅ Still recovers failed webhooks that were previously dropped\n ❌ Duplicates reach receivers on every ambiguous outcome, and the only test is \"a 5xx schedules a retry\"\nC) Exclude webhooks from retry\n ✅ Preserves the exact current at-most-once guarantee with a one-assertion regression test\n ✅ Zero risk of duplicate side effects downstream\n ❌ Webhook failures stay unrecovered, which is the problem this plan set out to fix; the webhook rewrite then has no purpose\nNet: trading one header and one persisted flag for retries that cannot double-act on receivers.", + "header": "Webhook semantics", + "multiSelect": false, + "options": [ + { + "label": "Idempotent retried delivery (recommended)", + "description": "Retry failed webhook POSTs via the library hooks (D1), send a stable per-job idempotency-key header on every attempt, and persist a delivered marker on 2xx that is checked before each attempt. Regression contract: (1) 2xx on attempt 1 → exactly one POST, no retry; (2) 5xx then 2xx → two POSTs with identical key; (3) send succeeds, response times out → retry carries the same key; (4) marker set but ack lost → no second POST. ✅ Recovers failed sends and dedups repeats. ✅ Guarantee is executable, not implied (human: ~2 days / CC: ~30 min). ❌ Receivers must honor the header; one persisted field per job. Completeness 9/10. R2, R4, R6–R9 stay pending." + }, + { + "label": "Explicit at-least-once", + "description": "Retry failed webhook POSTs with no key or marker; document at-least-once in the changelog. Regression contract: (1) 5xx → retry scheduled; (2) duplicates on ambiguous outcomes documented. ✅ Simplest code. ✅ Recovers failed webhooks. ❌ Duplicates reach receivers on every ambiguous outcome. Completeness 5/10. R2, R4, R6–R9 stay pending." + }, + { + "label": "Exclude webhooks from retry", + "description": "processWebhookJob opts out of the retry hooks and keeps exact at-most-once. Regression contract: (1) 5xx → job lands in failed state once, no retry; (2) request shape unchanged. ✅ Exact current guarantee preserved. ✅ Zero duplicate risk. ❌ Webhook failures stay unrecovered and the rewrite loses its purpose. Completeness 6/10. R2, R4, R6–R9 stay pending." + } + ] + } + }, + { + "sessionId": "29344c71-f9f0-40f3-b654-20ee9525ca09", + "toolUseId": "toolu_01C1fNQhCuZ3ogymwmdD4fgu", + "answeredAt": "2026-10-04T09:59:57.788Z", + "answer": "Full jitter (recommended)", + "question": { + "question": "D3 — Should retry delays be randomized (jitter), and how?\nProject/branch/task: main — reviewing PLAN.md \"Add background job retry framework\"; the backoff callback approved in D1 needs its curve defined, starting with jitter.\nELI10: When a downstream service goes down, every job that failed at the same moment computes the same retry delay and comes back at the same moment, so the recovering service gets hit by a wave and falls over again. Jitter adds randomness to each delay so retries spread out. Full jitter picks any delay from zero up to the computed value; equal jitter keeps half the delay fixed and randomizes the other half.\nStakes if we pick wrong: synchronized retry storms after every outage, or (with equal jitter) a longer mandatory floor than needed.\nRecommendation: A because full jitter spreads load best for the same code size [Layer 1]; the only cost is that a retry can occasionally fire almost immediately.\nCompleteness: A=9/10, B=8/10, C=4/10\nPros / cons:\nA) Full jitter (recommended)\n ✅ Spreads retries across the whole window, so a recovering service sees a trickle instead of a wave (human: ~2 hr / CC: ~5 min incl. seeded test)\n ✅ Same one-line callback as the plain curve; well documented standard practice\n ❌ A retry may fire almost immediately after failure, which wastes one attempt when the outage is still ongoing\nB) Equal jitter\n ✅ Guarantees a minimum wait of half the computed delay, so no near-instant retry\n ✅ Still breaks synchronization across jobs\n ❌ Slightly more contention than full jitter and a larger mandatory floor at high attempt counts\nC) No jitter\n ✅ Fully deterministic delays are easy to reason about and test exactly\n ✅ Zero extra code\n ❌ Every job that failed together retries together; the thundering-herd failure mode is left open\nNet: trading exact predictability of one delay for a recovering dependency that is not re-flattened by our own retries.", + "header": "Backoff jitter", + "multiSelect": false, + "options": [ + { + "label": "Full jitter (recommended)", + "description": "Backoff callback returns random(0, base × 2^attempt). Unit test with seeded RNG: delay within [0, formula]; two jobs at the same attempt get different delays. ✅ Spreads retries across the whole window (human: ~2 hr / CC: ~5 min). ✅ One-line callback, standard practice. ❌ A retry may fire almost immediately. Completeness 9/10. R7, R8 pending (cap applies pre-jitter when decided); R2, R4, R9 pending." + }, + { + "label": "Equal jitter", + "description": "Callback returns half the formula plus random(0, half). Unit test: delay within [half, full]; two jobs differ. ✅ Guaranteed minimum wait. ✅ Still desynchronizes. ❌ More contention than full jitter and a larger floor at high attempts. Completeness 8/10. R7, R8, R2, R4, R9 pending." + }, + { + "label": "No jitter", + "description": "Callback returns base × 2^attempt exactly. Unit test: exact values per attempt. ✅ Deterministic and exact. ✅ Zero extra code. ❌ Thundering herd after outages stays open. Completeness 4/10. R7, R8, R2, R4, R9 pending." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T09:55:09.093Z", + "write": "# Eng review working plan — Add background job retry framework\n\nReview target (fixed): `PLAN.md` in (\"Plan: Add background job retry framework\"), branch `main`, commit d539bb1.\nReviewer: /plan-eng-review (claude host), session 692-1791107614-d3e1e871.\nEvidence limits: the repository contains only CLAUDE.md and PLAN.md. No worker source, job library manifest or tests are present, so every code-level claim below is marked unknown unless the plan text itself states it.\n\n## Original plan (unchanged copy)\n\n```markdown\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n" + }, + { + "at": "2026-10-04T09:56:20.513Z", + "old": "iterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n", + "new": "iterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Scope Challenge\n\nComplexity gate: ~5 worker files + webhook job flow, 0–1 new component. Under 8 files and under 2 new classes, so B skipped. Search check (WebSearch, Aside absent): BullMQ exposes a custom `backoffStrategy` hook plus built-in fixed/exponential with jitter; documented pitfalls are hook inputs (attempt number only) and silent fallthrough on unknown strategy names.\n\nFindings:\n1. [P1] (confidence: 8/10) PLAN.md:7-9 — custom inline exponential-backoff scheduler per worker instead of the library's retry hooks. Reviewer: plan-eng-review (claude).\n\n## Decision ledger\n\n### R1: Retry mechanism — library retry hooks with custom backoff vs custom inline scheduler\nFinding: #1, P1, confidence 8/10, PLAN.md:7-9 (\"We'll roll a custom exponential-backoff scheduler inline in each worker rather than use the existing job library's built-in retry hooks. Same shape as the library version, but we want full control over the curve.\"), reviewer plan-eng-review (claude)\nPlan baseline: original proposal — custom inline scheduler in each of 5 workers; nothing approved yet\nRuntime evidence: unknown — no worker source or library manifest in this repo. External: BullMQ docs show a custom backoffStrategy hook and built-in exponential + jitter (https://docs.bullmq.io/guide/retrying-failing-jobs); other mainstream libraries (Sidekiq, Celery, Oban) expose equivalent per-job backoff callbacks (in-distribution knowledge, unverified against the installed library)\nComparison grid:\n\n| Choice | Current (plan) | A) Library hooks + custom curve | B) Custom inline scheduler | C) Investigate library hook first |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler per worker (unapproved proposal) | library retry hooks + one custom backoff callback; retry state persisted by the queue store | custom inline scheduler per worker (original) | undecided; bounded probe: read installed library's backoff hook docs + one worker's current retry config; approves nothing |\n| R2 retry-envelope duplication (5 copies) | copy-pasted, \"refactor later\" | pending (A shrinks its surface; still its own answer) | pending | pending |\n| R3 at-most-once regression coverage for processWebhookJob | none planned | pending | pending | pending |\n| R4 payload re-fetch / dependency-graph caching | re-fetch every retry, not planned | pending | pending | pending |\n| R5 delivery semantics (at-most-once → retried) | implicit, unstated | pending | pending | pending |\n\nQuestion D1:\nD1 — Build the retry scheduler inline, or use the job library's retry hooks with a custom backoff curve?\nProject/branch/task: main — reviewing PLAN.md \"Add background job retry framework\"; the plan proposes a hand-rolled exponential-backoff scheduler inside each of 5 workers.\nELI10: The job library already knows how to retry a failed job: it stores the attempt count, waits, and re-runs it, and almost every mainstream library lets you plug in your own delay formula. The plan rebuilds that machinery inside each worker so \"we control the curve.\" That control is already available through the library's backoff callback, and the rebuilt version has to re-solve persistence, visibility and crash safety on its own.\nStakes if we pick wrong: an in-process retry timer dies with the worker, so a crash or deploy mid-backoff silently drops the job; retry counts and dead-letter state vanish from the library's dashboards and tooling.\nRecommendation: A because the library's hook gives the same curve control with persistence and tooling for free [Layer 1]; rebuilding it is accidental complexity in 5 places.\nCompleteness: A=9/10, B=4/10, C=n/a (bounded investigation, decides nothing)\nPros / cons:\nA) Library retry hooks + custom backoff callback (recommended)\n ✅ Retry state (attempt count, next-run time, dead-letter) persists in the queue store and survives worker crashes and deploys\n ✅ Existing library tooling (dashboards, admin retry, metrics) keeps working; one backoff function instead of five schedulers (human: ~1 day / CC: ~20 min)\n ❌ Curve is bounded by the hook's inputs (often attempt number only), so error-specific delays like Retry-After need a small wrapper\nB) Custom inline scheduler in each worker (plan as written)\n ✅ Unlimited control over delay math and per-error behavior without touching library config\n ✅ No dependency on the library's backoff API shape or version\n ❌ Retries live in worker memory unless you also build persistence; a crash during backoff loses the job silently and bypasses the library's retry visibility (human: ~1 week / CC: ~1 hr, plus 5 copies to maintain)\nC) Investigate first: bounded probe of the library's backoff hook\n ✅ Confirms whether the installed library's hook supports the exact curve before committing either way\n ✅ Cheap: read the library docs and one worker's current retry config (human: ~1 hr / CC: ~5 min)\n ❌ Decides nothing; R1 stays pending and Sections 1–4 proceed against the plan as written\nNet: trading a bounded callback signature for crash-safe, observable retries you don't have to write five times.\nHeader: Retry mechanism\nOptions:\nA) Library hooks + custom curve (recommended)\nUse the job library's built-in retry hooks and supply one custom backoff callback for the curve. ✅ Retry state (attempt count, next-run time, dead-letter) persists in the queue store and survives worker crashes and deploys. ✅ Existing library tooling (dashboards, admin retry, metrics) keeps working; one backoff function instead of five schedulers (human: ~1 day / CC: ~20 min). ❌ Curve is bounded by the hook's inputs (often attempt number only), so error-specific delays like Retry-After need a small wrapper. Completeness 9/10. R2–R5 stay pending.\nB) Custom inline scheduler\nKeep the plan as written: a hand-rolled exponential-backoff scheduler inside each of the 5 workers. ✅ Unlimited control over delay math and per-error behavior without touching library config. ✅ No dependency on the library's backoff API shape or version. ❌ Retries live in worker memory unless you also build persistence; a crash during backoff loses the job silently and bypasses the library's retry visibility (human: ~1 week / CC: ~1 hr, plus 5 copies to maintain). Completeness 4/10. R2–R5 stay pending.\nC) Investigate library hook first\nBounded probe only: read the installed library's backoff hook docs and one worker's current retry config, then return to this question. ✅ Confirms whether the hook supports the exact curve before committing either way. ✅ Cheap (human: ~1 hr / CC: ~5 min). ❌ Decides nothing; R1 stays pending and Sections 1–4 proceed against the plan as written. No completeness score (investigation, not coverage). R2–R5 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T09:56:52.825Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hooks + custom curve (D1 answer, AskUserQuestion)\nAccepted scope: Replace the inline per-worker exponential-backoff scheduler with the job library's built-in retry hooks plus one custom backoff callback that implements the desired curve; retry state (attempt count, next-run time, dead-letter) is persisted by the queue store. Includes the callback's unit tests and the plan's Architecture section rewrite. R2–R5 remain pending.\nHistory: none\n\nScope Challenge result: scope accepted as-is (R1 changes the mechanism, not the feature scope; B skipped).\n" + }, + { + "at": "2026-10-04T09:58:39.713Z", + "old": "Scope Challenge result: scope accepted as-is (R1 changes the mechanism, not the feature scope; B skipped).\n", + "new": "Scope Challenge result: scope accepted as-is (R1 changes the mechanism, not the feature scope; B skipped).\n\n## Section 1: Architecture review — findings\n\n1. [P1] (confidence: 8/10) PLAN.md:17-19 — retrying `processWebhookJob()` converts at-most-once delivery into at-least-once; a POST that lands but whose response times out is retried and the receiver gets a duplicate. No idempotency key or delivered marker planned. → R5\n2. [P2] (confidence: 7/10) PLAN.md:7-9 — no retry bounds: no max attempts, delay cap or jitter; lockstep retry storms after an outage and unbounded delay growth. → R6 (jitter), R7 (max attempts), R8 (delay cap)\n3. [P2] (confidence: 7/10) PLAN.md:6-9 — no retry state diagram; factual addition, see Diagrams. No decision.\n4. Worker crash during in-memory backoff loses the retry — resolved by D1 (library persists retry state). Accepted via D1.\n\n### R5: processWebhookJob delivery semantics and regression contract\nFinding: #1 (Architecture), P1, confidence 8/10, PLAN.md:17-19 (\"The existing `processWebhookJob()` flow gets rewritten as part of this change. No regression test for the prior at-most-once delivery guarantee is planned.\"), reviewer plan-eng-review (claude)\nPlan baseline: original proposal — rewrite the flow under the retry framework with no stated semantics and no regression test; nothing approved\nRuntime evidence: unknown — processWebhookJob source not in this repo; the plan itself asserts the current guarantee is at-most-once\nComparison grid:\n\n| Choice | Current (plan) | A) Idempotent retried delivery | B) Explicit at-least-once | C) Exclude webhooks from retry |\n|---|---|---|---|---|\n| R5 behavior preserved | implicit, unstated | receiver never gets the same event twice on a successful path (dedup via stable key + delivered marker) | none; duplicates allowed | exact at-most-once: a failed POST is never re-sent |\n| R5 intentional change | implicit, unstated | failed POST (connect error, 5xx, timeout) is retried via library hooks | failed POST retried; duplicates possible on ambiguous outcomes | none for webhooks; other 4 workers still get retries |\n| R5 acceptance assertions (regression contract) | none planned | (1) 2xx on attempt 1 → exactly one POST, no retry scheduled; (2) 5xx then 2xx → two POSTs with identical idempotency-key header; (3) send succeeds, response times out → retry carries the same key; (4) delivered marker set but ack lost → no second POST | (1) 5xx → retry scheduled; (2) key header absent, duplicate on ambiguous outcome documented in changelog | (1) 5xx → job lands in failed state once, no retry; (2) unchanged request shape |\n| R1 retry mechanism | approved A (D1) | approved A, unchanged | approved A, unchanged | approved A, unchanged (webhook opts out) |\n| R2, R4, R6, R7, R8, R9 | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — What delivery guarantee does the retried processWebhookJob keep, and what regression test proves it?\nProject/branch/task: main — reviewing PLAN.md \"Add background job retry framework\"; the webhook job is rewritten under the new retry path and currently guarantees at-most-once delivery.\nELI10: Today a webhook is sent once; if it fails, it is gone. Adding retries means the same event can be sent again. Sometimes the first send actually arrived but the response got lost, so the retry delivers a duplicate. Receivers that bill, email or create records on each webhook will double-act unless we give them a way to recognise a repeat.\nStakes if we pick wrong: downstream systems process duplicate events (double charges, double notifications) with no test telling us the guarantee changed, or webhook failures stay unrecovered forever.\nRecommendation: A because it recovers failed sends and keeps receivers safe from duplicates, and the four assertions make the guarantee executable rather than implied.\nCompleteness: A=9/10, B=5/10, C=6/10\nPros / cons:\nA) Idempotent retried delivery (recommended)\n ✅ Failed webhooks are retried, and a stable per-job idempotency-key header plus a persisted delivered marker let receivers and the worker dedup repeats\n ✅ Four concrete assertions cover first-try success, 5xx-then-2xx, lost response and lost ack, so the regression test fails on any semantic slip (human: ~2 days / CC: ~30 min)\n ❌ Receivers must honor the idempotency header to be fully safe; the marker adds one persisted field per job\nB) Explicit at-least-once\n ✅ Simplest code: library retry with no key or marker; one changelog note documents the new semantics\n ✅ Still recovers failed webhooks that were previously dropped\n ❌ Duplicates reach receivers on every ambiguous outcome, and the only test is \"a 5xx schedules a retry\"\nC) Exclude webhooks from retry\n ✅ Preserves the exact current at-most-once guarantee with a one-assertion regression test\n ✅ Zero risk of duplicate side effects downstream\n ❌ Webhook failures stay unrecovered, which is the problem this plan set out to fix; the webhook rewrite then has no purpose\nNet: trading one header and one persisted flag for retries that cannot double-act on receivers.\nHeader: Webhook semantics\nOptions:\nA) Idempotent retried delivery (recommended)\nRetry failed webhook POSTs via the library hooks (D1), send a stable per-job idempotency-key header on every attempt, and persist a delivered marker on 2xx that is checked before each attempt. Regression contract: (1) 2xx on attempt 1 → exactly one POST, no retry; (2) 5xx then 2xx → two POSTs with identical key; (3) send succeeds, response times out → retry carries the same key; (4) marker set but ack lost → no second POST. ✅ Recovers failed sends and dedups repeats. ✅ Guarantee is executable, not implied (human: ~2 days / CC: ~30 min). ❌ Receivers must honor the header; one persisted field per job. Completeness 9/10. R2, R4, R6–R9 stay pending.\nB) Explicit at-least-once\nRetry failed webhook POSTs with no key or marker; document at-least-once in the changelog. Regression contract: (1) 5xx → retry scheduled; (2) duplicates on ambiguous outcomes documented. ✅ Simplest code. ✅ Recovers failed webhooks. ❌ Duplicates reach receivers on every ambiguous outcome. Completeness 5/10. R2, R4, R6–R9 stay pending.\nC) Exclude webhooks from retry\nprocessWebhookJob opts out of the retry hooks and keeps exact at-most-once. Regression contract: (1) 5xx → job lands in failed state once, no retry; (2) request shape unchanged. ✅ Exact current guarantee preserved. ✅ Zero duplicate risk. ❌ Webhook failures stay unrecovered and the rewrite loses its purpose. Completeness 6/10. R2, R4, R6–R9 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T09:59:40.286Z", + "old": "Completeness 6/10. R2, R4, R6–R9 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "Completeness 6/10. R2, R4, R6–R9 stay pending.\n\nState: approved\nActual answer: A) Idempotent retried delivery (D2 answer, AskUserQuestion)\nAccepted scope: processWebhookJob retries failed POSTs (connect error, 5xx, timeout) via the library hooks (D1); every attempt sends a stable per-job idempotency-key header; a delivered marker is persisted on 2xx and checked before each attempt. Regression contract (CRITICAL test, carried into Test review as approved): (1) 2xx on attempt 1 → exactly one POST, no retry scheduled; (2) 5xx then 2xx → two POSTs with identical idempotency key; (3) send succeeds, response times out → retry carries the same key; (4) marker set but ack lost → no second POST. Changelog note documenting the new header. R2, R4, R6–R9 remain pending.\nHistory: none\n\n### R6: Backoff jitter\nFinding: #2 (Architecture), P2, confidence 7/10, PLAN.md:7-9 (\"custom exponential-backoff scheduler ... full control over the curve\"), reviewer plan-eng-review (claude). The curve names exponential growth but no jitter.\nPlan baseline: original proposal — exponential curve, jitter unspecified; nothing approved\nRuntime evidence: unknown — no scheduler source in this repo. External: BullMQ built-in exponential accepts a 0–1 jitter factor (https://docs.bullmq.io/guide/retrying-failing-jobs); AWS Architecture Blog \"Exponential Backoff and Jitter\" shows full jitter minimizes contention and total work (in-distribution knowledge)\nComparison grid:\n\n| Choice | Current (plan) | A) Full jitter | B) Equal jitter | C) No jitter |\n|---|---|---|---|---|\n| R6 jitter | unspecified, pending | delay = random(0, base × 2^attempt) | delay = half of base × 2^attempt + random(0, that half) | delay = base × 2^attempt exactly |\n| R6 proof (unit test, seeded RNG) | none | delay ∈ [0, base × 2^attempt]; two jobs with same attempt get different delays | delay ∈ [half, full]; two jobs differ | delay equals formula exactly |\n| R1 retry mechanism | approved A (D1) | unchanged | unchanged | unchanged |\n| R5 webhook semantics | approved A (D2) | unchanged | unchanged | unchanged |\n| R7 max attempts, R8 delay cap | pending | pending (cap applies to the pre-jitter value when decided) | pending | pending |\n| R2, R4, R9 | pending | pending | pending | pending |\n\nQuestion D3:\nD3 — Should retry delays be randomized (jitter), and how?\nProject/branch/task: main — reviewing PLAN.md \"Add background job retry framework\"; the backoff callback approved in D1 needs its curve defined, starting with jitter.\nELI10: When a downstream service goes down, every job that failed at the same moment computes the same retry delay and comes back at the same moment, so the recovering service gets hit by a wave and falls over again. Jitter adds randomness to each delay so retries spread out. Full jitter picks any delay from zero up to the computed value; equal jitter keeps half the delay fixed and randomizes the other half.\nStakes if we pick wrong: synchronized retry storms after every outage, or (with equal jitter) a longer mandatory floor than needed.\nRecommendation: A because full jitter spreads load best for the same code size [Layer 1]; the only cost is that a retry can occasionally fire almost immediately.\nCompleteness: A=9/10, B=8/10, C=4/10\nPros / cons:\nA) Full jitter (recommended)\n ✅ Spreads retries across the whole window, so a recovering service sees a trickle instead of a wave (human: ~2 hr / CC: ~5 min incl. seeded test)\n ✅ Same one-line callback as the plain curve; well documented standard practice\n ❌ A retry may fire almost immediately after failure, which wastes one attempt when the outage is still ongoing\nB) Equal jitter\n ✅ Guarantees a minimum wait of half the computed delay, so no near-instant retry\n ✅ Still breaks synchronization across jobs\n ❌ Slightly more contention than full jitter and a larger mandatory floor at high attempt counts\nC) No jitter\n ✅ Fully deterministic delays are easy to reason about and test exactly\n ✅ Zero extra code\n ❌ Every job that failed together retries together; the thundering-herd failure mode is left open\nNet: trading exact predictability of one delay for a recovering dependency that is not re-flattened by our own retries.\nHeader: Backoff jitter\nOptions:\nA) Full jitter (recommended)\nBackoff callback returns random(0, base × 2^attempt). Unit test with seeded RNG: delay within [0, formula]; two jobs at the same attempt get different delays. ✅ Spreads retries across the whole window (human: ~2 hr / CC: ~5 min). ✅ One-line callback, standard practice. ❌ A retry may fire almost immediately. Completeness 9/10. R7, R8 pending (cap applies pre-jitter when decided); R2, R4, R9 pending.\nB) Equal jitter\nCallback returns half the formula plus random(0, half). Unit test: delay within [half, full]; two jobs differ. ✅ Guaranteed minimum wait. ✅ Still desynchronizes. ❌ More contention than full jitter and a larger floor at high attempts. Completeness 8/10. R7, R8, R2, R4, R9 pending.\nC) No jitter\nCallback returns base × 2^attempt exactly. Unit test: exact values per attempt. ✅ Deterministic and exact. ✅ Zero extra code. ❌ Thundering herd after outages stays open. Completeness 4/10. R7, R8, R2, R4, R9 pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37195203538-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37195203538-t1-pass.json new file mode 100644 index 000000000..94c0be328 --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37195203538-t1-pass.json @@ -0,0 +1,136 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "ded5316419aefeaf", + "source": { + "run": "37195203538", + "sha": "41287009bd6b524244edced7770c87fc3103eb59", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37195203538-1-eval-slices-1/plan-eng-review-1791109541361-oBdGAF" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "148ebb73-116f-48ed-a898-e9a1ec6c7a20", + "toolUseId": "toolu_01JBahHE3xuK2uiL2acmpLvV", + "answeredAt": "2026-10-04T10:28:26.210Z", + "answer": "Library hook + custom backoff fn (recommended)", + "question": { + "question": "D1 — Use the job library's retry hooks or roll a custom inline scheduler?\nProject/branch/task: `main` of the retry-framework plan; the plan currently rolls its own scheduler inside each of 5 workers (PLAN.md:7-9).\nELI10: Your job library already knows how to retry a failed job: it counts attempts, waits, re-runs, and gives up after N tries. The plan rebuilds that waiting logic by hand inside every worker because the team wants to pick the exact wait curve. Most libraries let you plug in your own curve as a small function while they keep doing the counting and re-running. Rebuilding it means you also own attempt tracking, surviving a worker restart mid-wait, max-attempt cutoff and dead-lettering.\nStakes if we pick wrong: a hand-rolled scheduler that loses attempt state on restart retries forever or never, and the bug appears in five places at once, at 3am.\nRecommendation: A because the plan's only stated reason (\"full control over the curve\") is exactly what a backoff-strategy hook provides, and it is an already-installed dependency (reuse ladder rung 4).\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Library hook + custom backoff function (recommended)\n ✅ Attempt counting, persistence across restarts, max attempts and dead-lettering stay with proven library code\n ✅ The curve lives in one function (exponential, cap, jitter) instead of five inline copies (human: ~0.5 day / CC: ~10 min)\n ❌ Curve is constrained to whatever the hook's signature allows (usually a delay in ms from attempt number and error)\nB) Custom inline scheduler per worker, as planned\n ✅ No dependency on the library's hook contract; any curve shape or side effect is possible\n ✅ No need to learn or verify the hook API before starting (human: ~2 days / CC: ~30 min, plus ongoing maintenance)\n ❌ Re-implements attempt state, restart recovery, cutoff and dead-letter handling, each a new 3am failure mode across 5 files\nC) Investigate the hook first, decide after\n ✅ Bounded check (hook signature, custom curve, max attempts, jitter, cap) settles the unknown with evidence (human: ~1 hour / CC: ~5 min)\n ✅ Nothing is built on an assumption; R1 stays pending, R2–R4 stay pending\n ❌ Delays the architecture decision; if the hook obviously supports a custom curve this is wasted time\nNet: trading the library's proven retry lifecycle against total freedom over a curve the library almost certainly already lets you set.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hook + custom backoff fn (recommended)", + "description": "Register one custom backoff function with the job library's built-in retry hook; the function owns the curve (exponential base, cap, jitter). The library keeps attempt counting, persistence across worker restarts, max-attempt cutoff and dead-lettering. Human: ~0.5 day / CC: ~10 min. Risk: curve limited to the hook's signature. R2–R4 remain pending." + }, + { + "label": "Custom inline scheduler", + "description": "Keep the plan as written: a hand-rolled exponential-backoff scheduler inline in each worker, bypassing the library's retry hooks. Full curve freedom. Human: ~2 days / CC: ~30 min plus maintenance. Risk: re-implements attempt state, restart recovery, cutoff and dead-lettering in 5 places. R2–R4 remain pending." + }, + { + "label": "Investigate hook first", + "description": "Bounded investigation only: read the library's retry hook signature and confirm it supports a custom curve, max attempts, jitter and a delay cap. No implementation approved; R1 returns to this question with evidence. Human: ~1 hour / CC: ~5 min. R2–R4 remain pending." + } + ] + } + }, + { + "sessionId": "148ebb73-116f-48ed-a898-e9a1ec6c7a20", + "toolUseId": "toolu_01RoMYnxxpq4VAbMfbGVddrp", + "answeredAt": "2026-10-04T10:29:58.185Z", + "answer": "At-least-once + idempotency id (recommended)", + "question": { + "question": "D2 — What delivery guarantee does the webhook job keep once it can retry?\nProject/branch/task: `main` of the retry-framework plan; `processWebhookJob()` is rewritten and loses its at-most-once guarantee (PLAN.md:17-19).\nELI10: Today a webhook is sent at most once: if it fails, it is gone, but the receiver never sees a duplicate. Adding retries flips that: the receiver might now get the same event twice, for example when the POST lands but the worker dies before marking it done. You can either tell receivers \"you may see duplicates, here is an id so you can ignore the second one,\" keep the old promise by never retrying once a send was attempted, or retry blindly and hope receivers cope.\nStakes if we pick wrong: receivers double-process events (double charges, double emails), or you keep at-most-once and the reliability win from retries never reaches the webhook path.\nRecommendation: A because at-least-once plus a stable delivery id is the standard webhook contract and receivers can dedupe cheaply, while B gives up most of the reliability you are building this for.\nCompleteness: A=9/10, B=7/10, C=3/10\nPros / cons:\nA) At-least-once with idempotency id (recommended)\n ✅ Retries apply to the whole webhook path, including transient receiver 5xx and network timeouts\n ✅ Stable delivery id on every attempt lets receivers dedupe; this is the documented contract most webhook consumers expect (human: ~1 day / CC: ~15 min)\n ❌ Receivers that ignore the id will see duplicates; the contract change must be documented and announced\nB) Preserve at-most-once\n ✅ No behavior change visible to receivers; no duplicate risk and no receiver-side work\n ✅ Retries still cover pre-send failures such as payload fetch or serialization errors (human: ~0.5 day / CC: ~10 min)\n ❌ A receiver 503 or a timeout after send is final: the event is dead-lettered, so the webhook path gains almost nothing from this project\nC) At-least-once, no dedupe id (plan as written)\n ✅ Smallest change: just let the library retry the existing handler as-is\n ✅ No contract documentation or header work (human: ~2 hours / CC: ~5 min)\n ❌ Duplicates are undetectable by receivers; the first incident is a double side effect with no way to reconcile\nNet: trading a documented, dedupe-able duplicate against either silent duplicates or a webhook path that cannot be retried.", + "header": "Webhook guarantee", + "multiSelect": false, + "options": [ + { + "label": "At-least-once + idempotency id (recommended)", + "description": "`processWebhookJob()` retries through the library hook (D1). Every attempt sends the same stable delivery id (header and body field) so receivers can dedupe; the contract change is documented for receivers. Completeness 9/10. Human: ~1 day / CC: ~15 min. Risk: receivers ignoring the id see duplicates. R2, R3, R4 remain pending." + }, + { + "label": "Preserve at-most-once", + "description": "Retries cover only failures before the HTTP send is attempted (payload fetch, serialization, auth). Once a send is attempted, no retry; failure goes to dead-letter. Receivers see no behavior change. Completeness 7/10. Human: ~0.5 day / CC: ~10 min. Risk: post-send transient failures are final. R2, R3, R4 remain pending." + }, + { + "label": "At-least-once, no dedupe id", + "description": "Let the library retry the existing handler unchanged. No delivery id, no contract documentation. Completeness 3/10. Human: ~2 hours / CC: ~5 min. Risk: undetectable duplicate side effects at receivers. R2, R3, R4 remain pending." + } + ] + } + }, + { + "sessionId": "148ebb73-116f-48ed-a898-e9a1ec6c7a20", + "toolUseId": "toolu_012wPP3MnH4ha91TjftEnYM8", + "answeredAt": "2026-10-04T10:31:24.991Z", + "answer": "Library event listener (recommended)", + "question": { + "question": "D3 — Where does retry-attempt logging live once the library drives retries?\nProject/branch/task: `main` of the retry-framework plan; the retry envelope is copy-pasted into 5 worker files and the plan leaves it duplicated (PLAN.md:12-14).\nELI10: After D1 the library computes the delay and re-dispatches, so two of the three copy-pasted pieces disappear on their own. The third piece, \"log this attempt\", still has to live somewhere. Most job libraries fire an event every time a job fails and is scheduled to retry; one listener on that event logs every worker's attempts with zero code in the workers. The alternatives are a tiny shared function each worker calls, or leaving five copies and promising to clean up later.\nStakes if we pick wrong: five copies drift, one worker logs attempts differently or not at all, and the 3am incident has a hole exactly where the failing worker is.\nRecommendation: A because it uses an already-installed dependency (reuse ladder rung 4), needs no change inside the workers, and cannot drift across them.\nCompleteness: A=10/10, B=8/10, C=3/10\nPros / cons:\nA) One library lifecycle-event listener (recommended)\n ✅ Zero retry code left in any worker; every worker's attempts logged identically from one place (human: ~2 hours / CC: ~5 min)\n ✅ Uses the library's own failed/retry event, so logging stays correct even when a new worker is added later\n ❌ Depends on the library emitting a usable event with attempt number and next delay; verify at implementation time\nB) Shared `logRetryAttempt()` helper called per worker\n ✅ Works even if the library emits no retry event; explicit call site in each worker (human: ~3 hours / CC: ~10 min)\n ✅ Single implementation, so log format cannot drift between workers\n ❌ Every worker still carries a call and a new worker can forget it; one more seam to test per caller\nC) Leave the 5 inline copies, refactor later\n ✅ No change to the 5 workers beyond removing the inline scheduler already approved in D1\n ✅ Nothing new to learn or test right now (human: ~0 / CC: ~0)\n ❌ \"Later\" rarely arrives; five copies drift and the duplication finding stands unresolved in the ship review\nNet: trading a dependency on one library event against carrying retry code in five workers that already should have none.", + "header": "Attempt logging", + "multiSelect": false, + "options": [ + { + "label": "Library event listener (recommended)", + "description": "Register one listener on the job library's failed/retry lifecycle event at worker boot. It logs job id, attempt number, next delay and error class for every worker. Delete the inline envelope from all 5 workers (compute-delay and dispatch already move to the library under D1). Completeness 10/10. Human: ~2 hours / CC: ~5 min. Risk: library must emit the event with attempt data; verify at implementation. R3, R4 remain pending." + }, + { + "label": "Shared logRetryAttempt() helper", + "description": "Create one small helper that logs job id, attempt number, next delay and error class; each of the 5 workers calls it from its failure path. Delete the rest of the inline envelope. Completeness 8/10. Human: ~3 hours / CC: ~10 min. Risk: a new worker can forget the call; one caller-integration test per worker. R3, R4 remain pending." + }, + { + "label": "Leave duplication, refactor later", + "description": "Keep the per-worker attempt logging in all 5 workers as planned (PLAN.md:13-14); only the inline scheduler approved in D1 is removed. Completeness 3/10. Human: ~0 / CC: ~0. Risk: copies drift; finding stays open. R3, R4 remain pending." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T10:26:42.581Z", + "write": "# Eng Review Report — Plan: Add background job retry framework\n\nReview target (fixed): `PLAN.md` in `` (branch `main`, commit `bf0c669`).\nReviewer: /plan-eng-review (gstack), session `692-1791109527-353559b7`, 2026-10-04.\nReport file: this file (user-requested destination).\n\n## Original plan (unchanged copy of PLAN.md)\n\n```markdown\nProceed directly to the requested engineering review; skip the optional /office-hours prerequisite.\nPlease review this plan thoroughly. As you go, write your plan-mode plan to (use Edit/Write to that exact path).\n\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Review evidence notes\n\n- The repository contains only `CLAUDE.md` and `PLAN.md`. None of the code the plan names (the 5 worker files, `processWebhookJob()`, the job library) is checked in here. All runtime evidence below is therefore **unknown**; findings are calibrated from the plan text alone and quote `PLAN.md:line`.\n- `git log -- PLAN.md CLAUDE.md`: one commit (`bf0c669 Seed review plan`); no reverts.\n- No `TODOS.md` in the repo.\n- No design doc found; user asked to skip `/office-hours`.\n" + }, + { + "at": "2026-10-04T10:28:04.398Z", + "old": "- No design doc found; user asked to skip `/office-hours`.\n", + "new": "- No design doc found; user asked to skip `/office-hours`.\n- Test framework detection: no runtime markers, no test config, `TESTFILES:0`. Framework unknown.\n- Outside Voice preflight: `CODEX_MODE: disabled` (`codex_reviews=disabled`); Codex CLI also not installed.\n- Web research (host WebSearch, Aside unavailable): library-level retry hooks are the norm. BullMQ exposes a custom `backoffStrategy` hook (return -1 = fail now, 0 = requeue immediately, N = delay ms); un-jittered exponential backoff is the worst-performing strategy in AWS's jitter experiments; delivery is at-least-once so handlers must be idempotent; always cap attempts. Sources: https://oneuptime.com/blog/post/2026-01-21-bullmq-retry-exponential-backoff/view , https://www.digitalapplied.com/blog/background-job-queue-patterns-2026-engineering-reference , https://knowledgelib.io/software/patterns/retry-exponential-backoff/2026 , https://github.com/autumn-foundation/autumn/issues/3054\n\n## Step 0: Scope Challenge\n\n### A. Assessment\n- **What already solves each sub-problem:** The plan itself names the solution: the existing job library's built-in retry hooks (`PLAN.md:8`). Reuse ladder rung 4 (already-installed dependency). The stated motivation, \"full control over the curve\" (`PLAN.md:9`), is what a backoff-strategy hook exists to provide. **[Layer 1]**: use the library's hook with a custom curve function.\n- **Minimum change:** one backoff function registered with the library, plus the handler changes for `processWebhookJob()`. The inline-per-worker scheduler and the duplicated envelope are accidental complexity, not essential.\n- **Complexity check:** 5 proposed changed worker files (`PLAN.md:13`), 0 new classes/services (the scheduler is inline). Below the 8-file / 2-class gate, so **B is skipped**.\n- **Search check:** done (see evidence notes). No departure from standard practice is justified by the plan text.\n- **TODOS cross-reference:** no `TODOS.md`.\n- **Completeness check:** the plan defers three things (refactor, regression test, graph cache) that each cost CC minutes. Reviewed in Sections 2–4.\n- **Distribution check:** no new artifact; not applicable.\n\n### C. Findings\n1. **[P1] (confidence: 8/10) PLAN.md:7-9 — Custom inline scheduler re-implements the job library's built-in retry hooks.** Motivating lines: \"We'll roll a custom exponential-backoff scheduler inline in each worker / rather than use the existing job library's built-in retry hooks. Same / shape as the library version, but we want full control over the curve.\" Runtime evidence: library identity and hook signature unknown (no code in repo), hence 8 not 9. An inline scheduler must also re-implement attempt counting, persistence of attempt state across worker restarts, max-attempt cutoff and dead-lettering, all of which the library already owns. Remedy: R1 / D1 below.\n\n## Decision ledger\n\n### R1: Retry mechanism — library retry hook vs custom inline scheduler\nFinding: #1, P1, confidence 8/10, PLAN.md:7-9, reviewer: native (plan-eng-review)\nPlan baseline: original proposal — custom exponential-backoff scheduler inline in each of the 5 workers; library hooks bypassed (PLAN.md:7-9). Nothing approved yet.\nRuntime evidence: unknown. The job library, its hook signature, and the 5 worker files are not in this repository. Web research shows mainstream libraries accept a custom backoff function (BullMQ `backoffStrategy`), but this library is unverified.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler per worker (proposed, unapproved) | library built-in retry hook + one custom backoff function (curve, cap, jitter) | custom inline scheduler per worker, as planned | pending; bounded investigation of the library hook's signature (custom curve, max attempts, jitter, cap) before choosing |\n| R2 retry envelope duplication across 5 workers | duplicated (PLAN.md:12-14), pending | pending | pending | pending |\n| R3 regression coverage for at-most-once guarantee | none planned (PLAN.md:17-19), pending | pending | pending | pending |\n| R4 dependency-graph recompute per retry | recompute every retry (PLAN.md:22-24), pending | pending | pending | pending |\n\nQuestion D1:\nD1 — Use the job library's retry hooks or roll a custom inline scheduler?\nProject/branch/task: `main` of the retry-framework plan; the plan currently rolls its own scheduler inside each of 5 workers (PLAN.md:7-9).\nELI10: Your job library already knows how to retry a failed job: it counts attempts, waits, re-runs, and gives up after N tries. The plan rebuilds that waiting logic by hand inside every worker because the team wants to pick the exact wait curve. Most libraries let you plug in your own curve as a small function while they keep doing the counting and re-running. Rebuilding it means you also own attempt tracking, surviving a worker restart mid-wait, max-attempt cutoff and dead-lettering.\nStakes if we pick wrong: a hand-rolled scheduler that loses attempt state on restart retries forever or never, and the bug appears in five places at once, at 3am.\nRecommendation: A because the plan's only stated reason (\"full control over the curve\") is exactly what a backoff-strategy hook provides, and it is an already-installed dependency (reuse ladder rung 4).\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Library hook + custom backoff function (recommended)\n ✅ Attempt counting, persistence across restarts, max attempts and dead-lettering stay with proven library code\n ✅ The curve lives in one function (exponential, cap, jitter) instead of five inline copies (human: ~0.5 day / CC: ~10 min)\n ❌ Curve is constrained to whatever the hook's signature allows (usually a delay in ms from attempt number and error)\nB) Custom inline scheduler per worker, as planned\n ✅ No dependency on the library's hook contract; any curve shape or side effect is possible\n ✅ No need to learn or verify the hook API before starting (human: ~2 days / CC: ~30 min, plus ongoing maintenance)\n ❌ Re-implements attempt state, restart recovery, cutoff and dead-letter handling, each a new 3am failure mode across 5 files\nC) Investigate the hook first, decide after\n ✅ Bounded check (hook signature, custom curve, max attempts, jitter, cap) settles the unknown with evidence (human: ~1 hour / CC: ~5 min)\n ✅ Nothing is built on an assumption; R1 stays pending, R2–R4 stay pending\n ❌ Delays the architecture decision; if the hook obviously supports a custom curve this is wasted time\nNet: trading the library's proven retry lifecycle against total freedom over a curve the library almost certainly already lets you set.\nHeader: Retry mechanism\nOptions:\nA) Library hook + custom backoff fn (recommended)\nRegister one custom backoff function with the job library's built-in retry hook; the function owns the curve (exponential base, cap, jitter). The library keeps attempt counting, persistence across worker restarts, max-attempt cutoff and dead-lettering. Human: ~0.5 day / CC: ~10 min. Risk: curve limited to the hook's signature. R2–R4 remain pending.\nB) Custom inline scheduler\nKeep the plan as written: a hand-rolled exponential-backoff scheduler inline in each worker, bypassing the library's retry hooks. Full curve freedom. Human: ~2 days / CC: ~30 min plus maintenance. Risk: re-implements attempt state, restart recovery, cutoff and dead-lettering in 5 places. R2–R4 remain pending.\nC) Investigate hook first\nBounded investigation only: read the library's retry hook signature and confirm it supports a custom curve, max attempts, jitter and a delay cap. No implementation approved; R1 returns to this question with evidence. Human: ~1 hour / CC: ~5 min. R2–R4 remain pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T10:28:56.211Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hook + custom backoff fn — D1 answer (AskUserQuestion, this session)\nAccepted scope: Replace the inline per-worker scheduler (PLAN.md:7-9) with one custom backoff function registered on the job library's built-in retry hook. The function owns the curve: exponential base, delay cap and jitter. The library keeps attempt counting, attempt-state persistence across worker restarts, max-attempt cutoff and dead-lettering. Includes the code, tests and docs for that one function. R2, R3, R4 remain pending.\nHistory: none\n" + }, + { + "at": "2026-10-04T10:28:59.270Z", + "old": "### C. Findings\n1. **[P1] (confidence: 8/10) PLAN.md:7-9", + "new": "### C. Findings\nDispositions: #1 accepted via D1 (A). Scope Challenge result: **scope accepted as-is** (B skipped; the D1 change is an arrangement change that preserves the feature set, not a scope reduction).\n\n1. **[P1] (confidence: 8/10) PLAN.md:7-9" + }, + { + "at": "2026-10-04T10:29:11.583Z", + "old": "## Decision ledger\n\n### R1: Retry mechanism", + "new": "## Section 1: Architecture review\n\nFlow under the approved D1 arrangement (proposed, not yet code):\n\n```\nenqueue(job) ──► library runner ──► worker handler (e.g. processWebhookJob)\n ▲ │\n │ success ├── ok ──► ack / complete\n │ └── throw ──► library retry hook\n │ │\n │ backoffFn(attempt, err)\n │ ├── attempt < max ──► delay = min(cap, base·2^attempt) ± jitter ──► requeue\n │ └── attempt ≥ max ──► dead-letter / failed\n └──────────────── requeue after delay ───────────┘\n```\n\nFindings:\n2. **[P1] (confidence: 8/10) PLAN.md:17-19 — Adding retries to `processWebhookJob()` silently converts at-most-once delivery into at-least-once; duplicate webhooks reach receivers.** Motivating lines: \"The existing `processWebhookJob()` flow gets rewritten as part of this / change. No regression test for the prior at-most-once delivery guarantee / is planned.\" Realistic production failure: the HTTP POST to the receiver succeeds but the worker crashes or times out before acking; the retry re-sends, and the receiver processes the same event twice (double charge, double notification). The plan names the old guarantee and drops it without naming the new one or how receivers dedupe. Runtime evidence: unknown (handler not in repo). Remedy: R5 / D2 below. Test coverage for whichever guarantee is chosen is R3 (Section 3).\n3. **[P2] (confidence: 7/10) PLAN.md:7-9 — Curve parameters (max attempts, delay cap, jitter) are not specified.** Motivating line: \"we want full control over the curve.\" Un-jittered, uncapped exponential backoff is the worst strategy in AWS's published experiments (see evidence notes). Disposition: covered by D1 (A): the accepted scope makes the backoff function own base, cap and jitter and the library own max attempts. Values are implementation detail of that approved behavior; no separate question. Implementation task will name defaults.\n\nDispositions: #2 pending (D2). #3 accepted via D1 (A), carried forward. Security: no new auth or data-access boundary is introduced; the retry path reuses the existing payload fetch (performance implications in Section 4). Distribution: no new artifact.\n\n## Decision ledger\n\n### R1: Retry mechanism" + }, + { + "at": "2026-10-04T10:29:37.837Z", + "old": "Accepted scope: Replace the inline per-worker scheduler (PLAN.md:7-9) with one custom backoff function registered on the job library's built-in retry hook. The function owns the curve: exponential base, delay cap and jitter. The library keeps attempt counting, attempt-state persistence across worker restarts, max-attempt cutoff and dead-lettering. Includes the code, tests and docs for that one function. R2, R3, R4 remain pending.\nHistory: none\n", + "new": "Accepted scope: Replace the inline per-worker scheduler (PLAN.md:7-9) with one custom backoff function registered on the job library's built-in retry hook. The function owns the curve: exponential base, delay cap and jitter. The library keeps attempt counting, attempt-state persistence across worker restarts, max-attempt cutoff and dead-lettering. Includes the code, tests and docs for that one function. R2, R3, R4 remain pending.\nHistory: none\n\n### R5: Webhook delivery guarantee under retry\nFinding: #2, P1, confidence 8/10, PLAN.md:17-19, reviewer: native (plan-eng-review)\nPlan baseline: original proposal — `processWebhookJob()` is rewritten to retry; the prior at-most-once guarantee is dropped with no replacement guarantee named (PLAN.md:17-19). Nothing approved for R5.\nRuntime evidence: unknown. Handler, HTTP client and receiver contracts are not in this repository. Industry practice (evidence notes): webhook delivery is at-least-once with a stable delivery id so receivers can dedupe.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R5 webhook delivery guarantee | at-most-once today; plan drops it, replacement unspecified | at-least-once with a stable idempotency/delivery id sent on every attempt (header + body field), documented for receivers | at-most-once preserved: retry only failures that occur before the HTTP send is attempted; once a send is attempted, no retry (dead-letter instead) | at-least-once with no dedupe id, as the plan implies |\n| R1 retry mechanism | approved D1 (A) | unchanged | unchanged | unchanged |\n| R2 envelope duplication | pending | pending | pending | pending |\n| R3 regression coverage for the chosen guarantee | pending | pending (Section 3) | pending (Section 3) | pending (Section 3) |\n| R4 graph recompute | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — What delivery guarantee does the webhook job keep once it can retry?\nProject/branch/task: `main` of the retry-framework plan; `processWebhookJob()` is rewritten and loses its at-most-once guarantee (PLAN.md:17-19).\nELI10: Today a webhook is sent at most once: if it fails, it is gone, but the receiver never sees a duplicate. Adding retries flips that: the receiver might now get the same event twice, for example when the POST lands but the worker dies before marking it done. You can either tell receivers \"you may see duplicates, here is an id so you can ignore the second one,\" keep the old promise by never retrying once a send was attempted, or retry blindly and hope receivers cope.\nStakes if we pick wrong: receivers double-process events (double charges, double emails), or you keep at-most-once and the reliability win from retries never reaches the webhook path.\nRecommendation: A because at-least-once plus a stable delivery id is the standard webhook contract and receivers can dedupe cheaply, while B gives up most of the reliability you are building this for.\nCompleteness: A=9/10, B=7/10, C=3/10\nPros / cons:\nA) At-least-once with idempotency id (recommended)\n ✅ Retries apply to the whole webhook path, including transient receiver 5xx and network timeouts\n ✅ Stable delivery id on every attempt lets receivers dedupe; this is the documented contract most webhook consumers expect (human: ~1 day / CC: ~15 min)\n ❌ Receivers that ignore the id will see duplicates; the contract change must be documented and announced\nB) Preserve at-most-once\n ✅ No behavior change visible to receivers; no duplicate risk and no receiver-side work\n ✅ Retries still cover pre-send failures such as payload fetch or serialization errors (human: ~0.5 day / CC: ~10 min)\n ❌ A receiver 503 or a timeout after send is final: the event is dead-lettered, so the webhook path gains almost nothing from this project\nC) At-least-once, no dedupe id (plan as written)\n ✅ Smallest change: just let the library retry the existing handler as-is\n ✅ No contract documentation or header work (human: ~2 hours / CC: ~5 min)\n ❌ Duplicates are undetectable by receivers; the first incident is a double side effect with no way to reconcile\nNet: trading a documented, dedupe-able duplicate against either silent duplicates or a webhook path that cannot be retried.\nHeader: Webhook guarantee\nOptions:\nA) At-least-once + idempotency id (recommended)\n`processWebhookJob()` retries through the library hook (D1). Every attempt sends the same stable delivery id (header and body field) so receivers can dedupe; the contract change is documented for receivers. Completeness 9/10. Human: ~1 day / CC: ~15 min. Risk: receivers ignoring the id see duplicates. R2, R3, R4 remain pending.\nB) Preserve at-most-once\nRetries cover only failures before the HTTP send is attempted (payload fetch, serialization, auth). Once a send is attempted, no retry; failure goes to dead-letter. Receivers see no behavior change. Completeness 7/10. Human: ~0.5 day / CC: ~10 min. Risk: post-send transient failures are final. R2, R3, R4 remain pending.\nC) At-least-once, no dedupe id\nLet the library retry the existing handler unchanged. No delivery id, no contract documentation. Completeness 3/10. Human: ~2 hours / CC: ~5 min. Risk: undetectable duplicate side effects at receivers. R2, R3, R4 remain pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T10:30:44.867Z", + "old": "Let the library retry the existing handler unchanged. No delivery id, no contract documentation. Completeness 3/10. Human: ~2 hours / CC: ~5 min. Risk: undetectable duplicate side effects at receivers. R2, R3, R4 remain pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "Let the library retry the existing handler unchanged. No delivery id, no contract documentation. Completeness 3/10. Human: ~2 hours / CC: ~5 min. Risk: undetectable duplicate side effects at receivers. R2, R3, R4 remain pending.\n\nState: approved\nActual answer: A) At-least-once + idempotency id — D2 answer (AskUserQuestion, this session)\nAccepted scope: `processWebhookJob()` retries through the library hook (D1). Every attempt sends the same stable delivery id, as both a request header and a body field, generated once at enqueue time and stored with the job so it survives retries and restarts. The receiver-facing contract change (at-most-once to at-least-once with dedupe id) is documented. Includes the code, tests and docs for that behavior; the regression/acceptance assertions for this guarantee are settled in R3 (Section 3). R2, R3, R4 remain pending.\nHistory: none\n\n### R2: Retry envelope duplication across 5 workers\nFinding: #4, P2, confidence 7/10, PLAN.md:12-14, reviewer: native (plan-eng-review)\nPlan baseline: original proposal — the envelope (compute delay, log attempt, dispatch) is copy-pasted into 5 worker files and left duplicated (PLAN.md:12-14). D1 (A) already moves \"compute delay\" into one backoff function and \"dispatch\" into the library runner. What remains unassigned is \"log attempt\".\nRuntime evidence: unknown. The 5 worker files are not in this repository; \"5\" and the envelope contents come from the plan text. Line estimates below are labelled estimates.\nShared-code rubric:\n- Callers: 5 proposed worker files (PLAN.md:13), proposed callers labelled as plan assumptions, not verified source.\n- Reuse before extracting: the job library's failure/retry lifecycle events (rung 4) are the reuse target for attempt logging; no new helper is needed if the library emits them.\n- Helper size: one listener registered once at worker boot, logging job id, attempt number, next delay and error class.\n- Whole change (estimates): removes ~5 × 15–30 lines of inline envelope = ~75–150 lines; adds ~15–25 lines (backoff function, already approved in D1) + ~10–20 lines (listener). Net savings ~40–125 lines before tests. Tests may add ~40–80 lines total.\n- Blast radius of a shared failure: a broken listener loses retry logging for all workers but does not alter retry behavior, since the library still drives retries.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R2 attempt-logging location | inline in each of 5 workers (duplicated, PLAN.md:12-14) | one listener on the library's failed/retry lifecycle event, registered once at worker boot; inline envelopes deleted from all 5 workers | a small shared `logRetryAttempt()` helper called from each of the 5 workers; inline envelopes deleted except the helper call | inline envelopes left duplicated in 5 workers, as planned (\"refactor later\") |\n| R1 retry mechanism | approved D1 (A) | unchanged | unchanged | unchanged |\n| R5 webhook guarantee | approved D2 (A) | unchanged | unchanged | unchanged |\n| R3 regression coverage | pending | pending | pending | pending |\n| R4 graph recompute | pending | pending | pending | pending |\n\nQuestion D3:\nD3 — Where does retry-attempt logging live once the library drives retries?\nProject/branch/task: `main` of the retry-framework plan; the retry envelope is copy-pasted into 5 worker files and the plan leaves it duplicated (PLAN.md:12-14).\nELI10: After D1 the library computes the delay and re-dispatches, so two of the three copy-pasted pieces disappear on their own. The third piece, \"log this attempt\", still has to live somewhere. Most job libraries fire an event every time a job fails and is scheduled to retry; one listener on that event logs every worker's attempts with zero code in the workers. The alternatives are a tiny shared function each worker calls, or leaving five copies and promising to clean up later.\nStakes if we pick wrong: five copies drift, one worker logs attempts differently or not at all, and the 3am incident has a hole exactly where the failing worker is.\nRecommendation: A because it uses an already-installed dependency (reuse ladder rung 4), needs no change inside the workers, and cannot drift across them.\nCompleteness: A=10/10, B=8/10, C=3/10\nPros / cons:\nA) One library lifecycle-event listener (recommended)\n ✅ Zero retry code left in any worker; every worker's attempts logged identically from one place (human: ~2 hours / CC: ~5 min)\n ✅ Uses the library's own failed/retry event, so logging stays correct even when a new worker is added later\n ❌ Depends on the library emitting a usable event with attempt number and next delay; verify at implementation time\nB) Shared `logRetryAttempt()` helper called per worker\n ✅ Works even if the library emits no retry event; explicit call site in each worker (human: ~3 hours / CC: ~10 min)\n ✅ Single implementation, so log format cannot drift between workers\n ❌ Every worker still carries a call and a new worker can forget it; one more seam to test per caller\nC) Leave the 5 inline copies, refactor later\n ✅ No change to the 5 workers beyond removing the inline scheduler already approved in D1\n ✅ Nothing new to learn or test right now (human: ~0 / CC: ~0)\n ❌ \"Later\" rarely arrives; five copies drift and the duplication finding stands unresolved in the ship review\nNet: trading a dependency on one library event against carrying retry code in five workers that already should have none.\nHeader: Attempt logging\nOptions:\nA) Library event listener (recommended)\nRegister one listener on the job library's failed/retry lifecycle event at worker boot. It logs job id, attempt number, next delay and error class for every worker. Delete the inline envelope from all 5 workers (compute-delay and dispatch already move to the library under D1). Completeness 10/10. Human: ~2 hours / CC: ~5 min. Risk: library must emit the event with attempt data; verify at implementation. R3, R4 remain pending.\nB) Shared logRetryAttempt() helper\nCreate one small helper that logs job id, attempt number, next delay and error class; each of the 5 workers calls it from its failure path. Delete the rest of the inline envelope. Completeness 8/10. Human: ~3 hours / CC: ~10 min. Risk: a new worker can forget the call; one caller-integration test per worker. R3, R4 remain pending.\nC) Leave duplication, refactor later\nKeep the per-worker attempt logging in all 5 workers as planned (PLAN.md:13-14); only the inline scheduler approved in D1 is removed. Completeness 3/10. Human: ~0 / CC: ~0. Risk: copies drift; finding stays open. R3, R4 remain pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T10:31:04.953Z", + "old": "## Decision ledger\n\n### R1: Retry mechanism", + "new": "## Section 2: Code quality review\n\nFindings:\n4. **[P2] (confidence: 7/10) PLAN.md:12-14 — Retry envelope copy-pasted across 5 workers and deliberately left duplicated.** Motivating lines: \"The retry envelope (compute delay, log attempt, dispatch) is duplicated / across 5 worker files with copy-pasted bodies. We will leave the / duplication for now and refactor \\\"later.\\\"\" Under D1 (A), compute-delay and dispatch already leave the workers; only attempt logging remains unassigned. Shared-code rubric evidence is in the R2 ledger record. Runtime evidence unknown (worker files not in repo); 7 because the 5-caller claim is the plan's own statement. Remedy: R2 / D3 below.\n5. **[P2] (confidence: 7/10) PLAN.md:17-18 — Rewriting `processWebhookJob()` and changing its delivery semantics in one step mixes refactor with behavior change.** Motivating line: \"The existing `processWebhookJob()` flow gets rewritten as part of this / change.\" Make-change-easy-first (Beck): land the mechanical move onto the library hook (D1) as its own commit with the existing behavior preserved, then land the at-least-once + delivery-id change (D2) as a second commit with its own tests (R3). Disposition: sequencing guidance carried into Implementation Tasks; no new behavior choice, so no question.\n6. **[P2] (confidence: 7/10) PLAN.md:7-9 — Error classification missing: the plan retries every failure the same way.** Motivating line: \"custom exponential-backoff scheduler ... full control over the curve.\" A 4xx from a webhook receiver (bad URL, revoked secret) or a malformed payload will never succeed on retry; retrying it burns the whole attempt budget and delays dead-lettering. Under D1 (A) the backoff function receives the error and can return the library's fail-now value for permanent errors. Disposition: covered by D1 (A) accepted scope (\"the function owns the curve\"), with the permanent-vs-transient split named as a required branch in Section 3's diagram and Implementation Tasks. No separate question.\n\nEdge cases and error handling flagged for implementation (no decision needed, carried into tasks): attempt number missing or 0 on the first failure; error object without a status code; delay cap smaller than base delay (misconfiguration); jitter producing a negative delay; job payload deleted between attempts (see Section 4).\n\nDispositions: #4 pending (D3). #5 accepted as sequencing guidance (no behavior choice). #6 accepted via D1 (A), carried forward.\n\n## Decision ledger\n\n### R1: Retry mechanism" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37198445662-t1-derived-refuse.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37198445662-t1-derived-refuse.json new file mode 100644 index 000000000..4797d9ce3 --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37198445662-t1-derived-refuse.json @@ -0,0 +1,57 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "ded5316419aefeaf", + "source": { + "run": "37198445662", + "sha": "fd6854b5356431e5e115fca082b21bf237559558", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37198445662-1-eval-slices-1/plan-eng-review-1791113008502-OINJKr" + }, + "census_outcome": "passed", + "expected": "refuse", + "justification": "Batching shape: only D1 was asked; the remaining findings went into the plan without their own questions, so fewer than FLOOR=3 review questions are credited.", + "derived": "First native question of the 37198445662 capture only, with the plan writes up to its answer.", + "input": { + "calls": [ + { + "sessionId": "44fb791b-fa0f-460b-ad12-8b81b2668726", + "toolUseId": "toolu_01G2rk39qmpS2qHHiyoYTdDB", + "answeredAt": "2026-10-04T11:26:55.928Z", + "answer": "Library hooks + custom curve (recommended)", + "question": { + "question": "D1 — Use the job library's retry hooks or roll a custom scheduler in each worker?\nProject/branch/task: gstack-plan-count-ORkyrO on main, plan \"Add background job retry framework\".\nELI10: The job library you already run knows how to retry a failed job: it remembers how many times it tried, waits, and tries again. The plan wants to write that logic by hand inside every worker so the wait times can be tuned. But the wait-time curve is just a function you hand the library; you don't need to rebuild the whole retry engine to change it. Rebuilding means five hand-rolled copies of state tracking that the library's dashboards and dead-letter tooling can't see.\nStakes if we pick wrong: a hand-rolled scheduler that drifts from the library (lost attempt counts on worker crash, retries invisible in the queue UI) means silent job loss or infinite retries that nobody can see.\nRecommendation: A because the plan's only stated reason for a custom scheduler (curve control) is satisfied by passing a custom backoff function to the existing hook, and it removes five copies of state-tracking code. [Layer 1]\nCompleteness: A=9/10, B=5/10, C=n/a (investigation only; approves no implementation)\nPros / cons:\nA) Library hooks + custom curve (recommended) (human: ~1 day / CC: ~20 min)\n ✅ Attempt counts, scheduling and retry visibility stay in the library's persisted state, so a worker crash mid-retry doesn't lose the count\n ✅ The custom curve is one small pure function you can unit test, and the five inline schedulers are deleted rather than maintained\n ❌ The curve is bounded by what the hook exposes (per-attempt delay); anything the hook can't express needs a library upgrade or wrapper\nB) Custom inline scheduler (human: ~3-4 days / CC: ~1-2 h)\n ✅ Total control over timing, state storage and logging with no library constraint\n ✅ No dependency on the library's hook semantics or version\n ❌ Re-implements attempt persistence, scheduling and visibility that the library already has; five copies to keep correct, invisible to the library's queue UI and dead-letter flow\nC) Probe library hook API (CC: ≤30 min, implementation stays pending)\n ✅ Confirms the hook accepts a custom delay function and attempt cap before committing\n ✅ Costs nothing if the answer is already known from the library docs\n ❌ Delays the decision; R1 stays unresolved until the probe result is reviewed\nNet: trading unconstrained control for persisted, visible, library-managed retry state; the plan already admits the shapes are the same.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hooks + custom curve (recommended)", + "description": "Configure each worker's retry via the job library's built-in retry hook, passing a custom exponential backoff function for the curve. Delete the planned inline schedulers. Attempt counts and scheduling stay in the library's persisted state and are visible in its queue UI. Human: ~1 day / CC: ~20 min. Completeness 9/10. Other choices (R2–R6) stay pending." + }, + { + "label": "Custom inline scheduler", + "description": "Keep the plan as written: a hand-rolled exponential-backoff scheduler inline in each of the 5 workers, bypassing the library's retry hooks. Full control over timing and storage, but re-implements attempt persistence, scheduling and visibility, and is invisible to the library's queue UI and dead-letter flow. Human: ~3-4 days / CC: ~1-2 h. Completeness 5/10. Other choices (R2–R6) stay pending." + }, + { + "label": "Probe library hook API", + "description": "Bounded investigation (CC ≤30 min): read the job library's retry hook API to confirm it accepts a custom delay function and an attempt cap, then re-ask R1 with the evidence. Approves no implementation; the retry mechanism and R2–R6 stay pending." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T11:25:36.588Z", + "write": "# Eng Review: Add background job retry framework\n\nReview target (fixed): `PLAN.md` in `gstack-plan-count-ORkyrO` (branch `main`, commit f1f8ffc).\nReviewer: /plan-eng-review (claude host). Report file: this document (user-requested path).\n\n## Original plan (unchanged copy of PLAN.md)\n\n```markdown\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Review preparation\n\n- Design doc: none found. /office-hours prerequisite skipped per the user's instruction.\n- Brain context: no product or recent-decisions digest (cold).\n- Prior learnings: none (cross-project learnings disabled).\n- Retrospective history: the repo contains only `CLAUDE.md` and `PLAN.md` (single commit f1f8ffc). Worker, webhook and job-library paths named in the plan are `not available` in this checkout; nothing can be inferred about their history.\n- Test framework: unknown (no runtime markers, no test files in repo).\n- Web research: Aside not installed; host WebSearch used (read-only, untrusted).\n- Outside voice: `codex_reviews` is disabled in gstack config (CODEX_MODE: disabled).\n" + }, + { + "at": "2026-10-04T11:26:36.008Z", + "old": "- Outside voice: `codex_reviews` is disabled in gstack config (CODEX_MODE: disabled).\n", + "new": "- Outside voice: `codex_reviews` is disabled in gstack config (CODEX_MODE: disabled).\n\n## Scope Challenge\n\nComplexity count: ~5 worker files + `processWebhookJob()` rewrite (likely one of them), 0 new classes/services. Under the 8-file / 2-service gate, so Section B skipped.\n\nFindings:\n1. [P1] (confidence: 8/10) PLAN.md:7-9 — Rebuilds the job library's retry scheduler inline in each worker although the plan admits parity (\"same shape as the library version\"). Curve control is a hook parameter in every mainstream job library; the custom path loses attempt persistence, dashboards and dead-letter handling. Reviewer: Scope Challenge (reuse ladder, Layer 1).\n\n## Decision ledger\n\n### R1: Retry mechanism (library hook vs custom inline scheduler)\nFinding: #1, P1, confidence 8/10, PLAN.md:7-9, reviewer: Scope Challenge\nPlan baseline: original proposal — custom exponential-backoff scheduler written inline in each of the 5 workers; library retry hooks not used (PLAN.md:7-9)\nRuntime evidence: unknown — worker files and job library are not in this checkout; the library's hook API (custom delay function, attempt cap) is unverified\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler per worker (proposal) | library retry hook + custom backoff function per worker; delete inline schedulers | custom inline scheduler as planned | pending; bounded probe (CC ≤30 min): read the library's retry hook API to confirm custom delay fn + attempt cap, then re-ask R1 |\n| R2 shared retry envelope | duplicated in 5 workers (proposal), pending | pending | pending | pending |\n| R3 webhook delivery semantics | at-most-once dropped silently (proposal), pending | pending | pending | pending |\n| R4 dependency-graph cache | refetch per retry (proposal), pending | pending | pending | pending |\n| R5 attempt cap + terminal disposition | unspecified, pending | pending | pending | pending |\n| R6 jitter | unspecified, pending | pending | pending | pending |\n\nQuestion D1:\nD1 — Use the job library's retry hooks or roll a custom scheduler in each worker?\nProject/branch/task: gstack-plan-count-ORkyrO on main, plan \"Add background job retry framework\".\nELI10: The job library you already run knows how to retry a failed job: it remembers how many times it tried, waits, and tries again. The plan wants to write that logic by hand inside every worker so the wait times can be tuned. But the wait-time curve is just a function you hand the library; you don't need to rebuild the whole retry engine to change it. Rebuilding means five hand-rolled copies of state tracking that the library's dashboards and dead-letter tooling can't see.\nStakes if we pick wrong: a hand-rolled scheduler that drifts from the library (lost attempt counts on worker crash, retries invisible in the queue UI) means silent job loss or infinite retries that nobody can see.\nRecommendation: A because the plan's only stated reason for a custom scheduler (curve control) is satisfied by passing a custom backoff function to the existing hook, and it removes five copies of state-tracking code. [Layer 1]\nCompleteness: A=9/10, B=5/10, C=n/a (investigation only; approves no implementation)\nPros / cons:\nA) Library hooks + custom curve (recommended) (human: ~1 day / CC: ~20 min)\n ✅ Attempt counts, scheduling and retry visibility stay in the library's persisted state, so a worker crash mid-retry doesn't lose the count\n ✅ The custom curve is one small pure function you can unit test, and the five inline schedulers are deleted rather than maintained\n ❌ The curve is bounded by what the hook exposes (per-attempt delay); anything the hook can't express needs a library upgrade or wrapper\nB) Custom inline scheduler (human: ~3-4 days / CC: ~1-2 h)\n ✅ Total control over timing, state storage and logging with no library constraint\n ✅ No dependency on the library's hook semantics or version\n ❌ Re-implements attempt persistence, scheduling and visibility that the library already has; five copies to keep correct, invisible to the library's queue UI and dead-letter flow\nC) Probe library hook API (CC: ≤30 min, implementation stays pending)\n ✅ Confirms the hook accepts a custom delay function and attempt cap before committing\n ✅ Costs nothing if the answer is already known from the library docs\n ❌ Delays the decision; R1 stays unresolved until the probe result is reviewed\nNet: trading unconstrained control for persisted, visible, library-managed retry state; the plan already admits the shapes are the same.\nHeader: Retry mechanism\nOptions:\nA) Library hooks + custom curve (recommended)\nConfigure each worker's retry via the job library's built-in retry hook, passing a custom exponential backoff function for the curve. Delete the planned inline schedulers. Attempt counts and scheduling stay in the library's persisted state and are visible in its queue UI. Human: ~1 day / CC: ~20 min. Completeness 9/10. Other choices (R2–R6) stay pending.\nB) Custom inline scheduler\nKeep the plan as written: a hand-rolled exponential-backoff scheduler inline in each of the 5 workers, bypassing the library's retry hooks. Full control over timing and storage, but re-implements attempt persistence, scheduling and visibility, and is invisible to the library's queue UI and dead-letter flow. Human: ~3-4 days / CC: ~1-2 h. Completeness 5/10. Other choices (R2–R6) stay pending.\nC) Probe library hook API\nBounded investigation (CC ≤30 min): read the job library's retry hook API to confirm it accepts a custom delay function and an attempt cap, then re-ask R1 with the evidence. Approves no implementation; the retry mechanism and R2–R6 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37198445662-t1-pass.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37198445662-t1-pass.json new file mode 100644 index 000000000..81afa54b4 --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/37198445662-t1-pass.json @@ -0,0 +1,116 @@ +{ + "schema": "gstack-detector-corpus/v1", + "case": "plan-eng-multi-finding-batching", + "detector": "test/helpers/eng-seeded-coverage.ts#createEngBatchingIssueCounter", + "series_identity": "ded5316419aefeaf", + "source": { + "run": "37198445662", + "sha": "fd6854b5356431e5e115fca082b21bf237559558", + "lane": "periodic", + "artifact": "paid-slice-1-a1", + "path": "shards/skill-e2e-plan-eng-multi-finding-batching/pty-count/ci-37198445662-1-eval-slices-1/plan-eng-review-1791113008502-OINJKr" + }, + "census_outcome": "passed", + "expected": "pass", + "justification": "Census pass: three review-phase questions each bound to their own saved decision-ledger record (R IDs) in the plan; the counter credits each once.", + "input": { + "calls": [ + { + "sessionId": "44fb791b-fa0f-460b-ad12-8b81b2668726", + "toolUseId": "toolu_01G2rk39qmpS2qHHiyoYTdDB", + "answeredAt": "2026-10-04T11:26:55.928Z", + "answer": "Library hooks + custom curve (recommended)", + "question": { + "question": "D1 — Use the job library's retry hooks or roll a custom scheduler in each worker?\nProject/branch/task: gstack-plan-count-ORkyrO on main, plan \"Add background job retry framework\".\nELI10: The job library you already run knows how to retry a failed job: it remembers how many times it tried, waits, and tries again. The plan wants to write that logic by hand inside every worker so the wait times can be tuned. But the wait-time curve is just a function you hand the library; you don't need to rebuild the whole retry engine to change it. Rebuilding means five hand-rolled copies of state tracking that the library's dashboards and dead-letter tooling can't see.\nStakes if we pick wrong: a hand-rolled scheduler that drifts from the library (lost attempt counts on worker crash, retries invisible in the queue UI) means silent job loss or infinite retries that nobody can see.\nRecommendation: A because the plan's only stated reason for a custom scheduler (curve control) is satisfied by passing a custom backoff function to the existing hook, and it removes five copies of state-tracking code. [Layer 1]\nCompleteness: A=9/10, B=5/10, C=n/a (investigation only; approves no implementation)\nPros / cons:\nA) Library hooks + custom curve (recommended) (human: ~1 day / CC: ~20 min)\n ✅ Attempt counts, scheduling and retry visibility stay in the library's persisted state, so a worker crash mid-retry doesn't lose the count\n ✅ The custom curve is one small pure function you can unit test, and the five inline schedulers are deleted rather than maintained\n ❌ The curve is bounded by what the hook exposes (per-attempt delay); anything the hook can't express needs a library upgrade or wrapper\nB) Custom inline scheduler (human: ~3-4 days / CC: ~1-2 h)\n ✅ Total control over timing, state storage and logging with no library constraint\n ✅ No dependency on the library's hook semantics or version\n ❌ Re-implements attempt persistence, scheduling and visibility that the library already has; five copies to keep correct, invisible to the library's queue UI and dead-letter flow\nC) Probe library hook API (CC: ≤30 min, implementation stays pending)\n ✅ Confirms the hook accepts a custom delay function and attempt cap before committing\n ✅ Costs nothing if the answer is already known from the library docs\n ❌ Delays the decision; R1 stays unresolved until the probe result is reviewed\nNet: trading unconstrained control for persisted, visible, library-managed retry state; the plan already admits the shapes are the same.", + "header": "Retry mechanism", + "multiSelect": false, + "options": [ + { + "label": "Library hooks + custom curve (recommended)", + "description": "Configure each worker's retry via the job library's built-in retry hook, passing a custom exponential backoff function for the curve. Delete the planned inline schedulers. Attempt counts and scheduling stay in the library's persisted state and are visible in its queue UI. Human: ~1 day / CC: ~20 min. Completeness 9/10. Other choices (R2–R6) stay pending." + }, + { + "label": "Custom inline scheduler", + "description": "Keep the plan as written: a hand-rolled exponential-backoff scheduler inline in each of the 5 workers, bypassing the library's retry hooks. Full control over timing and storage, but re-implements attempt persistence, scheduling and visibility, and is invisible to the library's queue UI and dead-letter flow. Human: ~3-4 days / CC: ~1-2 h. Completeness 5/10. Other choices (R2–R6) stay pending." + }, + { + "label": "Probe library hook API", + "description": "Bounded investigation (CC ≤30 min): read the job library's retry hook API to confirm it accepts a custom delay function and an attempt cap, then re-ask R1 with the evidence. Approves no implementation; the retry mechanism and R2–R6 stay pending." + } + ] + } + }, + { + "sessionId": "44fb791b-fa0f-460b-ad12-8b81b2668726", + "toolUseId": "toolu_01KcwqZKjdsELndFyMSH34oW", + "answeredAt": "2026-10-04T11:28:34.286Z", + "answer": "Keep at-most-once (no retry)", + "question": { + "question": "D2 — When the webhook job retries, do receivers get duplicates, and how do we prove the chosen behavior?\nProject/branch/task: gstack-plan-count-ORkyrO on main, plan \"Add background job retry framework\".\nELI10: Today a webhook is sent at most once: if it fails, it fails, but the receiver never gets the same event twice. Turning on retries means a webhook that was actually delivered but whose job crashed right after can be sent again. Receivers that bill, email or create records on each webhook will double up. We have to choose on purpose: keep \"never twice\", or retry and give receivers a key so they can spot the repeat. Whatever we pick needs a test that fails if someone breaks it later.\nStakes if we pick wrong: duplicate webhooks cause double charges or double notifications downstream, or, if we keep no-retry, transient failures silently drop deliveries, which is today's behavior but now unexpected next to retried workers.\nRecommendation: B because the plan's purpose is to retry failed work, and an idempotency key is the standard, cheap way to keep duplicates harmless; pick A only if receivers cannot dedupe.\nCompleteness: A=8/10, B=9/10, C=4/10\nPros / cons:\nA) Keep at-most-once (no retry) (human: ~2 h / CC: ~10 min)\n ✅ Zero risk of duplicate side effects at receivers; nothing changes for consumers\n ✅ One small regression test proves the guarantee and guards the rewrite\n ❌ Webhook deliveries still die on the first transient failure, the exact pain the plan exists to fix\nB) At-least-once + idempotency key (recommended) (human: ~1 day / CC: ~30 min)\n ✅ Failed webhooks are actually redelivered, and receivers can discard repeats by key\n ✅ Three focused tests pin the contract: retry happens, key is stable across attempts, no re-send after success\n ❌ A consumer-facing contract change: receivers must honor the key, so it needs documentation and possibly a consumer heads-up\nC) At-least-once, no dedupe key (human: ~2 h / CC: ~10 min)\n ✅ Smallest change; retries work immediately with no header or doc work\n ✅ Fine if every receiver is already idempotent by its own design\n ❌ Any non-idempotent receiver silently gets duplicates after a crash between send and ack\nNet: trading a consumer-visible contract change (key header + docs) for redelivery without duplicate harm; the old guarantee is the alternative, not the default.", + "header": "Webhook delivery", + "multiSelect": false, + "options": [ + { + "label": "Keep at-most-once (no retry)", + "description": "Configure the webhook job's retry hook with a single attempt. Failures land in the failed/dead-letter state for manual replay. Regression test: a failing processWebhookJob makes exactly one outbound call, ends in failed state and is not re-dispatched. Human: ~2 h / CC: ~10 min. Completeness 8/10. R1 unchanged; R2, R4, R5 (other workers), R6 stay pending." + }, + { + "label": "At-least-once + idempotency key (recommended)", + "description": "Enable retries for the webhook job through the library hook and send a stable idempotency key (derived from the job id, identical across attempts) with every delivery. Document the semantic change for receivers. Regression tests: (1) a failed attempt is re-dispatched per the curve; (2) attempts carry the identical key; (3) after a 2xx the job is never re-sent. Human: ~1 day / CC: ~30 min. Completeness 9/10. R1 unchanged; R2, R4, R5, R6 stay pending." + }, + { + "label": "At-least-once, no dedupe key", + "description": "Plan as written: enable retries for the webhook job with no idempotency key. Regression test: a failed attempt is re-dispatched. No duplicate protection; receivers may get repeat deliveries after a crash between send and ack. Human: ~2 h / CC: ~10 min. Completeness 4/10. R1 unchanged; R2, R4, R5, R6 stay pending." + } + ] + } + }, + { + "sessionId": "44fb791b-fa0f-460b-ad12-8b81b2668726", + "toolUseId": "toolu_01XNP4nUXg44MciFpLrF6uHN", + "answeredAt": "2026-10-04T11:29:43.939Z", + "answer": "Cap 5 + dead-letter record (recommended)", + "question": { + "question": "D3 — How many times does a failing job retry, and what happens when it gives up?\nProject/branch/task: gstack-plan-count-ORkyrO on main, plan \"Add background job retry framework\".\nELI10: A retry curve says how long to wait between tries, but not when to stop. A job with a bug in it, or a payload the code can never handle, will fail every time. Without a limit it retries forever and quietly eats worker capacity. When it does stop, we need to keep the job's data and the last error somewhere a human can look at it and re-run it after a fix; otherwise the failure vanishes.\nStakes if we pick wrong: poisoned jobs loop forever and starve healthy ones, or exhausted jobs are dropped with no payload and no error to debug, so the user's work is silently lost.\nRecommendation: A because a cap plus a replayable dead-letter record is how every mainstream job library already behaves, and the record is what makes a 3am failure diagnosable.\nCompleteness: A=9/10, B=6/10, C=2/10\nPros / cons:\nA) Cap 5 + dead-letter record (recommended) (human: ~half day / CC: ~15 min)\n ✅ Poisoned jobs stop after a bounded number of tries and stop stealing capacity from healthy ones\n ✅ Exhausted jobs keep payload, attempt count and last error, so an engineer can diagnose and replay after a fix\n ❌ Someone has to own the dead-letter set; without an owner it becomes a graveyard (alerting is a separate choice)\nB) Cap 5, library default failed state (human: ~1 h / CC: ~5 min)\n ✅ Bounded retries with almost no extra code; relies on whatever the library stores by default\n ✅ Fewer moving parts than a custom dead-letter record\n ❌ If the library's default failed state lacks the last error or attempt count, exhausted jobs are hard to diagnose or replay\nC) Unbounded retries (plan as written) (human: 0 / CC: 0)\n ✅ No job is ever declared failed; transient outages of any length eventually recover\n ✅ Nothing to configure or test for exhaustion\n ❌ A job that can never succeed retries forever, consumes worker slots and hides the bug indefinitely\nNet: trading a small amount of exhaustion handling and ownership for bounded worker load and diagnosable, replayable failures.", + "header": "Retry cap", + "multiSelect": false, + "options": [ + { + "label": "Cap 5 + dead-letter record (recommended)", + "description": "Configure a per-worker max-attempts value through the library hook (default 5). On exhaustion the job moves to the library's failed/dead-letter set keeping payload, job id, attempt count, last error and stack, so it can be replayed. Required proof: attempt below cap re-dispatches; attempt at cap lands in the failed set with attempts and last error recorded. Human: ~half day / CC: ~15 min. Completeness 9/10. R1, R3 unchanged; R2, R4, R6, R7 pending." + }, + { + "label": "Cap 5, library default failed state", + "description": "Configure a per-worker max-attempts value (default 5) and rely on the library's default failed state with no added enrichment. Required proof: attempt at cap stops re-dispatch. Human: ~1 h / CC: ~5 min. Completeness 6/10. R1, R3 unchanged; R2, R4, R6, R7 pending." + }, + { + "label": "Unbounded retries (plan as written)", + "description": "No attempt cap and no terminal disposition; a failing job retries until it succeeds. No exhaustion test. Completeness 2/10. R1, R3 unchanged; R2, R4, R6, R7 pending." + } + ] + } + } + ], + "planEvents": [ + { + "at": "2026-10-04T11:25:36.588Z", + "write": "# Eng Review: Add background job retry framework\n\nReview target (fixed): `PLAN.md` in `gstack-plan-count-ORkyrO` (branch `main`, commit f1f8ffc).\nReviewer: /plan-eng-review (claude host). Report file: this document (user-requested path).\n\n## Original plan (unchanged copy of PLAN.md)\n\n```markdown\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Review preparation\n\n- Design doc: none found. /office-hours prerequisite skipped per the user's instruction.\n- Brain context: no product or recent-decisions digest (cold).\n- Prior learnings: none (cross-project learnings disabled).\n- Retrospective history: the repo contains only `CLAUDE.md` and `PLAN.md` (single commit f1f8ffc). Worker, webhook and job-library paths named in the plan are `not available` in this checkout; nothing can be inferred about their history.\n- Test framework: unknown (no runtime markers, no test files in repo).\n- Web research: Aside not installed; host WebSearch used (read-only, untrusted).\n- Outside voice: `codex_reviews` is disabled in gstack config (CODEX_MODE: disabled).\n" + }, + { + "at": "2026-10-04T11:26:36.008Z", + "old": "- Outside voice: `codex_reviews` is disabled in gstack config (CODEX_MODE: disabled).\n", + "new": "- Outside voice: `codex_reviews` is disabled in gstack config (CODEX_MODE: disabled).\n\n## Scope Challenge\n\nComplexity count: ~5 worker files + `processWebhookJob()` rewrite (likely one of them), 0 new classes/services. Under the 8-file / 2-service gate, so Section B skipped.\n\nFindings:\n1. [P1] (confidence: 8/10) PLAN.md:7-9 — Rebuilds the job library's retry scheduler inline in each worker although the plan admits parity (\"same shape as the library version\"). Curve control is a hook parameter in every mainstream job library; the custom path loses attempt persistence, dashboards and dead-letter handling. Reviewer: Scope Challenge (reuse ladder, Layer 1).\n\n## Decision ledger\n\n### R1: Retry mechanism (library hook vs custom inline scheduler)\nFinding: #1, P1, confidence 8/10, PLAN.md:7-9, reviewer: Scope Challenge\nPlan baseline: original proposal — custom exponential-backoff scheduler written inline in each of the 5 workers; library retry hooks not used (PLAN.md:7-9)\nRuntime evidence: unknown — worker files and job library are not in this checkout; the library's hook API (custom delay function, attempt cap) is unverified\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler per worker (proposal) | library retry hook + custom backoff function per worker; delete inline schedulers | custom inline scheduler as planned | pending; bounded probe (CC ≤30 min): read the library's retry hook API to confirm custom delay fn + attempt cap, then re-ask R1 |\n| R2 shared retry envelope | duplicated in 5 workers (proposal), pending | pending | pending | pending |\n| R3 webhook delivery semantics | at-most-once dropped silently (proposal), pending | pending | pending | pending |\n| R4 dependency-graph cache | refetch per retry (proposal), pending | pending | pending | pending |\n| R5 attempt cap + terminal disposition | unspecified, pending | pending | pending | pending |\n| R6 jitter | unspecified, pending | pending | pending | pending |\n\nQuestion D1:\nD1 — Use the job library's retry hooks or roll a custom scheduler in each worker?\nProject/branch/task: gstack-plan-count-ORkyrO on main, plan \"Add background job retry framework\".\nELI10: The job library you already run knows how to retry a failed job: it remembers how many times it tried, waits, and tries again. The plan wants to write that logic by hand inside every worker so the wait times can be tuned. But the wait-time curve is just a function you hand the library; you don't need to rebuild the whole retry engine to change it. Rebuilding means five hand-rolled copies of state tracking that the library's dashboards and dead-letter tooling can't see.\nStakes if we pick wrong: a hand-rolled scheduler that drifts from the library (lost attempt counts on worker crash, retries invisible in the queue UI) means silent job loss or infinite retries that nobody can see.\nRecommendation: A because the plan's only stated reason for a custom scheduler (curve control) is satisfied by passing a custom backoff function to the existing hook, and it removes five copies of state-tracking code. [Layer 1]\nCompleteness: A=9/10, B=5/10, C=n/a (investigation only; approves no implementation)\nPros / cons:\nA) Library hooks + custom curve (recommended) (human: ~1 day / CC: ~20 min)\n ✅ Attempt counts, scheduling and retry visibility stay in the library's persisted state, so a worker crash mid-retry doesn't lose the count\n ✅ The custom curve is one small pure function you can unit test, and the five inline schedulers are deleted rather than maintained\n ❌ The curve is bounded by what the hook exposes (per-attempt delay); anything the hook can't express needs a library upgrade or wrapper\nB) Custom inline scheduler (human: ~3-4 days / CC: ~1-2 h)\n ✅ Total control over timing, state storage and logging with no library constraint\n ✅ No dependency on the library's hook semantics or version\n ❌ Re-implements attempt persistence, scheduling and visibility that the library already has; five copies to keep correct, invisible to the library's queue UI and dead-letter flow\nC) Probe library hook API (CC: ≤30 min, implementation stays pending)\n ✅ Confirms the hook accepts a custom delay function and attempt cap before committing\n ✅ Costs nothing if the answer is already known from the library docs\n ❌ Delays the decision; R1 stays unresolved until the probe result is reviewed\nNet: trading unconstrained control for persisted, visible, library-managed retry state; the plan already admits the shapes are the same.\nHeader: Retry mechanism\nOptions:\nA) Library hooks + custom curve (recommended)\nConfigure each worker's retry via the job library's built-in retry hook, passing a custom exponential backoff function for the curve. Delete the planned inline schedulers. Attempt counts and scheduling stay in the library's persisted state and are visible in its queue UI. Human: ~1 day / CC: ~20 min. Completeness 9/10. Other choices (R2–R6) stay pending.\nB) Custom inline scheduler\nKeep the plan as written: a hand-rolled exponential-backoff scheduler inline in each of the 5 workers, bypassing the library's retry hooks. Full control over timing and storage, but re-implements attempt persistence, scheduling and visibility, and is invisible to the library's queue UI and dead-letter flow. Human: ~3-4 days / CC: ~1-2 h. Completeness 5/10. Other choices (R2–R6) stay pending.\nC) Probe library hook API\nBounded investigation (CC ≤30 min): read the job library's retry hook API to confirm it accepts a custom delay function and an attempt cap, then re-ask R1 with the evidence. Approves no implementation; the retry mechanism and R2–R6 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T11:28:04.336Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hooks + custom curve (D1 answer)\nAccepted scope: Configure each worker's retry through the job library's built-in retry hook with a custom exponential backoff function; delete the planned inline schedulers. Attempt counts and scheduling remain in the library's persisted state. R2–R6 stay pending.\nHistory: none\n\nScope Challenge result: scope accepted as-is (feature list unchanged; R1 changed the mechanism, not the scope). MODE = FULL_REVIEW. Dispositions: #1 accepted (D1 → A).\n\n## Section 1: Architecture review\n\nRetry flow with R1 applied (library hook + custom curve):\n\n```\nenqueue(job) ──► worker.process(job)\n │\n ├─ success ──► ack / complete (never re-dispatched)\n │\n └─ throw(err)\n │\n ├─ permanent error (4xx, validation) ──► fail now ──► dead-letter [R5]\n │\n └─ retryable error\n │\n ├─ attempt < cap [R5] ──► delay = curve(attempt) ± jitter [R6] ──► library re-dispatch\n │ (attempt count persisted by library)\n └─ attempt == cap ──► dead-letter {payload, job id, attempts, last error} [R5]\n\nprocessWebhookJob (today: at-most-once) processWebhookJob (with retries)\n send HTTP ──► fail ──► job marked failed, no re-send send HTTP ──► fail ──► re-send ──► receiver may get N copies [R3]\n```\n\nFindings:\n2. [P1] (confidence: 8/10) PLAN.md:17-19 — \"No regression test for the prior at-most-once delivery guarantee is planned.\" Adding retries to `processWebhookJob()` turns at-most-once into at-least-once: a webhook that reached the receiver but whose job failed afterward (timeout after send, crash before ack) is sent again. The plan drops a delivery guarantee silently with no decision on whether receivers can tolerate duplicates. Reviewer: Architecture.\n3. [P2] (confidence: 7/10) PLAN.md:7-9 — The curve is specified but no attempt cap or terminal disposition is. Without a cap a poisoned job retries forever; without a dead-letter record its payload and last error are lost. Reviewer: Architecture.\n4. [P2] (confidence: 7/10) PLAN.md:7 — Pure exponential backoff with no jitter. When a shared downstream (DB, third-party API) blips, every failed job across 5 workers retries at the same instant, repeating the outage (thundering herd). Reviewer: Architecture.\n\nProduction failure per new path: (a) worker crashes after the webhook HTTP send but before ack → library re-dispatches → duplicate webhook (R3). (b) Downstream API returns 429 for 10 minutes → all jobs hit cap in lockstep without jitter, dead-letter in a burst (R5/R6). (c) Permanent 4xx retried as if transient → wasted attempts, delayed dead-letter (R5).\nSecurity: no new auth or data-access boundary; dead-letter records contain full payloads, so they inherit the payload's data classification. Distribution: no new artifacts, no CI/CD changes.\n\n### R3: processWebhookJob delivery semantics under retry (behavior + regression contract)\nFinding: #2, P1, confidence 8/10, PLAN.md:17-19, reviewer: Architecture\nPlan baseline: original proposal — processWebhookJob rewritten to retry; at-most-once guarantee dropped with no regression test (PLAN.md:17-19)\nRuntime evidence: unknown — processWebhookJob source and its callers are not in this checkout; whether receivers already dedupe is unverified\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | approved: library hook + custom curve (D1 → A) | unchanged | unchanged | unchanged |\n| R3 webhook delivery semantics | at-most-once dropped silently (proposal) | at-most-once kept: webhook job attempts = 1 via hook config; failures go to failed/dead-letter state for manual replay | at-least-once with idempotency key: retries on, every attempt sends the same stable key (derived from job id), consumer-facing doc of the change | at-least-once, no dedupe key (proposal as written) |\n| R3 regression assertions (required proof) | none planned | test: failing webhook job → exactly 1 outbound call, job in failed state, not re-dispatched | tests: (1) failed attempt is re-dispatched per curve; (2) attempts 1..n carry identical idempotency key; (3) after 2xx the job is never re-sent | test: failed attempt is re-dispatched; no duplicate protection asserted |\n| R2 shared envelope | pending | pending | pending | pending |\n| R4 graph cache | pending | pending | pending | pending |\n| R5 cap + terminal disposition (non-webhook workers) | pending | pending (webhook cap fixed at 1 by this behavior) | pending | pending |\n| R6 jitter | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — When the webhook job retries, do receivers get duplicates, and how do we prove the chosen behavior?\nProject/branch/task: gstack-plan-count-ORkyrO on main, plan \"Add background job retry framework\".\nELI10: Today a webhook is sent at most once: if it fails, it fails, but the receiver never gets the same event twice. Turning on retries means a webhook that was actually delivered but whose job crashed right after can be sent again. Receivers that bill, email or create records on each webhook will double up. We have to choose on purpose: keep \"never twice\", or retry and give receivers a key so they can spot the repeat. Whatever we pick needs a test that fails if someone breaks it later.\nStakes if we pick wrong: duplicate webhooks cause double charges or double notifications downstream, or, if we keep no-retry, transient failures silently drop deliveries, which is today's behavior but now unexpected next to retried workers.\nRecommendation: B because the plan's purpose is to retry failed work, and an idempotency key is the standard, cheap way to keep duplicates harmless; pick A only if receivers cannot dedupe.\nCompleteness: A=8/10, B=9/10, C=4/10\nPros / cons:\nA) Keep at-most-once (no retry) (human: ~2 h / CC: ~10 min)\n ✅ Zero risk of duplicate side effects at receivers; nothing changes for consumers\n ✅ One small regression test proves the guarantee and guards the rewrite\n ❌ Webhook deliveries still die on the first transient failure, the exact pain the plan exists to fix\nB) At-least-once + idempotency key (recommended) (human: ~1 day / CC: ~30 min)\n ✅ Failed webhooks are actually redelivered, and receivers can discard repeats by key\n ✅ Three focused tests pin the contract: retry happens, key is stable across attempts, no re-send after success\n ❌ A consumer-facing contract change: receivers must honor the key, so it needs documentation and possibly a consumer heads-up\nC) At-least-once, no dedupe key (human: ~2 h / CC: ~10 min)\n ✅ Smallest change; retries work immediately with no header or doc work\n ✅ Fine if every receiver is already idempotent by its own design\n ❌ Any non-idempotent receiver silently gets duplicates after a crash between send and ack\nNet: trading a consumer-visible contract change (key header + docs) for redelivery without duplicate harm; the old guarantee is the alternative, not the default.\nHeader: Webhook delivery\nOptions:\nA) Keep at-most-once (no retry)\nConfigure the webhook job's retry hook with a single attempt. Failures land in the failed/dead-letter state for manual replay. Regression test: a failing processWebhookJob makes exactly one outbound call, ends in failed state and is not re-dispatched. Human: ~2 h / CC: ~10 min. Completeness 8/10. R1 unchanged; R2, R4, R5 (other workers), R6 stay pending.\nB) At-least-once + idempotency key (recommended)\nEnable retries for the webhook job through the library hook and send a stable idempotency key (derived from the job id, identical across attempts) with every delivery. Document the semantic change for receivers. Regression tests: (1) a failed attempt is re-dispatched per the curve; (2) attempts carry the identical key; (3) after a 2xx the job is never re-sent. Human: ~1 day / CC: ~30 min. Completeness 9/10. R1 unchanged; R2, R4, R5, R6 stay pending.\nC) At-least-once, no dedupe key\nPlan as written: enable retries for the webhook job with no idempotency key. Regression test: a failed attempt is re-dispatched. No duplicate protection; receivers may get repeat deliveries after a crash between send and ack. Human: ~2 h / CC: ~10 min. Completeness 4/10. R1 unchanged; R2, R4, R5, R6 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T11:29:22.757Z", + "old": "Plan as written: enable retries for the webhook job with no idempotency key. Regression test: a failed attempt is re-dispatched. No duplicate protection; receivers may get repeat deliveries after a crash between send and ack. Human: ~2 h / CC: ~10 min. Completeness 4/10. R1 unchanged; R2, R4, R5, R6 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "Plan as written: enable retries for the webhook job with no idempotency key. Regression test: a failed attempt is re-dispatched. No duplicate protection; receivers may get repeat deliveries after a crash between send and ack. Human: ~2 h / CC: ~10 min. Completeness 4/10. R1 unchanged; R2, R4, R5, R6 stay pending.\n\nState: approved\nActual answer: A) Keep at-most-once (no retry) (D2 answer)\nAccepted scope: processWebhookJob keeps at-most-once delivery. Its retry hook is configured with a single attempt; failures land in the failed/dead-letter state for manual replay. Regression contract (required proof, CRITICAL): a failing processWebhookJob makes exactly one outbound call, ends in failed state and is not re-dispatched. Document that the webhook job is intentionally excluded from retries. R1 unchanged; R2, R4, R5 (other workers), R6 stay pending.\nHistory: none\n\n### R5: Attempt cap and terminal disposition for the four retried workers\nFinding: #3, P2, confidence 7/10, PLAN.md:7-9, reviewer: Architecture\nPlan baseline: original proposal — exponential-backoff curve only; no attempt cap, no terminal disposition specified (PLAN.md:7-9)\nRuntime evidence: unknown — worker files and the library's failed-job/dead-letter facility are not in this checkout\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 retry mechanism | approved: library hook + custom curve (D1 → A) | unchanged | unchanged | unchanged |\n| R3 webhook delivery | approved: at-most-once, 1 attempt (D2 → A) | unchanged | unchanged | unchanged |\n| R5 attempt cap | unspecified (proposal) | per-worker max attempts via hook config, default 5 | per-worker max attempts via hook config, default 5 | none; retry until success (proposal) |\n| R5 terminal disposition on exhaustion | unspecified (proposal) | dead-letter record kept by the library's failed set with payload, job id, attempt count, last error + stack; replayable | library default failed state, no enrichment beyond what the library stores | none; job never terminates |\n| R5 required proof | none | tests: attempt n`\n(\"Plan: Add background job retry framework\"). Reviewer: /plan-eng-review,\nsession 702-1791142398-ec68c409, branch main, commit e767d72.\n\n## Original plan (unchanged copy)\n\n```markdown\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n```\n\n## Scope Challenge — assessment\n\n**Evidence available.** The review repo contains only `CLAUDE.md` and `PLAN.md`.\nThe five worker files, `processWebhookJob()`, the job library and its retry hooks\nare named by the plan but are `not available` in this checkout. Every finding\nbelow is calibrated against the plan text (file:line) plus industry practice;\nruntime behavior of the existing code is unknown.\n\n**What already solves each sub-problem.**\n- Exponential backoff scheduling: the plan says the existing job library already\n ships retry hooks and the custom version is \"same shape\" (PLAN.md:7-9). Reuse\n ladder rung 4 (already-installed dependency). Whether the hooks accept a custom\n backoff function is unknown; typical libraries (BullMQ, Sidekiq, Oban, Laravel\n queues, Celery) do.\n- Retry envelope (compute delay, log attempt, dispatch): no shared helper exists\n per the plan; five proposed copies (PLAN.md:12-14).\n- Delivery guarantee: today `processWebhookJob()` is at-most-once (PLAN.md:17-19).\n Any retry layer turns it into at-least-once unless dispatch is excluded or\n deduped.\n\n**Minimum change that achieves the goal.** Configure the library's retry hook with\na custom curve; one small envelope helper for the five workers only if the library\nhook cannot own the envelope; settle webhook delivery semantics explicitly and\ncover them with a regression test; compute the dependency graph once per job.\n\n**Complexity check (estimates).** Proposed changed files: 5 worker files, plus 0-1\nnew helper module. New classes/services: 0-1 (a scheduler/helper). Below the\n8-file / 2-class gate, so Scope Challenge B is skipped.\n\n**Search check.** Aside not installed; host WebSearch used (one read-only query,\nsanitized). Consensus: use the queue library's declarative retry/backoff with\njitter and a max-attempt cap, route exhausted jobs to a dead-letter queue, and\ntreat duplicates as policy under at-least-once, so side effects need idempotency\nkeys. Sources:\n[DEV: Node.js job queue retry/ack/idempotency](https://dev.to/thomasmoore157/how-to-build-a-nodejs-background-job-queue-retry-ack-and-idempotency-59ea),\n[Berkanium: job queues done right](https://www.berkanium.net/articles/job-queues-done-right),\n[Moments Log: retry policy checklist](https://www.momentslog.com/development/background-job-retry-policy-checklist-how-to-prevent-queues-from-amplifying-production-failures),\n[benmoataz: idempotent job queue](https://www.benmoataz.com/posts/idempotent-job-queue/),\n[Laravel queue best practices](https://dev.to/laravelo11y/laravel-background-jobs-12-best-practices-for-production-queues-5gcm).\nLayering: library retry hooks are **[Layer 1]**; a hand-rolled inline scheduler\nis **[Layer 3]** with no first-principles reason given beyond \"control over the\ncurve\", which a backoff callback already provides.\n\n**TODOS cross-reference.** No `TODOS.md` in the repo.\n\n**Completeness check.** A regression test for at-most-once delivery and a cached\ndependency graph each cost CC minutes; both are deferred in the plan.\n\n**Distribution check.** No new artifacts; nothing to publish.\n\n## Scope Challenge — findings\n\n1. `[P1] (confidence: 7/10) PLAN.md:7-9 — Plan rebuilds the job library's retry\n scheduling inline in each worker although it admits the result is \"same shape\"\n as the built-in; \"full control over the curve\" is normally a backoff callback\n on the existing hook, not a reason to bypass the library's attempt counting,\n dead-lettering and visibility.` Motivating lines: \"We'll roll a custom\n exponential-backoff scheduler inline in each worker rather than use the\n existing job library's built-in retry hooks. Same shape as the library\n version, but we want full control over the curve.\" Not 9/10 because the\n library is not in this checkout, so whether its hook accepts a custom curve\n is unverified.\n\nRelated findings on duplication, delivery semantics and per-retry refetch are\nhandled in Sections 2, 1/3 and 4 respectively; the first depends on R1's answer.\n\n## Decision ledger\n\n### R1: Retry scheduling mechanism (library hook vs custom inline scheduler)\nFinding: Scope Challenge #1, P1, confidence 7/10, PLAN.md:7-9, reviewer: plan-eng-review (Claude)\nPlan baseline: original proposal, custom exponential-backoff scheduler inline in each of the 5 workers, bypassing the library retry hooks (PLAN.md:7-9). Nothing approved yet.\nRuntime evidence: unknown. Job library, its hook API and the 5 worker files are not in this checkout. Industry practice (search check above) favors the library's declarative retry with a backoff callback.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R1 scheduling mechanism | custom inline scheduler per worker (proposed) | library retry hook with a custom backoff callback; fall back to custom only if a probe proves the hook cannot express the curve | custom inline scheduler as planned | bounded probe of the library hook API (one read of its retry/backoff config and attempt-counting semantics); mechanism stays pending |\n| Backoff curve shape (exponent, base, cap, jitter) | unspecified | unspecified, pending | unspecified, pending | unspecified, pending |\n| R2 envelope sharing (Section 2) | 5 copies, refactor later (proposed) | pending; may become moot if the hook owns the envelope | pending | pending |\n| R3 webhook delivery semantics (Sections 1/3) | at-most-once, no regression test (proposed) | pending | pending | pending |\n| R4 dependency-graph caching (Section 4) | recompute per retry (proposed) | pending | pending | pending |\n\nQuestion D1:\nD1 — Use the job library's retry hook or roll a custom inline scheduler?\nProject/branch/task: main branch of the retry-framework plan; the plan currently bypasses the existing job library's retry hooks in 5 workers.\nELI10: The job library you already run knows how to retry a failed job: it counts attempts, waits, re-queues, and gives up after a limit. The plan rewrites that logic by hand inside every worker because the team wants to shape the wait-time curve. Most libraries let you pass your own curve to the built-in retry, so hand-rolling buys little and loses the library's bookkeeping.\nStakes if we pick wrong: a hand-rolled scheduler owns attempt counting, dead-lettering and crash recovery itself; a bug there means jobs retry forever or vanish silently, in five places, and the library's dashboards and metrics stop telling the truth.\nRecommendation: A because the plan itself says the custom version is \"same shape\" as the library's, and a backoff callback gives full control of the curve without re-owning retry state.\nCompleteness: A=9/10, B=6/10, C=n/a (investigation only)\nPros / cons:\nA) Library hook + custom backoff callback (recommended)\n ✅ Library keeps attempt counting, max tries, dead-letter routing and crash recovery; one tested code path instead of five new ones\n ✅ Curve is still fully yours: exponent, cap and jitter live in one callback that is trivial to unit test (human: ~1 day / CC: ~10 min)\n ❌ If the library's hook turns out not to accept a custom curve, you spend a short probe before falling back to a custom scheduler\nB) Custom inline scheduler as planned\n ✅ No dependency on the library's hook API or its release cadence; the team controls every line\n ✅ Can express curves or side effects the hook cannot, if such a need exists\n ❌ Re-owns attempt counting, persistence across worker crashes and dead-lettering in 5 workers; high blast radius for a 3am bug (human: ~1 week / CC: ~45 min plus tests)\nC) Probe the library hook first, decide after\n ✅ Replaces an assumption with evidence: one read of the retry config API settles whether a backoff callback is supported\n ✅ Bounded: one hour human / a few CC minutes; no implementation changes until the answer lands\n ❌ Delays the decision; everything downstream (envelope sharing, tests) waits on it\nNet: trade a short API probe and a thin callback against re-owning retry state in five workers.\nHeader: Retry mechanism\nOptions:\nA) Library hook + custom backoff callback (recommended)\nConfigure the existing job library's retry hook with a custom backoff callback (exponent, cap, jitter in one function). The library keeps attempt counting, max tries, dead-letter routing and crash recovery. Fall back to a custom scheduler only if a probe proves the hook cannot express the curve. Completeness 9/10. Effort: human ~1 day / CC ~10 min. Backoff shape, envelope sharing (R2), webhook semantics (R3) and graph caching (R4) stay pending.\nB) Custom inline scheduler as planned\nKeep the plan: hand-rolled exponential-backoff scheduler inline in each of the 5 workers, bypassing the library's retry hooks. The team re-owns attempt counting, persistence across crashes and dead-lettering. Completeness 6/10. Effort: human ~1 week / CC ~45 min plus tests. Backoff shape, R2, R3 and R4 stay pending.\nC) Probe the library hook first, decide after\nBounded investigation only: read the job library's retry/backoff configuration API and attempt-counting semantics (one hour human / a few CC minutes), then return to this question. Approves no implementation; the mechanism and R2, R3, R4 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T19:37:47.896Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Library hook + custom backoff callback (D1 answer)\nAccepted scope: Replace the custom inline scheduler with the existing job library's retry hook configured with one custom backoff callback (exponent, cap, jitter in one function). The library keeps attempt counting, max tries, dead-letter routing and crash recovery. Fall back to a custom scheduler only if a probe proves the hook cannot express the curve. Backoff shape/parameters, R2, R3 and R4 stay pending.\nHistory: none\n\n**Scope Challenge result:** scope accepted as-is (D1 changes the mechanism, not the feature scope). Dispositions: finding #1 accepted via D1.\n\n### R2: Retry envelope sharing across the 5 workers\nFinding: Section 2 (Code quality), P2, confidence 8/10, PLAN.md:12-14, reviewer: plan-eng-review (Claude)\nPlan baseline: original proposal, copy-pasted envelope (compute delay, log attempt, dispatch) in 5 worker files, refactor \"later\". D1 (approved) moves delay computation and dispatch into the library hook plus one backoff callback.\nRuntime evidence: unknown; worker files not in this checkout. Shared-code rubric: 5 proposed callers (plan-stated, not verified source); common behavior per plan text; reliability gain is one place to fix curve/logging bugs.\nComparison grid: pending (prepared in Section 2)\nQuestion: pending\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n\n### R3: Webhook delivery guarantee under retry, with its regression contract\nFinding: Section 1 (Architecture) #1, P1, confidence 8/10, PLAN.md:17-19, reviewer: plan-eng-review (Claude)\nPlan baseline: original proposal: `processWebhookJob()` rewritten inside the retry change; prior guarantee at-most-once; no regression test planned. D1 (approved) puts retries under the library hook, which re-runs the whole job on failure or crash. No delivery semantics approved yet.\nRuntime evidence: unknown; `processWebhookJob()` and its callers/receivers are not in this checkout. Industry sources (search check) agree retries make duplicates policy unless side effects are idempotent.\nComparison grid:\n\n| Choice | Current | A | B | C |\n|---|---|---|---|---|\n| R3 webhook delivery guarantee | at-most-once (existing), plan silent on what retries do to it | at-most-once preserved: library retry applies only to failures before the HTTP send is attempted; the send stage runs with max attempts 1 (or a persisted \"dispatching\" marker checked before send) so crash-after-send is not re-sent | at-least-once: webhook retried on send failure with a stable idempotency key per job (not per attempt) in the request; contract change documented for receivers | bounded investigation: audit receivers for dedupe behavior and read `processWebhookJob()`; guarantee stays pending |\n| Regression assertions (required proof of the chosen guarantee) | none planned | test: failure injected after send → exactly one HTTP call, job marked failed/dead-lettered, not re-queued; test: failure before send → job retried per curve; test: worker crash simulated between send and ack → no second send | test: send failure → retried per curve with identical idempotency key on every attempt; test: key stable across attempts and process restart; test: exhausted attempts → dead-letter, receiver contract doc updated | none until the guarantee is chosen |\n| R1 mechanism (approved D1) | library hook + callback | unchanged | unchanged | unchanged |\n| R2 envelope sharing | pending | pending | pending | pending |\n| R4 dependency-graph caching | pending | pending | pending | pending |\n| R5 retry policy parameters | pending | pending | pending | pending |\n\nQuestion D2:\nD2 — What delivery guarantee does the webhook job keep once retries exist, and what test proves it?\nProject/branch/task: main branch of the retry-framework plan; `processWebhookJob()` is being rewritten under the newly approved library retry hook (D1).\nELI10: Today a webhook fires at most once: if it fails, it is not sent again, so receivers never see duplicates. A retry framework re-runs failed jobs, which by definition can send the same webhook twice. The plan rewrites the webhook job without saying which behavior receivers should get, and plans no test to prove it. We need to pick the guarantee and lock it with a regression test before the rewrite.\nStakes if we pick wrong: external receivers either start getting duplicate webhooks they never deduped (double charges, double notifications) or the team believes retries protect webhooks when they do not; either failure is silent because nothing tests it.\nRecommendation: A because it preserves the contract receivers already rely on, still lets retries cover transient pre-send failures, and keeps a later move to at-least-once a separate, reversible decision.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Keep at-most-once; retry only before send (recommended)\n ✅ Receivers see no behavior change; no coordination with external consumers or idempotency rollout needed\n ✅ Transient failures while loading the payload or building the graph still get retried; three regression tests pin the contract (human: ~1 day / CC: ~15 min)\n ❌ A webhook whose HTTP send fails stays failed and dead-lettered; operators must replay it by hand\nB) Move to at-least-once with an idempotency key\n ✅ Send failures self-heal through the retry curve; this is how most webhook platforms behave\n ✅ A per-job idempotency key gives receivers a clean way to dedupe, and tests pin key stability across attempts\n ❌ It is a receiver-facing contract change: every consumer must dedupe on the key before this ships, or they get duplicates (human: ~1 week incl. consumer outreach / CC: ~30 min code)\nC) Investigate receivers and the current job first\n ✅ Replaces guesses about receiver dedupe behavior and the current send path with evidence\n ✅ Bounded: read `processWebhookJob()` and the receiver list (human: ~half day / CC: ~5 min)\n ❌ The guarantee and its tests stay undecided; the rewrite cannot start safely until this returns\nNet: preserve the existing promise and retry only the safe half of the job, or change the promise and pay the consumer-coordination cost up front.\nHeader: Webhook guarantee\nOptions:\nA) Keep at-most-once; retry only before send (recommended)\nLibrary retry applies only to failures before the HTTP send is attempted; the send stage runs with max attempts 1 or checks a persisted \"dispatching\" marker so a crash after send does not re-send. Required regression tests: failure after send → exactly one HTTP call and the job is failed/dead-lettered, not re-queued; failure before send → retried per the curve; simulated crash between send and ack → no second send. Effort: human ~1 day / CC ~15 min. R2, R4 and R5 stay pending.\nB) Move to at-least-once with an idempotency key\nWebhook retried on send failure under the library curve, with a stable per-job (not per-attempt) idempotency key in the request; receiver-facing contract change documented. Required tests: send failure → retried with an identical key on every attempt; key stable across process restart; exhausted attempts → dead-letter. Effort: human ~1 week including consumer outreach / CC ~30 min code. R2, R4 and R5 stay pending.\nC) Investigate receivers and the current job first\nBounded investigation only: read `processWebhookJob()` and audit receivers for dedupe behavior (human ~half day / CC ~5 min), then return to this question. Approves no implementation or tests; the guarantee, R2, R4 and R5 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T19:38:59.721Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "State: approved\nActual answer: A) Keep at-most-once; retry only before send (D2 answer)\nAccepted scope: `processWebhookJob()` keeps its at-most-once delivery guarantee. Library retry applies only to failures before the HTTP send is attempted; the send stage runs with max attempts 1 or checks a persisted \"dispatching\" marker so a crash after send does not re-send. Required regression tests (carried into Section 3 as approved proof): failure after send → exactly one HTTP call and the job is failed/dead-lettered, not re-queued; failure before send → retried per the curve; simulated crash between send and ack → no second send. R2, R4 and R5 stay pending.\nHistory: none\n\n### R5: Retry exhaustion policy (bounded attempts and dead-letter)\nFinding: Section 1 (Architecture) #2, P2, confidence 8/10, PLAN.md:6-9, reviewer: plan-eng-review (Claude)\nPlan baseline: original proposal names \"exponential backoff\" and \"full control over the curve\" but sets no attempt limit and no behavior for a job that keeps failing. D1 (approved) puts retries under the library hook, which exposes max attempts and dead-letter routing. Nothing approved for exhaustion.\nRuntime evidence: unknown; library defaults not inspectable here. Search check: unlimited retries and missing dead-letter handling are the most-cited retry pitfalls (retry storms, poison jobs hidden forever).\nComparison grid:\n\n| Choice | Current | A | B |\n|---|---|---|---|\n| R5 exhaustion policy | unspecified; plan silent | bounded attempts (plan names a default, implementer tunes per job type) then dead-letter with attempt history logged and a metric/alert on dead-letter growth | leave to the library's default; plan stays silent |\n| Max delay cap | implied only by attempt count | derived: base × 2^attempts, chosen so the last wait is acceptable; documented in plan | unspecified |\n| R1 mechanism (D1), R3 guarantee (D2) | approved | unchanged | unchanged |\n| R2 envelope sharing, R4 graph caching, R6 jitter | pending | pending | pending |\n\nQuestion D3:\nD3 — How many times may a job retry, and what happens when it runs out?\nProject/branch/task: main branch of the retry-framework plan; retries now run under the library hook (D1) with the webhook send excluded (D2).\nELI10: Exponential backoff only says how long to wait between tries, not how many tries. Without a limit, a job that can never succeed (bad payload, deleted record) keeps coming back forever and floods the queue; with a limit but no landing spot, it just disappears when the limit is hit. The plan says nothing about either.\nStakes if we pick wrong: a poison job retries indefinitely and starves real work, or failed jobs vanish silently and nobody notices customers' work never happened.\nRecommendation: A because bounded attempts plus a dead-letter queue with an alert is the standard, boring shape, and the library already provides it under D1.\nCompleteness: A=9/10, B=4/10\nPros / cons:\nA) Bounded attempts, dead-letter, alert (recommended)\n ✅ Poison jobs stop after a known number of tries and land somewhere an operator can inspect and replay; attempt history is logged with them\n ✅ A metric or alert on dead-letter growth turns silent failure into a page; one config line per job type (human: ~half day / CC: ~10 min)\n ❌ Each job type needs a sensible attempt count and someone has to own the dead-letter queue\nB) Leave it to library defaults, plan stays silent\n ✅ Zero extra work now; the library's default limit applies if it has one\n ✅ No new operational surface (no dead-letter queue to watch) in this change\n ❌ Nobody has verified the default; it may be unlimited or silent-drop, and the plan would ship a retry framework with no defined end state\nNet: a few config lines and an alert now versus discovering the exhaustion behavior in production.\nHeader: Retry exhaustion\nOptions:\nA) Bounded attempts, dead-letter, alert (recommended)\nPlan names a default max-attempt count per job type (implementer tunes), exhausted jobs go to the library's dead-letter route with their attempt history logged, and a metric/alert fires on dead-letter growth. Max delay cap is derived from base × 2^attempts and documented. Completeness 9/10. Effort: human ~half day / CC ~10 min. R2, R4 and R6 stay pending.\nB) Leave it to library defaults, plan stays silent\nKeep the plan as written: no attempt limit, no dead-letter behavior and no alert specified; whatever the library does by default applies. Completeness 4/10. Effort: none. R2, R4 and R6 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T19:39:22.395Z", + "old": "Approves no implementation or tests; the guarantee, R2, R4 and R5 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "Approves no implementation or tests; the guarantee, R2, R4 and R5 stay pending.\n\nState: approved\nActual answer: A) Keep at-most-once; retry only before send (D2 answer)\nAccepted scope: `processWebhookJob()` keeps its at-most-once delivery guarantee. Library retry applies only to failures before the HTTP send is attempted; the send stage runs with max attempts 1 or checks a persisted \"dispatching\" marker so a crash after send does not re-send. Required regression tests (carried into Section 3 as approved proof): failure after send → exactly one HTTP call and the job is failed/dead-lettered, not re-queued; failure before send → retried per the curve; simulated crash between send and ack → no second send. R2, R4 and R5 stay pending.\nHistory: none\n\n### R5: Retry exhaustion policy (bounded attempts and dead-letter)\nFinding: Section 1 (Architecture) #2, P2, confidence 8/10, PLAN.md:6-9, reviewer: plan-eng-review (Claude)\nPlan baseline: original proposal names \"exponential backoff\" and \"full control over the curve\" but sets no attempt limit and no behavior for a job that keeps failing. D1 (approved) puts retries under the library hook, which exposes max attempts and dead-letter routing. Nothing approved for exhaustion.\nRuntime evidence: unknown; library defaults not inspectable here. Search check: unlimited retries and missing dead-letter handling are the most-cited retry pitfalls (retry storms, poison jobs hidden forever).\nComparison grid:\n\n| Choice | Current | A | B |\n|---|---|---|---|\n| R5 exhaustion policy | unspecified; plan silent | bounded attempts (plan names a default, implementer tunes per job type) then dead-letter with attempt history logged and a metric/alert on dead-letter growth | leave to the library's default; plan stays silent |\n| Max delay cap | implied only by attempt count | derived: base × 2^attempts, chosen so the last wait is acceptable; documented in plan | unspecified |\n| R1 mechanism (D1), R3 guarantee (D2) | approved | unchanged | unchanged |\n| R2 envelope sharing, R4 graph caching, R6 jitter | pending | pending | pending |\n\nQuestion D3:\nD3 — How many times may a job retry, and what happens when it runs out?\nProject/branch/task: main branch of the retry-framework plan; retries now run under the library hook (D1) with the webhook send excluded (D2).\nELI10: Exponential backoff only says how long to wait between tries, not how many tries. Without a limit, a job that can never succeed (bad payload, deleted record) keeps coming back forever and floods the queue; with a limit but no landing spot, it just disappears when the limit is hit. The plan says nothing about either.\nStakes if we pick wrong: a poison job retries indefinitely and starves real work, or failed jobs vanish silently and nobody notices customers' work never happened.\nRecommendation: A because bounded attempts plus a dead-letter queue with an alert is the standard, boring shape, and the library already provides it under D1.\nCompleteness: A=9/10, B=4/10\nPros / cons:\nA) Bounded attempts, dead-letter, alert (recommended)\n ✅ Poison jobs stop after a known number of tries and land somewhere an operator can inspect and replay; attempt history is logged with them\n ✅ A metric or alert on dead-letter growth turns silent failure into a page; one config line per job type (human: ~half day / CC: ~10 min)\n ❌ Each job type needs a sensible attempt count and someone has to own the dead-letter queue\nB) Leave it to library defaults, plan stays silent\n ✅ Zero extra work now; the library's default limit applies if it has one\n ✅ No new operational surface (no dead-letter queue to watch) in this change\n ❌ Nobody has verified the default; it may be unlimited or silent-drop, and the plan would ship a retry framework with no defined end state\nNet: a few config lines and an alert now versus discovering the exhaustion behavior in production.\nHeader: Retry exhaustion\nOptions:\nA) Bounded attempts, dead-letter, alert (recommended)\nPlan names a default max-attempt count per job type (implementer tunes), exhausted jobs go to the library's dead-letter route with their attempt history logged, and a metric/alert fires on dead-letter growth. Max delay cap is derived from base × 2^attempts and documented. Completeness 9/10. Effort: human ~half day / CC ~10 min. R2, R4 and R6 stay pending.\nB) Leave it to library defaults, plan stays silent\nKeep the plan as written: no attempt limit, no dead-letter behavior and no alert specified; whatever the library does by default applies. Completeness 4/10. Effort: none. R2, R4 and R6 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + }, + { + "at": "2026-10-04T19:40:25.492Z", + "old": "Keep the plan as written: no attempt limit, no dead-letter behavior and no alert specified; whatever the library does by default applies. Completeness 4/10. Effort: none. R2, R4 and R6 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n", + "new": "Keep the plan as written: no attempt limit, no dead-letter behavior and no alert specified; whatever the library does by default applies. Completeness 4/10. Effort: none. R2, R4 and R6 stay pending.\n\nState: approved\nActual answer: A) Bounded attempts, dead-letter, alert (D3 answer)\nAccepted scope: Plan names a default max-attempt count per job type (implementer tunes); exhausted jobs go to the library's dead-letter route with attempt history logged; a metric/alert fires on dead-letter growth; max delay cap derived from base × 2^attempts and documented. R2, R4 and R6 stay pending.\nHistory: none\n\n**Section 1 (Architecture) dispositions:** #1 webhook guarantee accepted via D2; #2 exhaustion policy accepted via D3.\n\n### R2 (continued): Retry envelope sharing across the 5 workers\nPlan baseline (updated): the original 5 copy-pasted envelopes (PLAN.md:12-14). Under D1 the library hook owns delay computation and re-dispatch; what remains per worker is the backoff callback, per-job-type retry config (max attempts per D3, webhook send stage at 1 attempt per D2) and attempt logging.\nRuntime evidence: unknown; worker files not in this checkout. Rubric accounting (estimates, plan-stated callers): 5 inline envelopes at ~25-35 lines each = ~125-175 lines proposed; one shared retry-policy module ~40-60 lines plus ~3-5 config lines per worker = ~55-85 lines; implementation savings ~50-110 lines. Shared-module unit tests add ~60-100 lines, so the total diff may grow while duplicated logic drops to zero. Blast radius of a shared bug: all 5 job types retry on the wrong curve; mitigated by the callback being pure and unit-tested, and by the library owning re-queueing.\nComparison grid:\n\n| Choice | Current | A | B |\n|---|---|---|---|\n| R2 envelope sharing | 5 copy-pasted envelopes, refactor \"later\" (proposed) | one shared retry-policy module: the backoff callback, a per-job-type policy table (attempts, send-stage exclusion) and one attempt-logging hook registered at queue level; each worker declares its policy in a few lines | keep 5 inline copies of the remaining envelope (callback invocation, logging), refactor later |\n| R1 mechanism (D1), R3 guarantee (D2), R5 exhaustion (D3) | approved | unchanged | unchanged |\n| R4 graph caching, R6 jitter, R7 error classification | pending | pending | pending |\n\nQuestion D4:\nD4 — One shared retry-policy module, or five inline copies refactored \"later\"?\nProject/branch/task: main branch of the retry-framework plan; with D1 the library owns re-dispatch, so what is left per worker is the curve callback, the per-job policy and attempt logging.\nELI10: The plan copies the same retry wrapper into five worker files and promises to clean it up later. Later rarely comes, and five copies means five places to fix when the curve or the logging is wrong. With the library hook doing the heavy lifting, the shared part is small: one curve function, one table of per-job settings, one logging hook.\nStakes if we pick wrong: a curve bug or a logging gap gets fixed in four files and missed in the fifth, and the next job type copies whichever version its author found first.\nRecommendation: A because five plan-stated callers with identical behavior clear the shared-code bar, the helper is tiny, and extracting before the behavior change keeps the refactor and the feature separate.\nCompleteness: A=9/10, B=4/10\nPros / cons:\nA) One shared retry-policy module (recommended)\n ✅ Curve, per-job attempt limits and attempt logging live in one tested place; adding a sixth job type is a one-line policy entry\n ✅ Roughly 50-110 fewer implementation lines than five copies, and the refactor lands before the behavior change (human: ~1 day / CC: ~15 min)\n ❌ A bug in the shared callback hits all five job types at once, so the module needs its own unit tests\nB) Five inline copies, refactor later\n ✅ Each worker is self-contained and readable without jumping to a shared file\n ✅ No shared-failure blast radius; a mistake stays inside one job type\n ❌ Five copies of the same logic drift; the plan already calls this \"duplicated\" and defers the fix with no owner or trigger\nNet: a small shared module with tests now, or five copies and a refactor nobody has scheduled.\nHeader: Envelope sharing\nOptions:\nA) One shared retry-policy module (recommended)\nExtract one module holding the backoff callback, a per-job-type policy table (max attempts per D3, webhook send stage excluded per D2) and one attempt-logging hook registered at queue level. Each of the 5 workers declares its policy in a few lines; no copy-pasted envelope remains. Shared-module unit tests are required proof. Completeness 9/10. Effort: human ~1 day / CC ~15 min. R4, R6 and R7 stay pending.\nB) Five inline copies, refactor later\nKeep the plan as written: each worker carries its own callback invocation, policy constants and attempt logging; refactor deferred with no trigger. Completeness 4/10. Effort: human ~0 extra now / CC ~0, debt carried. R4, R6 and R7 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: Scope Challenge placeholder record (grid pending) replaced by this prepared record in Section 2.\n" + }, + { + "at": "2026-10-04T19:41:23.353Z", + "old": "State: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: Scope Challenge placeholder record (grid pending) replaced by this prepared record in Section 2.\n", + "new": "State: approved\nActual answer: A) One shared retry-policy module (D4 answer)\nAccepted scope: One shared module holding the backoff callback, a per-job-type policy table (max attempts per D3, webhook send stage excluded per D2) and one attempt-logging hook registered at queue level. Each of the 5 workers declares its policy in a few lines; no copy-pasted envelope remains. Shared-module unit tests are required proof (carried into Section 3). R4, R6 and R7 stay pending.\nHistory: Scope Challenge placeholder record (grid pending) replaced by this prepared record in Section 2.\n\n### R7: Retryable vs permanent error classification\nFinding: Section 2 (Code quality) #2, P2, confidence 7/10, PLAN.md:6-9 and 12-14, reviewer: plan-eng-review (Claude)\nPlan baseline: original proposal retries on any failure; the envelope is \"compute delay, log attempt, dispatch\" with no mention of which errors are retryable. D3 (approved) bounds attempts and dead-letters on exhaustion. No classification approved.\nRuntime evidence: unknown; worker error types not in this checkout. Pattern: validation errors, missing records and 4xx-class responses never succeed on retry; retrying them burns the full attempt budget before dead-lettering.\nComparison grid:\n\n| Choice | Current | A | B |\n|---|---|---|---|\n| R7 error classification | retry every failure until exhaustion (implicit) | shared policy module distinguishes permanent errors (validation, not-found, non-retryable 4xx) from transient ones; permanent errors go straight to dead-letter with the reason, transient ones follow the curve | retry every failure until the D3 attempt bound, then dead-letter |\n| R1, R2, R3, R5 (approved D1-D4) | approved | unchanged | unchanged |\n| R4 graph caching, R6 jitter | pending | pending | pending |\n\nQuestion D5:\nD5 — Should permanent errors skip the retry curve and dead-letter immediately?\nProject/branch/task: main branch of the retry-framework plan; retries live in the shared policy module (D4) under the library hook (D1) with bounded attempts (D3).\nELI10: Some failures fix themselves if you wait, like a database hiccup. Others never will, like a payload that fails validation or a record that was deleted. Retrying the second kind just wastes every attempt and delays the moment someone notices. The plan treats all failures the same.\nStakes if we pick wrong: permanent failures sit in the queue for the whole backoff window before anyone sees them, and the queue spends worker time on jobs that cannot succeed; or, if classification is wrong, a transient error is mislabeled permanent and a recoverable job is dead-lettered early.\nRecommendation: A because the shared module already exists under D4, the classification is a small predicate, and it turns a slow silent failure into an immediate visible one.\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Classify: permanent errors dead-letter immediately (recommended)\n ✅ Bad payloads and missing records surface in the dead-letter queue on the first attempt with the reason attached, instead of after the full curve\n ✅ Worker time and the attempt budget go to failures that can actually recover; the predicate is one function in the shared module with its own tests (human: ~half day / CC: ~10 min)\n ❌ A misclassified transient error gets dead-lettered early; the predicate must default to retry when unsure\nB) Retry everything until the attempt bound\n ✅ Simplest rule, no error taxonomy to maintain across five job types\n ✅ No risk of mislabeling a recoverable error as permanent\n ❌ Permanent failures consume every attempt and the full backoff window before dead-lettering; a bad deploy fills the queue with hopeless retries\nNet: a small, retry-by-default predicate in the shared module versus burning the full curve on errors that can never succeed.\nHeader: Error classes\nOptions:\nA) Classify: permanent errors dead-letter immediately (recommended)\nThe shared policy module gains a retryable predicate: validation errors, not-found and non-retryable 4xx-class responses go straight to dead-letter with the reason; everything else follows the curve. Default is retry when unsure. Unit tests cover both branches (required proof). Completeness 9/10. Effort: human ~half day / CC ~10 min. R4 and R6 stay pending.\nB) Retry everything until the attempt bound\nEvery failure follows the D3 curve until attempts are exhausted, then dead-letters. No error taxonomy. Completeness 6/10. Effort: none. R4 and R6 stay pending.\n\nState: pending\nActual answer: unanswered\nAccepted scope: none\nHistory: none\n" + } + ] + } +} diff --git a/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/inventory.json b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/inventory.json new file mode 100644 index 000000000..f32fd968a --- /dev/null +++ b/test/fixtures/detector-corpora/plan-eng-multi-finding-batching/inventory.json @@ -0,0 +1,92 @@ +[ + { + "run": "37162480720", + "lane": "periodic", + "outcome": "passed", + "series_identity": "5553dbf327080983", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37162480720-t1-pass.json" + }, + { + "run": "37174266054", + "lane": "periodic", + "outcome": "passed", + "series_identity": "57a825b033078a7b", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37174266054-t1-pass.json" + }, + { + "run": "37176837432", + "lane": "periodic", + "outcome": "passed", + "series_identity": "57a825b033078a7b", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37176837432-t1-pass.json" + }, + { + "run": "37179171083", + "lane": "periodic", + "outcome": "passed", + "series_identity": "57a825b033078a7b", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37179171083-t1-pass.json" + }, + { + "run": "37182865432", + "lane": "periodic", + "outcome": "passed", + "series_identity": "c620ebc29ce470c6", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37182865432-t1-pass.json" + }, + { + "run": "37186854666", + "lane": "periodic", + "outcome": "passed", + "series_identity": "a3e4c3429b2cfef8", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37186854666-t1-pass.json" + }, + { + "run": "37193478719", + "lane": "periodic", + "outcome": "passed", + "series_identity": "a3e4c3429b2cfef8", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37193478719-t1-pass.json" + }, + { + "run": "37195203538", + "lane": "periodic", + "outcome": "passed", + "series_identity": "ded5316419aefeaf", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37195203538-t1-pass.json" + }, + { + "run": "37198445662", + "lane": "periodic", + "outcome": "passed", + "series_identity": "ded5316419aefeaf", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37198445662-t1-pass.json" + }, + { + "run": "37228573062", + "lane": "periodic", + "outcome": "failed", + "series_identity": "4a37ebd37e42d08d", + "artifact": "paid-slice-1-a1", + "available": true, + "entry": "37228573062-t1-fail.json" + } +] diff --git a/test/helpers/eng-seeded-coverage.ts b/test/helpers/eng-seeded-coverage.ts index 528e5138f..058faa2a8 100644 --- a/test/helpers/eng-seeded-coverage.ts +++ b/test/helpers/eng-seeded-coverage.ts @@ -174,7 +174,11 @@ function recordedBatchingIssue(call: NativePlanQuestionCall, savedPlan: string): const previous = tokens.slice(0, at).filter(t => t.type !== 'space').at(-1); const quotedContext = /\b(?:quoted|copied|historical|example|hypothetical|archived)\b[^\n]*:\s*$/i; if (previous?.type === 'paragraph' && quotedContext.test(previous.raw)) return []; - const parts = token.raw.split('\n').map(line => line.replace(/^\s*(?:[-*+]|\d+[.)])\s+/, '').replace(/[*_]/g, '').trim()); + // A declaration may open a later sentence of its line ("Report destination + // requested by the user. Review target (fixed): `PLAN.md` in ..."); each + // sentence start is a field position, a mid-sentence mention is not. + const parts = token.raw.split('\n').flatMap(line => line.replace(/^\s*(?:[-*+]|\d+[.)])\s+/, '').replace(/[*_]/g, '').trim() + .split(/(?<=[.!?])\s+(?=[A-Z])/)); return parts.filter((line, i) => TARGET_FIELD.test(line) && !parts.slice(0, i).some(part => quotedContext.test(part))); }); diff --git a/test/test-of-test-ratchet.test.ts b/test/test-of-test-ratchet.test.ts index d2fa4f3e0..857536153 100644 --- a/test/test-of-test-ratchet.test.ts +++ b/test/test-of-test-ratchet.test.ts @@ -66,6 +66,7 @@ const BASELINE = [ 'test/design-html-section-completion.test.ts', 'test/detector-corpus-auto-decide-preserved.test.ts', // B1 replay corpus: census captures through the paid case's detectors (approved plan item) 'test/detector-corpus-plan-ceo-section-loading.test.ts', // B1 replay corpus: census captures through the paid case's detectors (approved plan item) + 'test/detector-corpus-plan-eng-multi-finding-batching.test.ts', // B1 replay corpus (GSTA-23 red 37228573062): census captures through the paid case's review-question counter 'test/detector-corpus-shared-libs-plan-callers.test.ts', // B1 replay corpus: census captures through the paid case's actor (approved plan item) 'test/detector-corpus-ship-docsync-late-result.test.ts', // B1 replay corpus: census captures through the paid case's detectors (approved plan item) 'test/devex-peer-comparison-calibration.test.ts',