mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 09:29:49 +02:00
test(eng-batching): bind unsourced native briefs through the report's target
Run 36606688266 asked ten separate native review questions (D1-D9 bound to ledger records R1-R9) and failed reviewCount=0 < FLOOR=3: its briefs named the plan by title instead of citing PLAN.md, its report declared 'Review target (fixed): PLAN.md' under '# Engineering review: <plan>', and it kept an unfenced copy of the plan's own H1. The named-source route now accepts those spellings and non-inline ledger briefs. The same replay rejects a foreign, mixed, duplicate or missing target, another plan's title or copied H1, a brief naming another plan or file, a mismatched saved brief, and re-asks. The run-36597762183 capture still counts 3.
This commit is contained in:
garrytan committed
1 parent
5269452983
commit
8bd53faa6c
3 files changed
+409
-11
No files matched your search
@@ -1,7 +1,11 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import type { NativePlanQuestionCall } from './helpers/plan-count-transcript';
|
||||
import { isEngBatchingIssueAUQ } from './helpers/eng-seeded-coverage';
|
||||
import { nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { createEngBatchingIssueCounter, isEngBatchingIssueAUQ } from './helpers/eng-seeded-coverage';
|
||||
import { engSetupAUQ, hasCompletePlanReport, nativePlanCallFingerprint } from './helpers/claude-pty-runner';
|
||||
import batchingCapture from './fixtures/eng-batching-unsourced-brief-36606688266.json';
|
||||
|
||||
function question(call: NativePlanQuestionCall, text: string) {
|
||||
const answer = call.answers![call.questions[0]!.question]!;
|
||||
@@ -78,3 +82,81 @@ describe('batching caller counts completed issue decisions across setup boundari
|
||||
expect(check(quoted)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('batching replay of run 36606688266 (unsourced native briefs)', () => {
|
||||
// Run 36606688266 asked one native question per finding (D1-D9 bound to
|
||||
// ledger records R1-R9, D10 a TODO follow-up) but cited no PLAN.md line in the
|
||||
// native brief, so the old detector counted zero review decisions.
|
||||
const FLOOR = 3;
|
||||
const calls = batchingCapture.calls as unknown as NativePlanQuestionCall[];
|
||||
|
||||
function count(plan: string, edit: (calls: NativePlanQuestionCall[]) => void = () => {}) {
|
||||
const copy = structuredClone(calls);
|
||||
edit(copy);
|
||||
const counter = createEngBatchingIssueCounter(() => plan, engSetupAUQ);
|
||||
const counted = copy.filter((call, index) => counter.isReviewAUQ(nativePlanCallFingerprint(call, 0, true), copy.slice(0, index)));
|
||||
return { counted: counted.length, issues: counter.trace.map(entry => entry.issue) };
|
||||
}
|
||||
|
||||
test('the recorded failing verdict is the detector, not the review', () => {
|
||||
expect(batchingCapture.recordedOutcome).toEqual({ outcome: 'completion_summary', step0Count: 10, reviewCount: 0 });
|
||||
expect(calls.every(call => call.answered && call.questions.length === 1)).toBe(true);
|
||||
});
|
||||
|
||||
test('each ledger-bound native decision counts once without a native source citation', () => {
|
||||
const { counted, issues } = count(batchingCapture.plan);
|
||||
expect(issues).toEqual(['R1', 'R2', 'R3', 'R4', 'R5', 'R6', 'R7', 'R8', 'R9'].map(id => `record:${id}`));
|
||||
expect(counted).toBeGreaterThanOrEqual(FLOOR);
|
||||
});
|
||||
|
||||
test('a re-asked decision cannot inflate the count', () => {
|
||||
const { counted } = count(batchingCapture.plan, all => {
|
||||
const again = structuredClone(all[0]!);
|
||||
again.toolUseId += '-again';
|
||||
all.splice(1, 0, again);
|
||||
});
|
||||
expect(counted).toBe(9);
|
||||
});
|
||||
|
||||
const target = 'Review target (fixed): `PLAN.md`';
|
||||
for (const [name, plan] of [
|
||||
['a foreign target', batchingCapture.plan.replace(target, 'Review target (fixed): `OTHER.md`')],
|
||||
['a mixed target', batchingCapture.plan.replace(target, 'Review target (fixed): `OTHER.md` and `PLAN.md`')],
|
||||
['two target declarations', batchingCapture.plan.replace(target, `${target}\nReview target (fixed): \`PLAN.md\``)],
|
||||
['no target declaration', batchingCapture.plan.replace(target, 'Report scope: the fixture repo')],
|
||||
['a report title for another plan', batchingCapture.plan.replace('# Engineering review: Add background job retry framework', '# Engineering review: Replace all customer data')],
|
||||
['an archived report title', batchingCapture.plan.replace('# Engineering review:', '# Archived engineering review:')],
|
||||
['a copied H1 naming another plan', batchingCapture.plan.replace('# Plan: Add background job retry framework', '# Plan: Replace all customer data')],
|
||||
] as const) test(`the unsourced route rejects ${name}`, () => {
|
||||
expect(count(plan).counted).toBe(0);
|
||||
});
|
||||
|
||||
test('the unsourced route rejects a native brief naming another plan or file', () => {
|
||||
const rename = (from: string, to: string) => (all: NativePlanQuestionCall[]) => {
|
||||
for (const call of all) call.questions[0]!.question = call.questions[0]!.question.replace(from, to);
|
||||
};
|
||||
expect(count(batchingCapture.plan, rename('plan "Add background job retry framework"', 'plan "Replace all customer data"')).counted).toBe(0);
|
||||
expect(count(batchingCapture.plan, rename('plan "Add background job retry framework"', 'plan "Add background job retry framework", OTHER.md')).counted).toBe(0);
|
||||
expect(count(batchingCapture.plan, rename('plan "Add background job retry framework"', 'the plan')).counted).toBe(0);
|
||||
});
|
||||
|
||||
test('a saved record whose brief title differs from the native question does not bind it', () => {
|
||||
const plan = batchingCapture.plan.replace(/^Question D1:\n.*$/m, 'Question D1:\nD1 — Some other decision?');
|
||||
expect(count(plan).issues).not.toContain('record:R1');
|
||||
});
|
||||
|
||||
test('the completed report is the early outcome point; a partial report is not', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eng-batching-report-'));
|
||||
try {
|
||||
const report = path.join(dir, 'report.md');
|
||||
fs.writeFileSync(report, batchingCapture.plan);
|
||||
expect(hasCompletePlanReport(report, 0, Date.now() + 1_000)).toBe(true);
|
||||
fs.writeFileSync(report, batchingCapture.plan.slice(0, batchingCapture.plan.indexOf('## Completion summary')));
|
||||
expect(hasCompletePlanReport(report, 0, Date.now() + 1_000)).toBe(false);
|
||||
fs.writeFileSync(report, batchingCapture.plan.replace('## GSTACK REVIEW REPORT', '```\n## GSTACK REVIEW REPORT') + '\n```\n');
|
||||
expect(hasCompletePlanReport(report, 0, Date.now() + 1_000)).toBe(false);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,311 @@
|
||||
{
|
||||
"source": "run 36606688266 plan-eng-multi-finding-batching observation.json (transcript.calls) and the saved report replayed from its Write/Edit inputs",
|
||||
"recordedOutcome": {
|
||||
"outcome": "completion_summary",
|
||||
"step0Count": 10,
|
||||
"reviewCount": 0
|
||||
},
|
||||
"calls": [
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_011vfjpt7sgeSRMw1zhHATN8",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D1 — Use the job library's retry hooks or roll a custom inline scheduler?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: The job library you already run has a retry system built in. The plan wants to rebuild the same thing by hand inside each of the 5 workers, only so the delay curve can be tuned. Retry systems are easy to get subtly wrong: losing the attempt count when a worker restarts, retrying forever, or retrying twice at once. The library has already paid for those bugs; a hand-rolled copy pays for them again in production.\nStakes if we pick wrong: a hand-rolled scheduler that drops attempt state on restart or double-schedules turns one failed job into duplicate side effects or an infinite retry loop, with no dead-letter to catch it.\nRecommendation: A because the plan admits the shapes are identical, and the curve is pluggable in the library through a strategy function, so A delivers the same control with far less new code. (human: ~1 day / CC: ~20 min for A; human: ~1 week / CC: ~2 h for B plus ongoing ownership)\nCompleteness: A=9/10, B=5/10\nPros / cons:\nA) Library hooks + custom curve (recommended)\n ✅ Attempt counting, persistence across restarts and terminal handling come from tested library code, not new code\n ✅ The custom curve still lives in one strategy function, so \"full control over the curve\" is preserved\n ❌ Requires confirming the library's hook accepts a custom delay function; if it does not, we fall back to B for the curve only\nB) Custom inline scheduler\n ✅ Zero dependency on the library's retry semantics or its upgrade cadence\n ✅ Any curve shape, any bookkeeping, no hook constraints\n ❌ Rebuilds attempt state, restart persistence, concurrency guards and dead-lettering by hand, and those are the parts that fail at 3am\nNet: you are trading a one-line strategy function against owning a second retry engine forever.",
|
||||
"header": "Retry engine",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Library hooks + custom curve (recommended)",
|
||||
"description": "Register the exponential-backoff curve as one custom backoff strategy function with the job library's built-in retry hooks. Attempt counting, persistence across worker restarts, and terminal/dead-letter handling come from the library. Verify the hook accepts a delay function first; if it does not, fall back to a custom curve only for delay computation while keeping library scheduling. R2-R5 stay pending. Completeness 9/10. human: ~1 day / CC: ~20 min."
|
||||
},
|
||||
{
|
||||
"label": "Custom inline scheduler",
|
||||
"description": "Keep the plan as written: a custom exponential-backoff scheduler inline in each worker, bypassing the library's retry hooks. Attempt state, restart persistence, concurrency guards and terminal handling must be designed and tested by hand (tracked under R3). R2-R5 stay pending. Completeness 5/10. human: ~1 week / CC: ~2 h plus ongoing ownership."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D1 — Use the job library's retry hooks or roll a custom inline scheduler?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: The job library you already run has a retry system built in. The plan wants to rebuild the same thing by hand inside each of the 5 workers, only so the delay curve can be tuned. Retry systems are easy to get subtly wrong: losing the attempt count when a worker restarts, retrying forever, or retrying twice at once. The library has already paid for those bugs; a hand-rolled copy pays for them again in production.\nStakes if we pick wrong: a hand-rolled scheduler that drops attempt state on restart or double-schedules turns one failed job into duplicate side effects or an infinite retry loop, with no dead-letter to catch it.\nRecommendation: A because the plan admits the shapes are identical, and the curve is pluggable in the library through a strategy function, so A delivers the same control with far less new code. (human: ~1 day / CC: ~20 min for A; human: ~1 week / CC: ~2 h for B plus ongoing ownership)\nCompleteness: A=9/10, B=5/10\nPros / cons:\nA) Library hooks + custom curve (recommended)\n ✅ Attempt counting, persistence across restarts and terminal handling come from tested library code, not new code\n ✅ The custom curve still lives in one strategy function, so \"full control over the curve\" is preserved\n ❌ Requires confirming the library's hook accepts a custom delay function; if it does not, we fall back to B for the curve only\nB) Custom inline scheduler\n ✅ Zero dependency on the library's retry semantics or its upgrade cadence\n ✅ Any curve shape, any bookkeeping, no hook constraints\n ❌ Rebuilds attempt state, restart persistence, concurrency guards and dead-lettering by hand, and those are the parts that fail at 3am\nNet: you are trading a one-line strategy function against owning a second retry engine forever.": "Library hooks + custom curve (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:46:48.252Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_0182pLbYfj4VpsRE57iBzMYQ",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D2 — What delivery guarantee does `processWebhookJob()` keep once it retries?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: Today a webhook is sent once; if it fails, it is gone (at-most-once). The moment you retry, a request that timed out after the customer already received it gets sent again, so the customer sees the same event twice. You have to pick: either only retry when you are sure the request never left, or retry freely but stamp every attempt with the same id so the customer can ignore repeats. The plan does neither and just retries.\nStakes if we pick wrong: customers process duplicate events (double orders, double emails) with no way to detect them, or you ship a retry feature that almost never fires because most webhook failures are timeouts.\nRecommendation: B because it is the standard webhook contract (retry on timeout/5xx, stable event id per attempt) and is the only option where retrying actually improves delivery while giving receivers a way to dedupe. This is a receiver-visible contract change; A is the right pick if you cannot communicate it to receivers.\nCompleteness: A=7/10, B=9/10, C=3/10\nPros / cons:\nA) Keep at-most-once\n ✅ No change to what receivers see; the existing guarantee and its regression test stay valid as-is\n ✅ Smallest blast radius: no new headers, no receiver communication needed\n ❌ Retries only fire on connect/DNS/pre-send errors; timeouts and 5xx go straight to terminal, so most real failures are still not retried\nB) At-least-once + idempotency key (recommended)\n ✅ Timeouts and 5xx are retried, so delivery reliability actually improves for receivers\n ✅ Same delivery id on every attempt lets receivers dedupe; this is the contract Stripe/GitHub-style webhooks use\n ❌ Receiver-visible contract change: duplicates become possible and receivers must be told to dedupe on the id\nC) Plain retry (plan as written)\n ✅ Least code: no header, no classification, just retry on any failure\n ✅ Ships fastest\n ❌ Duplicates reach receivers with no way to tell them apart; silent double side effects\nNet: A keeps the promise but retries little; B changes the promise but makes retries worth having; C breaks the promise silently.",
|
||||
"header": "Webhook semantics",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Keep at-most-once",
|
||||
"description": "Preserve at-most-once for `processWebhookJob()`. Retry only failures where the request provably never left the process (connection refused, DNS failure, errors before send). Timeouts, 5xx and any post-send ambiguity are terminal and go to whatever terminal handling R3 decides. No new headers; receiver contract unchanged. R3, R4, R5, R7 stay pending. Completeness 7/10. human: ~half day / CC: ~10 min."
|
||||
},
|
||||
{
|
||||
"label": "At-least-once + idempotency key (recommended)",
|
||||
"description": "Retry timeouts and 5xx as well as pre-send failures. Every attempt carries the same stable delivery id (e.g. an `Idempotency-Key`/event-id header derived from the job id, generated once, persisted with the job). 4xx other than 408/429 is non-retryable for webhooks. Document to receivers that duplicates are possible and must be deduped on the id. R3, R4, R5, R7 stay pending. Completeness 9/10. human: ~1 day / CC: ~20 min."
|
||||
},
|
||||
{
|
||||
"label": "Plain retry (plan as written)",
|
||||
"description": "Retry `processWebhookJob()` on any failure with no delivery id and no classification. Duplicates reach receivers undetectably. R3, R4, R5, R7 stay pending. Completeness 3/10. human: ~1 h / CC: ~5 min."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D2 — What delivery guarantee does `processWebhookJob()` keep once it retries?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: Today a webhook is sent once; if it fails, it is gone (at-most-once). The moment you retry, a request that timed out after the customer already received it gets sent again, so the customer sees the same event twice. You have to pick: either only retry when you are sure the request never left, or retry freely but stamp every attempt with the same id so the customer can ignore repeats. The plan does neither and just retries.\nStakes if we pick wrong: customers process duplicate events (double orders, double emails) with no way to detect them, or you ship a retry feature that almost never fires because most webhook failures are timeouts.\nRecommendation: B because it is the standard webhook contract (retry on timeout/5xx, stable event id per attempt) and is the only option where retrying actually improves delivery while giving receivers a way to dedupe. This is a receiver-visible contract change; A is the right pick if you cannot communicate it to receivers.\nCompleteness: A=7/10, B=9/10, C=3/10\nPros / cons:\nA) Keep at-most-once\n ✅ No change to what receivers see; the existing guarantee and its regression test stay valid as-is\n ✅ Smallest blast radius: no new headers, no receiver communication needed\n ❌ Retries only fire on connect/DNS/pre-send errors; timeouts and 5xx go straight to terminal, so most real failures are still not retried\nB) At-least-once + idempotency key (recommended)\n ✅ Timeouts and 5xx are retried, so delivery reliability actually improves for receivers\n ✅ Same delivery id on every attempt lets receivers dedupe; this is the contract Stripe/GitHub-style webhooks use\n ❌ Receiver-visible contract change: duplicates become possible and receivers must be told to dedupe on the id\nC) Plain retry (plan as written)\n ✅ Least code: no header, no classification, just retry on any failure\n ✅ Ships fastest\n ❌ Duplicates reach receivers with no way to tell them apart; silent double side effects\nNet: A keeps the promise but retries little; B changes the promise but makes retries worth having; C breaks the promise silently.": "Keep at-most-once"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:48:33.604Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_012ueyTcioa6A2y9YE4gYPcR",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D3 — How many times may a job retry, and where does it go when it gives up?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: A retry curve without a stopping point is a job that runs forever when the thing it depends on is down for good. You need a maximum number of tries, and you need a place for jobs that used up their tries (a dead-letter set) so someone can look at them and replay them later. Otherwise failed work quietly disappears or quietly never stops.\nStakes if we pick wrong: either a poisoned job hammers a downstream forever and starves healthy jobs, or failed webhooks and jobs vanish with only a log line nobody reads.\nRecommendation: A because the library already provides the failed set, so the dead-letter and alert cost a config line and one log call, and it is the only option where an operator can find and replay a lost job.\nCompleteness: A=9/10, B=6/10, C=2/10\nPros / cons:\nA) Bounded + dead-letter + alert (recommended)\n ✅ Exhausted jobs are inspectable and replayable from the library's failed set, with the last error attached\n ✅ One structured log line plus a metric on dead-letter entry makes a downstream outage visible within minutes\n ❌ Needs a per-worker ceiling value and a dead-letter retention/cleanup policy to be chosen and documented\nB) Bounded + log-and-drop\n ✅ Bounds the retry loop with the least configuration\n ✅ No dead-letter retention to manage\n ❌ A dropped job is gone; the only trace is a log line, so replay after an outage is impossible\nC) Unbounded (plan as written)\n ✅ No ceiling to tune; a job eventually succeeds if the dependency ever recovers\n ✅ Zero extra code\n ❌ Permanently failing jobs retry forever, consume worker capacity and never surface as a problem\nNet: you are choosing whether a job that cannot succeed becomes a visible artifact, a log line, or a permanent background load.",
|
||||
"header": "Attempt ceiling",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Bounded + dead-letter + alert (recommended)",
|
||||
"description": "Set a maximum attempt count per worker (default 5, overridable per worker, configured in the same place as the backoff strategy). On exhaustion or on a non-retryable error, the job lands in the library's dead-letter/failed set with its last error; emit one structured error log and a metric on entry. Webhook timeouts/5xx (terminal per R2) land here too. Document the retention/replay procedure. R4, R5 stay pending. Completeness 9/10. human: ~half day / CC: ~15 min."
|
||||
},
|
||||
{
|
||||
"label": "Bounded + log-and-drop",
|
||||
"description": "Set the same per-worker maximum attempt count (default 5). On exhaustion, log the error and drop the job; no dead-letter set, no metric, no replay. R4, R5 stay pending. Completeness 6/10. human: ~1 h / CC: ~5 min."
|
||||
},
|
||||
{
|
||||
"label": "Unbounded (plan as written)",
|
||||
"description": "No attempt ceiling; the exponential curve continues until the job succeeds. No terminal handling. Conflicts with R2, which needs a terminal destination for webhook timeouts. R4, R5 stay pending. Completeness 2/10. human: 0 / CC: 0."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D3 — How many times may a job retry, and where does it go when it gives up?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: A retry curve without a stopping point is a job that runs forever when the thing it depends on is down for good. You need a maximum number of tries, and you need a place for jobs that used up their tries (a dead-letter set) so someone can look at them and replay them later. Otherwise failed work quietly disappears or quietly never stops.\nStakes if we pick wrong: either a poisoned job hammers a downstream forever and starves healthy jobs, or failed webhooks and jobs vanish with only a log line nobody reads.\nRecommendation: A because the library already provides the failed set, so the dead-letter and alert cost a config line and one log call, and it is the only option where an operator can find and replay a lost job.\nCompleteness: A=9/10, B=6/10, C=2/10\nPros / cons:\nA) Bounded + dead-letter + alert (recommended)\n ✅ Exhausted jobs are inspectable and replayable from the library's failed set, with the last error attached\n ✅ One structured log line plus a metric on dead-letter entry makes a downstream outage visible within minutes\n ❌ Needs a per-worker ceiling value and a dead-letter retention/cleanup policy to be chosen and documented\nB) Bounded + log-and-drop\n ✅ Bounds the retry loop with the least configuration\n ✅ No dead-letter retention to manage\n ❌ A dropped job is gone; the only trace is a log line, so replay after an outage is impossible\nC) Unbounded (plan as written)\n ✅ No ceiling to tune; a job eventually succeeds if the dependency ever recovers\n ✅ Zero extra code\n ❌ Permanently failing jobs retry forever, consume worker capacity and never surface as a problem\nNet: you are choosing whether a job that cannot succeed becomes a visible artifact, a log line, or a permanent background load.": "Bounded + dead-letter + alert (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:50:10.303Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_016fgg5mWHBjFGWxQPRz9Whr",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D4 — Add jitter to the backoff curve, or keep it deterministic?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: When a downstream service goes down, hundreds of jobs across all 5 workers fail at the same moment. With a pure exponential curve they all wake up at exactly the same moment too, and hit the recovering service as one wave, which can knock it over again. Jitter adds a random spread to each delay so the retries trickle back instead of stampeding.\nStakes if we pick wrong: a downstream that recovers from an outage gets re-flattened by your own synchronized retry wave, turning a 2-minute blip into a 20-minute incident.\nRecommendation: A because it is two lines inside the strategy function you already own and it is the standard mitigation for retry storms; deterministic curves are only useful in tests, which can seed or stub the random source.\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Equal jitter (recommended)\n ✅ Retries after a shared outage spread across the window instead of returning as one synchronized burst\n ✅ Lives inside the single strategy function from D1, so every worker gets it with no per-worker code\n ❌ Curve tests need an injectable random source to stay deterministic\nB) No jitter (pure curve)\n ✅ Exact, predictable retry times that are easy to reason about and assert in tests\n ✅ Zero extra code beyond the curve itself\n ❌ All jobs that fail together retry together, so the retry framework itself becomes a traffic amplifier during outages\nNet: predictability in tests against stampede protection in production; the test cost is one injected random source.",
|
||||
"header": "Jitter",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Equal jitter (recommended)",
|
||||
"description": "Inside the single backoff strategy function, compute the exponential delay and return half of it plus a random amount up to the other half (equal jitter). The random source is injectable so tests can pin it. Required proof: unit test that returned delays fall within [curve/2, curve] for each attempt, and that a pinned random source gives a deterministic value. R5 stays pending. Completeness 9/10. human: ~1 h / CC: ~5 min."
|
||||
},
|
||||
{
|
||||
"label": "No jitter (pure curve)",
|
||||
"description": "Return the exact exponential curve value with no random spread. Required proof: unit test of the exact value per attempt. R5 stays pending. Completeness 6/10. human: ~30 min / CC: ~3 min."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D4 — Add jitter to the backoff curve, or keep it deterministic?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: When a downstream service goes down, hundreds of jobs across all 5 workers fail at the same moment. With a pure exponential curve they all wake up at exactly the same moment too, and hit the recovering service as one wave, which can knock it over again. Jitter adds a random spread to each delay so the retries trickle back instead of stampeding.\nStakes if we pick wrong: a downstream that recovers from an outage gets re-flattened by your own synchronized retry wave, turning a 2-minute blip into a 20-minute incident.\nRecommendation: A because it is two lines inside the strategy function you already own and it is the standard mitigation for retry storms; deterministic curves are only useful in tests, which can seed or stub the random source.\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Equal jitter (recommended)\n ✅ Retries after a shared outage spread across the window instead of returning as one synchronized burst\n ✅ Lives inside the single strategy function from D1, so every worker gets it with no per-worker code\n ❌ Curve tests need an injectable random source to stay deterministic\nB) No jitter (pure curve)\n ✅ Exact, predictable retry times that are easy to reason about and assert in tests\n ✅ Zero extra code beyond the curve itself\n ❌ All jobs that fail together retry together, so the retry framework itself becomes a traffic amplifier during outages\nNet: predictability in tests against stampede protection in production; the test cost is one injected random source.": "Equal jitter (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:51:07.968Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_01AGgMxL2cDbt1vqtMqNz8dH",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D5 — Should workers name errors that must not be retried, or retry every failure to the ceiling?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: Some failures fix themselves if you wait (a database hiccup, a slow API). Others never will (a payload that fails validation, a revoked API key). Retrying the second kind five times with growing delays just wastes capacity and delays the moment someone notices. Letting each worker say \"these error types are permanent\" sends them straight to the dead-letter set on the first try.\nStakes if we pick wrong: a bad payload burns 5 attempts and up to the full backoff window before it surfaces, and during a bad deploy every job does this at once.\nRecommendation: A because it is a small per-worker list, the dead-letter path already exists from D3, and it turns a permanent failure into an immediate signal instead of a delayed one. Medium confidence on which types are permanent; verify against the actual error classes when implementing.\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Explicit non-retryable list (recommended)\n ✅ Permanent failures reach the dead-letter set on attempt 1, so operators see bad payloads or revoked credentials within seconds\n ✅ Unknown errors still default to retry, so nothing transient is accidentally dropped\n ❌ Each worker needs a short, reviewed list of permanent error types, and a wrong entry makes a transient error permanent\nB) Retry everything to ceiling\n ✅ No classification to get wrong; behavior is identical for every worker\n ✅ Nothing to maintain when new error types appear\n ❌ Permanent failures consume the full attempt budget and backoff window before anyone can see them\nNet: a short reviewed list per worker against a guaranteed delay on every permanent failure.",
|
||||
"header": "Error classes",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Explicit non-retryable list (recommended)",
|
||||
"description": "Each of the 4 non-webhook workers declares its non-retryable error types (validation errors, auth/permission errors, malformed payload). Those bypass retry and land in the dead-letter set (R3) on attempt 1 with the error attached. Any error not on the list retries per R3/R4. Required proof: per worker, one test that a listed error goes to dead-letter without a retry, and one test that an unlisted error retries. Completeness 9/10. human: ~half day / CC: ~15 min."
|
||||
},
|
||||
{
|
||||
"label": "Retry everything to ceiling",
|
||||
"description": "No classification. Every failure in the 4 non-webhook workers retries per R3/R4 until the ceiling, then lands in dead-letter. Required proof: covered by R3 tests. Completeness 6/10. human: 0 / CC: 0."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D5 — Should workers name errors that must not be retried, or retry every failure to the ceiling?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: Some failures fix themselves if you wait (a database hiccup, a slow API). Others never will (a payload that fails validation, a revoked API key). Retrying the second kind five times with growing delays just wastes capacity and delays the moment someone notices. Letting each worker say \"these error types are permanent\" sends them straight to the dead-letter set on the first try.\nStakes if we pick wrong: a bad payload burns 5 attempts and up to the full backoff window before it surfaces, and during a bad deploy every job does this at once.\nRecommendation: A because it is a small per-worker list, the dead-letter path already exists from D3, and it turns a permanent failure into an immediate signal instead of a delayed one. Medium confidence on which types are permanent; verify against the actual error classes when implementing.\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Explicit non-retryable list (recommended)\n ✅ Permanent failures reach the dead-letter set on attempt 1, so operators see bad payloads or revoked credentials within seconds\n ✅ Unknown errors still default to retry, so nothing transient is accidentally dropped\n ❌ Each worker needs a short, reviewed list of permanent error types, and a wrong entry makes a transient error permanent\nB) Retry everything to ceiling\n ✅ No classification to get wrong; behavior is identical for every worker\n ✅ Nothing to maintain when new error types appear\n ❌ Permanent failures consume the full attempt budget and backoff window before anyone can see them\nNet: a short reviewed list per worker against a guaranteed delay on every permanent failure.": "Explicit non-retryable list (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:52:12.006Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_016w6Sj4WK8TpN3bzensgKcm",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D6 — Extract one shared retry policy module now, or keep 5 copy-pasted envelopes?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: You just decided the curve shape, jitter, the attempt ceiling, the dead-letter alert and how errors are classified. Each of those has to live somewhere. If the retry envelope stays copy-pasted in 5 workers, every one of those decisions is copied 5 times, and the first bug fix or tuning change has to be made in 5 places and tested 5 times. One small shared module applies each decision once and every worker gets it.\nStakes if we pick wrong: a curve or ceiling bug fixed in 4 of 5 workers, or a jitter change that lands in 3, and nobody notices until the fifth worker stampedes a downstream.\nRecommendation: A because the plan already states the 5 bodies are identical, the module is about 50 lines, the migration is mechanical per worker, and doing the refactor commit before the behavior commit keeps each step reviewable. (human: ~1 day / CC: ~30 min)\nCompleteness: A=9/10, B=6/10, C=3/10\nPros / cons:\nA) Extract now, migrate all 5 (recommended)\n ✅ Curve, jitter, ceiling, dead-letter alert and classifier shape are each implemented and tested exactly once\n ✅ Refactor commit lands before the behavior-change commit, so each is small and reviewable on its own\n ❌ A defect in the shared module affects all 5 workers at once; the contract tests are the guard\nB) Extract now, migrate webhook only\n ✅ Smallest first step; proves the module against the worker whose behavior is changing anyway\n ✅ Other 4 workers are untouched in this change, so their risk is zero for now\n ❌ Leaves 4 copies carrying the new policy by hand, so the duplication the plan already called out gets worse, not better\nC) Leave duplication\n ✅ No refactor risk in this change at all\n ✅ Matches the plan as written\n ❌ Every approved policy decision is copy-pasted 5 times and drifts from the first fix onward\nNet: one 50-line module now against 5 hand-maintained copies of every retry decision.",
|
||||
"header": "Shared module",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Extract now, migrate all 5 (recommended)",
|
||||
"description": "Create one `retryPolicy` module exporting backoffStrategy(attempt, rng) with equal jitter and a configurable max-delay clamp, DEFAULT_MAX_ATTEMPTS, onDeadLetter(job, err) emitting the structured log and metric, and isNonRetryable(err, list). Structured attempt log fields: job id, attempt, delay, error class, decision. Inline ASCII state diagram in the module header. All 5 workers register with the library through it, each passing its own non-retryable list. Land the refactor commit before the behavior-change commit. Required proof: shared-contract unit tests for each export plus one integration test per worker that the library invokes the shared policy on failure. Completeness 9/10. human: ~1 day / CC: ~30 min."
|
||||
},
|
||||
{
|
||||
"label": "Extract now, migrate webhook only",
|
||||
"description": "Create the same `retryPolicy` module and migrate only the webhook worker in this change. The other 4 workers keep their copied envelopes and apply R3/R4/R5 by hand until a follow-up (TODO). Required proof: shared-contract unit tests plus one webhook integration test. Completeness 6/10. human: ~half day / CC: ~15 min."
|
||||
},
|
||||
{
|
||||
"label": "Leave duplication",
|
||||
"description": "Keep 5 copy-pasted envelopes as the plan proposes. Apply R3/R4/R5 policy in each copy. No shared module, no shared tests; per-copy tests only. Completeness 3/10. human: ~1 day of copy-paste / CC: ~20 min."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D6 — Extract one shared retry policy module now, or keep 5 copy-pasted envelopes?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: You just decided the curve shape, jitter, the attempt ceiling, the dead-letter alert and how errors are classified. Each of those has to live somewhere. If the retry envelope stays copy-pasted in 5 workers, every one of those decisions is copied 5 times, and the first bug fix or tuning change has to be made in 5 places and tested 5 times. One small shared module applies each decision once and every worker gets it.\nStakes if we pick wrong: a curve or ceiling bug fixed in 4 of 5 workers, or a jitter change that lands in 3, and nobody notices until the fifth worker stampedes a downstream.\nRecommendation: A because the plan already states the 5 bodies are identical, the module is about 50 lines, the migration is mechanical per worker, and doing the refactor commit before the behavior commit keeps each step reviewable. (human: ~1 day / CC: ~30 min)\nCompleteness: A=9/10, B=6/10, C=3/10\nPros / cons:\nA) Extract now, migrate all 5 (recommended)\n ✅ Curve, jitter, ceiling, dead-letter alert and classifier shape are each implemented and tested exactly once\n ✅ Refactor commit lands before the behavior-change commit, so each is small and reviewable on its own\n ❌ A defect in the shared module affects all 5 workers at once; the contract tests are the guard\nB) Extract now, migrate webhook only\n ✅ Smallest first step; proves the module against the worker whose behavior is changing anyway\n ✅ Other 4 workers are untouched in this change, so their risk is zero for now\n ❌ Leaves 4 copies carrying the new policy by hand, so the duplication the plan already called out gets worse, not better\nC) Leave duplication\n ✅ No refactor risk in this change at all\n ✅ Matches the plan as written\n ❌ Every approved policy decision is copy-pasted 5 times and drifts from the first fix onward\nNet: one 50-line module now against 5 hand-maintained copies of every retry decision.": "Extract now, migrate all 5 (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:53:59.810Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_01Pv8bKkKS2rSamqKz1rvNX1",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D7 — How do we protect the existing `processWebhookJob()` behavior through the rewrite?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: You are rewriting the code that sends webhooks to customers, and there are no tests around it. The rewrite is supposed to keep everything the same except how failures are handled. Without tests written against the current code first, there is no way to know whether the new version still sends the same request, with the same headers and signature, exactly once. The cheapest insurance is to pin the current behavior in tests before changing a line, then keep them green.\nStakes if we pick wrong: a subtly different request body or signature ships to every webhook receiver at once, or a duplicate send slips through, and the first signal is a customer complaint.\nRecommendation: A because characterization tests are cheap with AI, they are the only way to detect an unintended difference in a rewrite, and they become the permanent contract suite for the webhook path. (human: ~1 day / CC: ~20 min)\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Characterize first, then rewrite (recommended)\n ✅ Pins request shape, headers, signature and success bookkeeping against the current code, so any unintended difference fails a test\n ✅ Intentional changes (pre-send retry, dead-letter) are written as explicit tests, so the diff between old and new behavior is documented\n ❌ Requires reading the current implementation carefully and a day of test writing before the rewrite starts\nB) At-most-once assertions only\n ✅ Covers the one guarantee the plan named as at risk\n ✅ Faster to write; no characterization pass\n ❌ Request shape, headers, signature and success bookkeeping can change silently and no test notices\nNet: pin the whole current contract for a day of work, or protect one guarantee and hope the rest survived.",
|
||||
"header": "Webhook regression",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Characterize first, then rewrite (recommended)",
|
||||
"description": "Before modifying `processWebhookJob()`, write characterization tests (e.g. `processWebhookJob.test`) against the current implementation asserting: exact request body, headers and signature for a fixed payload; exactly one send on success, on timeout and on 5xx; success bookkeeping. Keep them green through the rewrite. Then add intentional-difference tests: pre-send failure schedules a retry; timeout/5xx lands in dead-letter with no second send. Flag the suite CRITICAL in the plan. R8 stays pending. Completeness 9/10. human: ~1 day / CC: ~20 min."
|
||||
},
|
||||
{
|
||||
"label": "At-most-once assertions only",
|
||||
"description": "After the rewrite, add tests asserting exactly one send on success, timeout and 5xx, and a retry on pre-send failure. No characterization of request shape, headers, signature or success bookkeeping. R8 stays pending. Completeness 6/10. human: ~2 h / CC: ~5 min."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D7 — How do we protect the existing `processWebhookJob()` behavior through the rewrite?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: You are rewriting the code that sends webhooks to customers, and there are no tests around it. The rewrite is supposed to keep everything the same except how failures are handled. Without tests written against the current code first, there is no way to know whether the new version still sends the same request, with the same headers and signature, exactly once. The cheapest insurance is to pin the current behavior in tests before changing a line, then keep them green.\nStakes if we pick wrong: a subtly different request body or signature ships to every webhook receiver at once, or a duplicate send slips through, and the first signal is a customer complaint.\nRecommendation: A because characterization tests are cheap with AI, they are the only way to detect an unintended difference in a rewrite, and they become the permanent contract suite for the webhook path. (human: ~1 day / CC: ~20 min)\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Characterize first, then rewrite (recommended)\n ✅ Pins request shape, headers, signature and success bookkeeping against the current code, so any unintended difference fails a test\n ✅ Intentional changes (pre-send retry, dead-letter) are written as explicit tests, so the diff between old and new behavior is documented\n ❌ Requires reading the current implementation carefully and a day of test writing before the rewrite starts\nB) At-most-once assertions only\n ✅ Covers the one guarantee the plan named as at risk\n ✅ Faster to write; no characterization pass\n ❌ Request shape, headers, signature and success bookkeeping can change silently and no test notices\nNet: pin the whole current contract for a day of work, or protect one guarantee and hope the rest survived.": "Characterize first, then rewrite (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:55:40.441Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_01N7xKjvDw5oJhSmyLK3zRAR",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D8 — Do the per-worker integration tests run the real job library, or a mocked hook?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: The whole point of D1 was that the library remembers attempt counts across worker restarts and keeps failed jobs somewhere you can find them. A test that fakes the library cannot check either of those; it only checks that your function was called. Running the real library against a throwaway test queue is slower but proves the parts you are relying on actually behave.\nStakes if we pick wrong: the retry count resets on every deploy or the failed set is empty when you need it, and every test was green because the mock said so.\nRecommendation: A because the library's persistence and failed set are load-bearing assumptions from D1 and D3, and a mock cannot verify either; the cost is a test backend fixture the library almost certainly already ships.\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Real library backend in tests (recommended)\n ✅ Proves attempt count survives a worker restart and that exhausted jobs are actually in the failed set with their last error\n ✅ Catches library-version behavior changes and off-by-one attempt numbering that a stub would hide\n ❌ Slower suite and a test backend fixture to maintain (in-process queue or container)\nB) Mocked library hooks\n ✅ Fast, deterministic, no external fixture\n ✅ Enough to prove the worker wiring calls the shared policy\n ❌ Restart persistence and failed-set contents stay unverified, which are exactly the guarantees D1 and D3 depend on\nNet: a slower fixture that verifies the library promises you are betting on, or fast tests that trust them.",
|
||||
"header": "Integration depth",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Real library backend in tests (recommended)",
|
||||
"description": "Per-worker integration tests (5) run the actual job library against a test backend (in-process or containerized queue). Assertions: strategy invoked with real attempt numbers; attempt count survives a simulated worker restart mid-backoff; after the ceiling the job is in the failed set with its last error; a listed non-retryable error is in the failed set after attempt 1. Mark [E2E]. Completeness 9/10. human: ~1 day / CC: ~30 min."
|
||||
},
|
||||
{
|
||||
"label": "Mocked library hooks",
|
||||
"description": "Per-worker tests stub the library retry hook and assert the shared policy is invoked with the expected arguments. No restart or failed-set verification. Completeness 6/10. human: ~2 h / CC: ~10 min."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D8 — Do the per-worker integration tests run the real job library, or a mocked hook?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: The whole point of D1 was that the library remembers attempt counts across worker restarts and keeps failed jobs somewhere you can find them. A test that fakes the library cannot check either of those; it only checks that your function was called. Running the real library against a throwaway test queue is slower but proves the parts you are relying on actually behave.\nStakes if we pick wrong: the retry count resets on every deploy or the failed set is empty when you need it, and every test was green because the mock said so.\nRecommendation: A because the library's persistence and failed set are load-bearing assumptions from D1 and D3, and a mock cannot verify either; the cost is a test backend fixture the library almost certainly already ships.\nCompleteness: A=9/10, B=6/10\nPros / cons:\nA) Real library backend in tests (recommended)\n ✅ Proves attempt count survives a worker restart and that exhausted jobs are actually in the failed set with their last error\n ✅ Catches library-version behavior changes and off-by-one attempt numbering that a stub would hide\n ❌ Slower suite and a test backend fixture to maintain (in-process queue or container)\nB) Mocked library hooks\n ✅ Fast, deterministic, no external fixture\n ✅ Enough to prove the worker wiring calls the shared policy\n ❌ Restart persistence and failed-set contents stay unverified, which are exactly the guarantees D1 and D3 depend on\nNet: a slower fixture that verifies the library promises you are betting on, or fast tests that trust them.": "Real library backend in tests (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:56:39.620Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_01XEdmtg1U25aqBrEGFb8dfg",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D9 — Cache the dependency graph across retries, or recompute it on every attempt?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: Every time a job retries, the plan reads the whole payload from the database again and rebuilds the same dependency graph from it. The payload never changes between attempts, so the answer is always the same. With up to 5 attempts, that is up to 5 reads and 5 builds per failing job, and failing jobs pile up exactly when something is already down. Building once and storing the result with the job removes almost all of that.\nStakes if we pick wrong: a downstream outage turns into a database load spike from your own retries, or you spend effort caching something that turns out to be cheap.\nRecommendation: A because the graph is derived from an immutable payload, the library already stores job data per attempt, and the guard (payload hash check) makes the cache safe; it also removes the redundant DB fetch. Confidence is medium: if payloads are tiny and the graph build is microseconds, C is acceptable and this becomes a TODO.\nCompleteness: A=9/10, B=5/10, C=4/10\nPros / cons:\nA) Compute once, store on job (recommended)\n ✅ Retries read no extra payload and build no graph; outage-time DB load drops from 5N to about N\n ✅ Survives worker restarts and works across workers because the cache lives in the job data, not in a process\n ❌ Adds serialized graph size to each job record and needs a payload-hash guard to stay correct\nB) In-process memo\n ✅ Simple to add, no change to job data shape\n ✅ Helps when the same worker process picks up the retry\n ❌ Retries usually land on a different worker or after a restart, so the memo misses most of the time and still re-fetches the payload\nC) Leave as-is\n ✅ Zero new code and no cache correctness to reason about\n ✅ Bounded at 5 attempts by D3, so the waste is finite\n ❌ Every retry storm during an outage multiplies database reads by up to 5\nNet: one persisted derived value with a hash guard, or accept a 5x read multiplier exactly when the system is least healthy.",
|
||||
"header": "Graph caching",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Compute once, store on job (recommended)",
|
||||
"description": "On attempt 1, read the payload (from the library's job data if it carries it, else one DB fetch), build the dependency graph, and persist the serialized graph plus a payload hash in the job data. On later attempts, verify the hash and deserialize; on mismatch, rebuild. Required proof: unit test that attempt 2+ performs no DB fetch and no graph build when the hash matches; test that a hash mismatch triggers a rebuild. Completeness 9/10. human: ~half day / CC: ~15 min."
|
||||
},
|
||||
{
|
||||
"label": "In-process memo",
|
||||
"description": "Memoize the built graph per job id in worker memory. Payload re-fetch unchanged. Required proof: test that a second attempt in the same process reuses the graph. Completeness 5/10. human: ~1 h / CC: ~5 min."
|
||||
},
|
||||
{
|
||||
"label": "Leave as-is",
|
||||
"description": "Re-fetch the payload and rebuild the graph on every attempt, as the plan proposes. No new tests. Completeness 4/10. human: 0 / CC: 0."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D9 — Cache the dependency graph across retries, or recompute it on every attempt?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: Every time a job retries, the plan reads the whole payload from the database again and rebuilds the same dependency graph from it. The payload never changes between attempts, so the answer is always the same. With up to 5 attempts, that is up to 5 reads and 5 builds per failing job, and failing jobs pile up exactly when something is already down. Building once and storing the result with the job removes almost all of that.\nStakes if we pick wrong: a downstream outage turns into a database load spike from your own retries, or you spend effort caching something that turns out to be cheap.\nRecommendation: A because the graph is derived from an immutable payload, the library already stores job data per attempt, and the guard (payload hash check) makes the cache safe; it also removes the redundant DB fetch. Confidence is medium: if payloads are tiny and the graph build is microseconds, C is acceptable and this becomes a TODO.\nCompleteness: A=9/10, B=5/10, C=4/10\nPros / cons:\nA) Compute once, store on job (recommended)\n ✅ Retries read no extra payload and build no graph; outage-time DB load drops from 5N to about N\n ✅ Survives worker restarts and works across workers because the cache lives in the job data, not in a process\n ❌ Adds serialized graph size to each job record and needs a payload-hash guard to stay correct\nB) In-process memo\n ✅ Simple to add, no change to job data shape\n ✅ Helps when the same worker process picks up the retry\n ❌ Retries usually land on a different worker or after a restart, so the memo misses most of the time and still re-fetches the payload\nC) Leave as-is\n ✅ Zero new code and no cache correctness to reason about\n ✅ Bounded at 5 attempts by D3, so the waste is finite\n ❌ Every retry storm during an outage multiplies database reads by up to 5\nNet: one persisted derived value with a hash guard, or accept a 5x read multiplier exactly when the system is least healthy.": "Compute once, store on job (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T17:58:34.047Z"
|
||||
},
|
||||
{
|
||||
"sessionId": "cd04b55a-5f1a-4672-b3ed-40ea1b1636bc",
|
||||
"toolUseId": "toolu_012XHAmgKwXGVbhDkoCK1K1R",
|
||||
"questions": [
|
||||
{
|
||||
"question": "D10 — Record the webhook at-least-once upgrade as a TODO?\nProject/branch/task: gstack-plan-count-yWJb6k on main, retry framework plan; follow-up to D2.\nELI10: In D2 you chose to keep webhooks at \"send at most once\", so a slow or erroring receiver means that delivery is dropped into the failed set instead of retried. The fix (retry with a delivery id the receiver can dedupe) needs receiver work first. This question only decides whether we write that follow-up down in TODOS.md so it does not get lost.\nStakes if we pick wrong: skipped, the only trace is a code comment and a decision-log row; built now, this PR grows and contradicts the D2 call.\nRecommendation: A because the upgrade has an external prerequisite and a clear trigger, which is exactly what a TODO is for.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n ✅ Creates TODOS.md with the What/Why/Context/Depends-on record, findable by /retro and future reviews\n ✅ Zero implementation cost now; the marker in code and the TODO entry point at each other\n ❌ One more file in the repo that someone has to keep honest as work lands\nB) Skip\n ✅ No new file; the decision log and gstack-shortcut marker already carry the trigger\n ✅ Avoids a TODO nobody may pick up if receivers never add dedupe\n ❌ The trigger lives only in a comment and a JSONL row, easy to miss when receivers do change\nC) Build it now in this PR\n ✅ Ships the stronger delivery guarantee in the same change as the retry framework\n ✅ Reuses the retryPolicy module while it is fresh\n ❌ Reverses D2 and depends on receiver dedupe that does not exist yet, so duplicates would reach receivers\nNet: a TODO entry now versus relying on a code marker alone; building now is off the table until receivers can dedupe.",
|
||||
"header": "Webhook TODO",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Add to TODOS.md (recommended)",
|
||||
"description": "Create TODOS.md at implementation time with the TODO record (What/Why/Context/Depends-on) under a `## Workers` section, P3, effort M. No product code change."
|
||||
},
|
||||
{
|
||||
"label": "Skip",
|
||||
"description": "Do not create TODOS.md. The decision log entry and the gstack-shortcut marker remain the only trail."
|
||||
},
|
||||
{
|
||||
"label": "Build it now in this PR",
|
||||
"description": "Extend the accepted scope to at-least-once delivery with delivery id and idempotency key; would reopen D2."
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"answered": true,
|
||||
"failed": false,
|
||||
"answers": {
|
||||
"D10 — Record the webhook at-least-once upgrade as a TODO?\nProject/branch/task: gstack-plan-count-yWJb6k on main, retry framework plan; follow-up to D2.\nELI10: In D2 you chose to keep webhooks at \"send at most once\", so a slow or erroring receiver means that delivery is dropped into the failed set instead of retried. The fix (retry with a delivery id the receiver can dedupe) needs receiver work first. This question only decides whether we write that follow-up down in TODOS.md so it does not get lost.\nStakes if we pick wrong: skipped, the only trace is a code comment and a decision-log row; built now, this PR grows and contradicts the D2 call.\nRecommendation: A because the upgrade has an external prerequisite and a clear trigger, which is exactly what a TODO is for.\nNote: options differ in kind, not coverage — no completeness score.\nPros / cons:\nA) Add to TODOS.md (recommended)\n ✅ Creates TODOS.md with the What/Why/Context/Depends-on record, findable by /retro and future reviews\n ✅ Zero implementation cost now; the marker in code and the TODO entry point at each other\n ❌ One more file in the repo that someone has to keep honest as work lands\nB) Skip\n ✅ No new file; the decision log and gstack-shortcut marker already carry the trigger\n ✅ Avoids a TODO nobody may pick up if receivers never add dedupe\n ❌ The trigger lives only in a comment and a JSONL row, easy to miss when receivers do change\nC) Build it now in this PR\n ✅ Ships the stronger delivery guarantee in the same change as the retry framework\n ✅ Reuses the retryPolicy module while it is fresh\n ❌ Reverses D2 and depends on receiver dedupe that does not exist yet, so duplicates would reach receivers\nNet: a TODO entry now versus relying on a code marker alone; building now is off the table until receivers can dedupe.": "Add to TODOS.md (recommended)"
|
||||
},
|
||||
"unansweredQuestionIndices": [],
|
||||
"answeredAt": "2026-09-29T18:01:55.194Z"
|
||||
}
|
||||
],
|
||||
"plan": "# Engineering review: Add background job retry framework\n\nReview target (fixed): `PLAN.md` in `/home/runner/.cache/gstack-paid-shard-mmiSh1/tmp/gstack-plan-count-yWJb6k` (branch `main`, commit `0ad2479`).\nReport file: this file (path requested by the user).\nReviewer: /plan-eng-review, session `642-1790703811-6005ed0c`, 2026-09-29.\n\n## Original plan (unchanged copy)\n\n# Plan: Add background job retry framework\n\n## Architecture\nWe'll roll a custom exponential-backoff scheduler inline in each worker\nrather than use the existing job library's built-in retry hooks. Same\nshape as the library version, but we want full control over the curve.\n\n## Code quality\nThe retry envelope (compute delay, log attempt, dispatch) is duplicated\nacross 5 worker files with copy-pasted bodies. We will leave the\nduplication for now and refactor \"later.\"\n\n## Tests\nThe existing `processWebhookJob()` flow gets rewritten as part of this\nchange. No regression test for the prior at-most-once delivery guarantee\nis planned.\n\n## Performance\nOn every retry we re-fetch the full job payload from the database, then\niterate the payload to recompute the dependency graph. Could cache the\ngraph on the first attempt; not planned.\n\n## Scope Challenge record\n\nEvidence available: plan text only. The repo contains `PLAN.md` and `CLAUDE.md`; the 5 worker files, `processWebhookJob()`, the job library and its retry hooks are `not available` in this checkout. Findings quote plan lines and are calibrated as plan-text findings.\n\nComplexity count (estimates from plan text): ~5-6 changed files (5 worker files; `processWebhookJob()` may live in one of them), 0 new classes/services (scheduler is inline). Below the 8-file / 2-class gate, so the complexity selectors (B) are skipped.\n\nSearch check: Aside unavailable, host WebSearch used. Industry default [Layer 1]: library built-in retry, exponential backoff + jitter, bounded attempts, dead-letter, idempotent handlers.\n\n## Decision ledger\n\n### R1: Retry scheduler mechanism (library hooks vs custom inline scheduler)\nFinding: SC-1, P1, confidence 8/10, PLAN.md:7-9, reviewer: plan-eng-review (native)\nPlan baseline: original proposal, \"custom exponential-backoff scheduler inline in each worker rather than use the existing job library's built-in retry hooks\" (PLAN.md:7-9). Nothing approved yet.\nRuntime evidence: unknown. Job library and worker files not available in this checkout; plan text states the library has built-in retry hooks and the custom version is the \"same shape\".\nComparison grid:\n\n| Choice | Current | A) Library hooks + custom curve | B) Custom inline scheduler |\n|---|---|---|---|\n| R1 retry mechanism | custom inline scheduler (proposed) | library retry hooks, backoff supplied as one strategy function | custom scheduler inline per worker, as proposed |\n| Backoff curve ownership | \"full control\" wanted | full control via strategy function (verify hook accepts a function; else fall back to B) | full control |\n| Attempt count persistence / terminal handling | unspecified | inherited from library | must be hand-built (pending, R3) |\n| R2 webhook delivery semantics | pending | pending | pending |\n| R3 attempt bound + dead-letter | pending | pending | pending |\n| R4 jitter | pending | pending | pending |\n| R5 shared envelope | pending | pending | pending |\n\nQuestion D1:\nD1 — Use the job library's retry hooks or roll a custom inline scheduler?\nProject/branch/task: `main` of the plan fixture repo, plan \"Add background job retry framework\".\nELI10: The job library you already run has a retry system built in. The plan wants to rebuild the same thing by hand inside each of the 5 workers, only so the delay curve can be tuned. Retry systems are easy to get subtly wrong: losing the attempt count when a worker restarts, retrying forever, or retrying twice at once. The library has already paid for those bugs; a hand-rolled copy pays for them again in production.\nStakes if we pick wrong: a hand-rolled scheduler that drops attempt state on restart or double-schedules turns one failed job into duplicate side effects or an infinite retry loop, with no dead-letter to catch it.\nRecommendation: A because the plan admits the shapes are identical, and the curve is pluggable in the library through a strategy function, so A delivers the same control with far less new code. (human: ~1 day / CC: ~20 min for A; human: ~1 week / CC: ~2 h for B plus ongoing ownership)\nCompleteness: A=9/10, B=5/10\nPros / cons:\nA) Library hooks + custom curve (recommended)\n ✅ Attempt counting, persistence across restarts and terminal handling come from tested library code, not new code\n ✅ The custom curve still lives in one strategy function, so \"full control over the curve\" is preserved\n ❌ Requires confirming the library's hook accepts a custom delay function; if it does not, we fall back to B for the curve only\nB) Custom inline scheduler\n ✅ Zero dependencLine truncated
|
||||
}
|
||||
@@ -117,6 +117,9 @@ export function isEngBatchingIssueAUQ(fp: AskUserQuestionFingerprint, priorCalls
|
||||
return !priorCalls.some(prior => batchingIssueNumber(prior) === issue);
|
||||
}
|
||||
|
||||
// The report's one target declaration, in the skill's own spellings.
|
||||
const TARGET_FIELD = /^(?:Reviewed |Review )?[Tt]arget(?: \(fixed\))?:/;
|
||||
|
||||
/** A native brief can use its D number and topic while its stable R identity
|
||||
* lives in the required saved ledger. Count that owned choice, not a title
|
||||
* spelling. This does not approve the row or validate the implementation. */
|
||||
@@ -160,10 +163,10 @@ function recordedBatchingIssue(call: NativePlanQuestionCall, savedPlan: string):
|
||||
const rawSourceNames = [...(lines[1] ?? '').matchAll(/\b[\w./-]+\.md\b/g)];
|
||||
const directSource = sourceNames.length > 0 && sourceNames.every(name => name === 'PLAN.md') &&
|
||||
new Set([...metadata.matchAll(/\bPLAN\.md:([1-9]\d*(?:[-–][1-9]\d*)?)\b/g)].map(match => match[1])).size <= 1;
|
||||
const targetName = (s: string) => clean(s).replace(/^Eng(?:ineering)? review:\s*/i, '')
|
||||
const targetName = (s: string) => clean(s).replace(/^Eng(?:ineering)? review\s*[:—–-]\s*/i, '')
|
||||
.replace(/^Plan\s*[:—–-]\s*/i, '').toLowerCase();
|
||||
const named = [...(lines[1] ?? '').matchAll(/"(Plan:\s*[^"\n]+)"|“(Plan:\s*[^”\n]+)”/g)]
|
||||
.map(match => targetName(match[1] ?? match[2]!));
|
||||
const named = [...(lines[1] ?? '').matchAll(/"(Plan:\s*[^"\n]+)"|“(Plan:\s*[^”\n]+)”|\b[Pp]lan\s+"([^"\n]+)"|\b[Pp]lan\s+“([^”\n]+)”/g)]
|
||||
.map(match => targetName(match[1] ?? match[2] ?? match[3] ?? match[4]!));
|
||||
const titles = tokens.slice(0, start).filter(token => token.type === 'heading' && token.depth === 1);
|
||||
const targetFields = tokens.slice(0, start).flatMap((token, at) => {
|
||||
if (token.type !== 'paragraph' || !currentHeading(at)) return [];
|
||||
@@ -171,14 +174,16 @@ function recordedBatchingIssue(call: NativePlanQuestionCall, savedPlan: string):
|
||||
const quotedContext = /\b(?:quoted|copied|historical|example|hypothetical|archived)\b[^\n]*:\s*$/i;
|
||||
if (previous?.type === 'paragraph' && quotedContext.test(previous.raw)) return [];
|
||||
const parts = token.raw.split('\n');
|
||||
return parts.filter((line, i) => /^Reviewed target:/.test(line) &&
|
||||
return parts.filter((line, i) => TARGET_FIELD.test(line) &&
|
||||
!parts.slice(0, i).some(part => quotedContext.test(part)));
|
||||
});
|
||||
const namedSource = !rawSourceNames.length && named.length === 1 && titles.length === 1 &&
|
||||
// The report title owns the target; an unfenced copy of the reviewed plan
|
||||
// may add its own H1 only when it names that same plan.
|
||||
const namedSource = !rawSourceNames.length && named.length === 1 && titles.length >= 1 &&
|
||||
titles[0]!.type === 'heading' && currentHeading(tokens.indexOf(titles[0]!)) &&
|
||||
/^Eng(?:ineering)? review:\s*Plan\s*[:—–-]/i.test(clean(titles[0]!.text)) &&
|
||||
targetName(titles[0]!.text) === named[0] && targetFields.length === 1 &&
|
||||
/^Reviewed target:\s*`?PLAN\.md`?(?:\s|$)/.test(targetFields[0]!) &&
|
||||
/^Eng(?:ineering)? review\s*[:—–-]\s*\S/i.test(clean(titles[0]!.text)) &&
|
||||
titles.every(title => title.type === 'heading' && targetName(title.text) === named[0]) && targetFields.length === 1 &&
|
||||
new RegExp(`${TARGET_FIELD.source}\\s*\`?PLAN\\.md\`?(?:\\s|$)`).test(targetFields[0]!) &&
|
||||
[...targetFields[0]!.matchAll(/\b[\w./-]+\.md\b/g)].length === 1;
|
||||
if (!directSource && !namedSource) return;
|
||||
const withdrawn = (value: string, owners: string) => new RegExp(
|
||||
@@ -265,7 +270,7 @@ function recordedBatchingIssue(call: NativePlanQuestionCall, savedPlan: string):
|
||||
if (questions.length !== 1) continue;
|
||||
const inlineBrief = fields[questions[0]!]!.slice(marker.length).trim();
|
||||
const inline = Boolean(inlineBrief);
|
||||
if (!inline && (namedSource || clean(fields[questions[0]! + 1] ?? '') !== clean(title))) continue;
|
||||
if (!inline && clean(fields[questions[0]! + 1] ?? '') !== clean(title)) continue;
|
||||
const sources = [...finding[0]!.matchAll(/\b([\w./-]+\.md)(?::([1-9]\d*(?:[-–][1-9]\d*)?))?\b/g)];
|
||||
if (sources.length !== 1 || sources[0]![1] !== 'PLAN.md' ||
|
||||
!inline && !sources[0]![2] || source && sources[0]![2] !== source) continue;
|
||||
|
||||
Reference in new issue
Block a user