diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 8226db504..c67312474 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -337,6 +337,7 @@ gstack_result_anchor() { RECEIPT_CARRY_REFUSED) printf '%s' 'receipt-carry-refused' ;; PROSE_LINT_FOUND) printf '%s' 'prose-lint-found' ;; REVIEW_LOOP_EXCEPTION_INVALID) printf '%s' 'review-loop-exception-invalid' ;; + CI_CARRY_RERUN_MALFORMED) printf '%s' 'ci-carry-rerun-malformed' ;; RECEIPT_CARRY_TREE_MISMATCH) printf '%s' 'receipt-carry-tree-mismatch' ;; RECEIPT_CARRY_PRODUCT_CHANGED) printf '%s' 'receipt-carry-product-changed' ;; RECEIPT_CARRY_SCOPED_REVIEW) printf '%s' 'receipt-carry-scoped-review' ;; diff --git a/bin/gstack-evidence b/bin/gstack-evidence index 753a4c445..924f8cccd 100755 --- a/bin/gstack-evidence +++ b/bin/gstack-evidence @@ -101,7 +101,7 @@ import { isUnavailable, loadPolicyFromBase, resolveBaseBranch } from "../lib/shi import { versionAtRef } from "../lib/restamp"; import { bashCommandInvocation, runScriptSync, wtreeFingerprint } from "../lib/script-invocation"; import { carriedFromLine, findCarry, laneMapping, mappingCoverage, perFileCommand } from "../lib/ci-carry"; -import { DEFAULT_MAX_AGE_HOURS, runChanged } from "../lib/run-evidence"; +import { DEFAULT_MAX_AGE_HOURS, rerunByRun, rerunMarker, runChanged } from "../lib/run-evidence"; import { isConfigUnavailable, loadConfigFromBase } from "../lib/pregate/config"; import { buildRunCheck } from "../lib/pregate/run-check"; import { readTestLaneExceptions, testLaneExceptionText, type BoundLaneException } from "../lib/test-lane-exception"; @@ -149,6 +149,8 @@ interface EvidenceRecord { skipped_here?: { label: string; files: string[]; tests_ran: number | "unknown"; not_owed?: string[] }; /** The owner's test-lane exception that let a lane carry with uncovered inventory files, and the preflight of exactly them here (GST-1595). */ exception?: BoundLaneException; + /** Per carried run (`/`) whose tree.json lists files that passed only on a re-run: how many, and the run's attempts (record-ci, GST-3054). */ + rerun?: Record; } /** Bytes of streamed output kept for the tests_ran parse (the 2MB log would truncate the summary). */ @@ -714,7 +716,7 @@ function reused(file: string, label: string, cmdSha: string, maxAgeHours: number /** record-ci's record line, the one lib/ci-carry.ts recordCiResult reads. */ const recordedCiLine = (r: EvidenceRecord): string => - `gstack-evidence: recorded label=${r.label} exit=0 tests_ran=${r.tests_ran}${r.tests_unit ? ` unit=${r.tests_unit}` : ""} log=- source=ci runs=${r.runs!.join(",")} policy=${r.policy}${r.skipped_here ? ` skipped_here=${r.skipped_here.files.length} tests_ran_here=${r.skipped_here.tests_ran}${r.skipped_here.not_owed ? ` not_owed_here=${r.skipped_here.not_owed.length}` : ""}` : ""}`; + `gstack-evidence: recorded label=${r.label} exit=0 tests_ran=${r.tests_ran}${r.tests_unit ? ` unit=${r.tests_unit}` : ""} log=- source=ci runs=${r.runs!.join(",")}${rerunMarker(Object.values(r.rerun ?? {}))} policy=${r.policy}${r.skipped_here ? ` skipped_here=${r.skipped_here.files.length} tests_ran_here=${r.skipped_here.tests_ran}${r.skipped_here.not_owed ? ` not_owed_here=${r.skipped_here.not_owed.length}` : ""}` : ""}`; /** * record-ci's reuse (GST-1795): the label's newest ledger row stands only when it is the CI row this record-ci would write again @@ -802,7 +804,9 @@ async function cmdRecordCi(argv: string[]): Promise { } if (!v.ok) return none(v.reason, `${v.detail}${v.reason === "inventory_uncovered" && !atHead.length && elsewhere.length ? `; test-lane exception ${elsewhere.join(", ")} binds another head, not HEAD ${head.slice(0, 12)}` : ""}`, v.diagnosis); const runs = v.runs.map((r) => `${r.run_id}/${r.run_attempt}`); - const prior = v.excepted?.length ? null : reusableCiRow(label, { commit: head, wtree: tree, cmd_sha256: sha256(command), runs, profiles: m.profiles, policy: loaded.label, jobs: m.jobs, tests_ran: v.tests, tests_unit: v.unit, not_owed: v.notOwed }, v.remainder ?? [], m.per_file); + const reran = rerunByRun(v.runs); + const rerun = reran.size ? Object.fromEntries(reran) : undefined; + const prior = v.excepted?.length ? null : reusableCiRow(label, { commit: head, wtree: tree, cmd_sha256: sha256(command), runs, profiles: m.profiles, policy: loaded.label, jobs: m.jobs, tests_ran: v.tests, tests_unit: v.unit, not_owed: v.notOwed, rerun }, v.remainder ?? [], m.per_file); if (prior) { console.error(recordedCiLine(prior)); if (prior.not_owed) console.error(`gstack-evidence: not owed: ${prior.not_owed}`); @@ -867,7 +871,7 @@ async function cmdRecordCi(argv: string[]): Promise { } const rec: EvidenceRecord = { ts: new Date(v.runs.map((r) => r.finished_at).sort()[0]!).toISOString(), label, command, cmd_sha256: sha256(command), exit: 0, duration_s: 0, - commit: head, tree, dirty: false, wtree: tree, tests_ran: testsRan, source: "ci", runs, profiles: m.profiles, policy: loaded.label, ...(m.jobs ? { jobs: m.jobs } : {}), ...(v.unit ? { tests_unit: v.unit } : {}), ...(v.notOwed ? { not_owed: v.notOwed } : {}), ...(here ? { skipped_here: here } : {}), ...(exception ? { exception } : {}), + commit: head, tree, dirty: false, wtree: tree, tests_ran: testsRan, source: "ci", runs, profiles: m.profiles, policy: loaded.label, ...(m.jobs ? { jobs: m.jobs } : {}), ...(v.unit ? { tests_unit: v.unit } : {}), ...(v.notOwed ? { not_owed: v.notOwed } : {}), ...(here ? { skipped_here: here } : {}), ...(exception ? { exception } : {}), ...(rerun ? { rerun } : {}), }; appendJsonl(paths.file, rec, { mode: 0o600 }); console.error(recordedCiLine(rec)); @@ -897,9 +901,14 @@ function cmdCarried(argv: string[]): number { for (const r of records) latest.set(r.label, r); const carried = [...latest.values()].filter((r) => r.source === "ci" && !!wtree && r.wtree === wtree && Array.isArray(r.runs) && typeof r.policy === "string" && r.policy.startsWith(policyRef) && gradeRecord(r, { maxAgeHours: DEFAULT_MAX_AGE_HOURS, allowPaths: [], wtreeNow: wtree }).verdict === "FRESH") - .flatMap((r) => r.runs!.map((ref) => { const [run, attempt] = ref.split("/").map(Number); return { lane: r.label, run, attempt, tree: r.wtree!, ...(r.exception ? { exception: r.exception } : {}) }; })); + .flatMap((r) => r.runs!.map((ref) => { + const [run, attempt] = ref.split("/").map(Number); + const re = r.rerun?.[ref]; + const reran = re && Number.isInteger(re.files) && re.files > 0 && Array.isArray(re.attempts) && re.attempts.every((n) => Number.isInteger(n) && n > 0) ? { rerun: re.files, attempts: re.attempts } : {}; + return { lane: r.label, run, attempt, tree: r.wtree!, ...reran, ...(r.exception ? { exception: r.exception } : {}) }; + })); if (argv.includes("--json")) console.log(JSON.stringify(carried)); - else for (const c of carried) console.log(`CI_CARRIED: ${c.lane} run=${c.run}/${c.attempt} tree=${c.tree.slice(0, 12)}`); + else for (const c of carried) console.log(`CI_CARRIED: ${c.lane} run=${c.run}/${c.attempt} tree=${c.tree.slice(0, 12)}${"rerun" in c ? rerunMarker([{ files: c.rerun!, attempts: c.attempts! }]) : ""}`); return 0; } diff --git a/changelog.d/ci-carry-marks-rerun-passes.md b/changelog.d/ci-carry-marks-rerun-passes.md new file mode 100644 index 000000000..0f4d5bff0 --- /dev/null +++ b/changelog.d/ci-carry-marks-rerun-passes.md @@ -0,0 +1 @@ +- **A CI run that passed only after its failed jobs were re-run is now marked wherever gstack reuses it.** When a run's `tree.json` lists `rerun_files` and `attempts` (gbrain writes these since gbrain#9690), the `CI_CARRY:` line, the `SHIP_RECEIPT:` line's `ci_carried=` field, and the receipt line `gstack-ship-queue ready` and `land` print now add `rerun= attempts=`, so a pass on a re-run reads as flaky instead of as a clean first-attempt pass. The run is still accepted. A `tree.json` without those fields prints exactly as before, and a malformed `rerun_files` or `attempts` refuses that run with `CI_CARRY_RERUN_MALFORMED`. diff --git a/docs/pregate.md b/docs/pregate.md index b37dec29f..25d3c12a4 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -1339,7 +1339,10 @@ each mapped lane from the green run at that exact tree, reuses or runs the rest, checks every lane FRESH and reruns the pregate there, whose tests stage carries the same run. `gstack-ship-receipt write` records the carried runs as `ci_carried: [{ lane, run, attempt, tree }]`, and the `SHIP_RECEIPT:` line names -them (`ci_carried=:/`). Each `record-ci` gets the base +them (`ci_carried=:/`). A run whose `tree.json` lists +`rerun_files` adds `rerun` and `attempts` to its entries and +`rerun= attempts=` after `ci_carried=`, as its `CI_CARRY:` line does +([ship-policy.md](ship-policy.md#ci-carry-rerun)). Each `record-ci` gets the base policy's [`ci_record_timeout_s`](ship-policy.md#ci-record-timeout) (600 s by default, 3600 s at most), and accept prints it as `CI_RECORD_TIMEOUT:`. diff --git a/docs/ship-policy.md b/docs/ship-policy.md index 3dbda5760..aba79c318 100644 --- a/docs/ship-policy.md +++ b/docs/ship-policy.md @@ -2730,6 +2730,17 @@ body plain: they become main's history. A `pull_request` CI run tests GitHub's merge commit rather than your branch, so on a branch behind main `gstack-ci-carry` reports `tree_mismatch`: dispatch the workflow on the branch (`gh workflow run --ref `). +- **A carried run that passed only after a re-run is marked, not refused.** + When only a run's failed jobs were re-run, its `tree.json` may list + `rerun_files` (the files whose newest result came from a re-run attempt) and + `attempts` (for example `[1, 2]`). The carry still accepts the run, but the + `CI_CARRY:` line, the `SHIP_RECEIPT:` line's `ci_carried=` field, and the + receipt line `ready` and `land` print all add `rerun= attempts=`, + so a pass on a re-run reads as flaky, never as a clean first-attempt pass. A + `tree.json` without `rerun_files`, or with an empty list, prints exactly as + before. A `rerun_files` that is not a list of paths, or `attempts` that is not + a list of attempt numbers, refuses that run + ([`CI_CARRY_RERUN_MALFORMED`](troubleshooting.md#ci-carry-rerun-malformed)). #### Every check land makes diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 27b7bf46f..103cbb718 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7251,6 +7251,23 @@ pass the `gstack-review-scope summary` file as `--review-loop`, put the printed line in the PR body, and write again ([ship-policy.md](ship-policy.md#ship-receipt)). + +### `CI_CARRY_RERUN_MALFORMED` + +**Meaning.** A CI run's `tree.json` names `rerun_files` or `attempts`, the +fields that say some test files passed only on a re-run attempt, but in a shape +gstack cannot read: `rerun_files` is not a list of file paths, or `attempts` is +not a list of attempt numbers of 1 or more. gstack fails closed: that run is not +evidence, the carry refuses it (`rerun_malformed` in the `CI_CARRY: none` or +`no_verified_run` detail), and the lane runs here instead. A well-formed record +is accepted and marked `rerun= attempts=` +([ship-policy.md](ship-policy.md#ci-carry-rerun)). + +**Fix.** Fix the workflow step that writes `tree.json` so it writes +`"rerun_files": ["test/a.test.ts", …]` and `"attempts": [1, 2]`, or leaves both +out. Then dispatch a fresh run at the same head and rerun +`gstack-ci-carry wait`. + ### `RECEIPT_CARRY_TREE_MISMATCH` diff --git a/lib/ci-carry.ts b/lib/ci-carry.ts index d183da7eb..15a3c5d1f 100644 --- a/lib/ci-carry.ts +++ b/lib/ci-carry.ts @@ -83,7 +83,7 @@ function newestVerified(src: RunSource, name: string, profile: RunProfile, heads for (const run of listed) { const v = verifyRun(src.deps, { repo: src.repo, runRef: String(run.id), profileName: name, profile, vetoHeads: heads, now: src.now, unchanged: src.unchanged, docsOnly }); if (v.ok) return v; - refusals.push(`run ${run.id}: ${v.reason}${v.fix ? ` (the pull request's base moved after the run; fix: ${v.fix})` : ''}`); + refusals.push(`run ${run.id}: ${v.reason}${v.reason === 'rerun_malformed' ? ` (${v.detail})` : ''}${v.fix ? ` (the pull request's base moved after the run; fix: ${v.fix})` : ''}`); } return no('no_verified_run', `${name}: ${refusals.join('; ')}`); } diff --git a/lib/headless-artifacts.ts b/lib/headless-artifacts.ts index 983ceb4ac..e612e1047 100644 --- a/lib/headless-artifacts.ts +++ b/lib/headless-artifacts.ts @@ -104,8 +104,8 @@ export interface ShipReceipt { gate?: Record; ci?: Record; spend_usd?: number | string; predecessor?: string; /** The run's calls from the call log (lib/ship-clock.ts callsSince: gstack-command calls plus keep-alives), so a 200-call run shows at the receipt; absent on older receipts. */ calls?: number; - /** GSTA-258: the lanes CI carried at this head's tree, one entry per run (gstack-evidence carried). */ - ci_carried?: Array<{ lane: string; run: number; attempt: number; tree: string; exception?: BoundLaneException }>; + /** GSTA-258: the lanes CI carried at this head's tree, one entry per run (gstack-evidence carried); `rerun` and `attempts` when files passed only on a re-run (GST-3054). */ + ci_carried?: Array<{ lane: string; run: number; attempt: number; tree: string; rerun?: number; attempts?: number[]; exception?: BoundLaneException }>; session_kind?: string; artifacts_consumed?: string; run?: string; // C-stamp (C3/C5/C7): the tree receipt's verdicts and the policy provenance, all optional. thread?: string; tree?: string; gate_reuse?: string; policy?: string; queue_mode?: string; preregistration?: string; history?: string; @@ -267,6 +267,7 @@ export const ARTIFACT_SCHEMAS = { schema_version: { const: 1 }, pr: { type: 'integer' }, head: str, version: str, base: str, gated_tree: str, gate: { type: 'object' }, ci: { type: 'object' }, spend_usd: { type: ['number', 'string'] }, calls: { type: 'integer', minimum: 0 }, predecessor: str, ci_carried: { type: 'array', items: { type: 'object', required: ['lane', 'run', 'attempt', 'tree'], properties: { lane: str, run: { type: 'integer', minimum: 1 }, attempt: { type: 'integer', minimum: 1 }, tree: str, + rerun: { type: 'integer', minimum: 1 }, attempts: { type: 'array', items: { type: 'integer', minimum: 1 } }, exception: { type: 'object', required: ['head', 'files', 'by', 'reason', 'at', 'approval', 'preflight'], properties: { head: str, files: strList, by: str, reason: str, at: str, approval: str, preflight: { type: 'object', required: ['label', 'tests_ran'], properties: { label: str, tests_ran: { type: 'integer', minimum: 1 } } } } } } } }, session_kind: str, artifacts_consumed: { ...str, description: '/' }, run: str, diff --git a/lib/result-codes-release.ts b/lib/result-codes-release.ts index e396cc9be..1ff6ce0f2 100644 --- a/lib/result-codes-release.ts +++ b/lib/result-codes-release.ts @@ -420,6 +420,11 @@ export const RELEASE_RESULT_CODES = { summary: 'write --review-loop-exception got an exception that is malformed, accepts a stop the review loop does not show (a fix_check with review_loop.fix_checks 0 or missing, or a second_pass with fewer than two passes or no deferred finding), names a head that is not the receipt head or its ancestor, or is missing from the PR body', fix: 'pass { accepted_by, date, via, reason } plus fix_check: { head, findings } or second_pass: { head, deferred: [] } with the owner’s words, the --review-loop summary that shows that stop, and the printed "Review-loop exception: …" line in the body', }, + CI_CARRY_RERUN_MALFORMED: { + anchor: 'ci-carry-rerun-malformed', + summary: 'a CI run\'s tree.json carries rerun_files or attempts in a shape gstack cannot read (rerun_files not a list of paths, attempts not a list of attempt numbers), so the run is not evidence and no lane carries from it', + fix: 'fix the workflow step that writes tree.json (rerun_files: ["test/a.test.ts", …], attempts: [1, 2]), dispatch a fresh run at the same head, then rerun gstack-ci-carry wait', + }, RECEIPT_CARRY_TREE_MISMATCH: { anchor: 'receipt-carry-tree-mismatch', summary: 'the branch was rebased or force-pushed since its /ship receipt, and HEAD\'s tree is not a tree the receipt covers, so no review saw this code', diff --git a/lib/run-evidence.ts b/lib/run-evidence.ts index 4d51dada6..1edf6a23f 100644 --- a/lib/run-evidence.ts +++ b/lib/run-evidence.ts @@ -55,7 +55,7 @@ export interface ApiRun { repository?: { full_name: string } | null; head_repository?: { full_name: string } | null; head_branch?: string | null; } export interface ApiJob { name: string; conclusion: string | null; started_at?: string | null; completed_at?: string | null } -export interface TreeRecord { run_id?: unknown; run_attempt?: unknown; event?: unknown; sha?: unknown; head_sha?: unknown; tree?: unknown; reused?: unknown; reuse?: unknown; sizing_experiment?: unknown; os?: unknown; files?: unknown; tests?: unknown; skipped_files?: unknown } +export interface TreeRecord { run_id?: unknown; run_attempt?: unknown; event?: unknown; sha?: unknown; head_sha?: unknown; tree?: unknown; reused?: unknown; reuse?: unknown; sizing_experiment?: unknown; os?: unknown; files?: unknown; tests?: unknown; skipped_files?: unknown; rerun_files?: unknown; attempts?: unknown } /** What a red run's reader needs beyond RunDeps (lib/pregate/flaky-exception.ts): every file of one artifact, and a pull request's head. */ export interface RunArtifactDeps { @@ -78,10 +78,11 @@ export interface RunDeps { export type RunRefusal = 'run_ref_invalid' | 'other_repository' | 'fork_run' | 'workflow_mismatch' | 'event_mismatch' | 'run_in_progress' | 'run_not_success' | 'job_missing' | 'job_ambiguous' | 'job_skipped' | 'job_not_success' | 'artifact_missing' | 'artifact_mismatch' - | 'artifact_reused' | 'sizing_experiment' | 'tree_mismatch' | 'newer_run_vetoes' | 'run_changed' | 'run_stale' | 'os_mismatch' | 'api_error'; + | 'artifact_reused' | 'sizing_experiment' | 'tree_mismatch' | 'newer_run_vetoes' | 'run_changed' | 'run_stale' | 'os_mismatch' | 'rerun_malformed' | 'api_error'; /** `skipped`: the files whose every test skipped on the runner (tree.json `skipped_files`, GST-1190); undefined when an older tree.json does not say. */ /** `docs_only`: a pull_request run whose gate skipped every skippable job on a change lib/docs-only.ts calls human docs only; it ran and recorded no tests (R81). */ -export interface RunEvidence { run_id: number; run_attempt: number; head_sha: string; tree: string; profile: string; event: string; finished_at: string; files: string[]; tests: number | null; skipped?: string[]; skipped_jobs?: string[]; docs_only?: true } +/** `rerun`: the files whose newest result came from a re-run attempt (tree.json `rerun_files`, gbrain#9690), with the run's `attempts`; set only when a file passed on a re-run (GST-3054). */ +export interface RunEvidence { run_id: number; run_attempt: number; head_sha: string; tree: string; profile: string; event: string; finished_at: string; files: string[]; tests: number | null; skipped?: string[]; skipped_jobs?: string[]; docs_only?: true; rerun?: string[]; attempts?: number[] } /** `fix`: the one command that clears a refusal the caller can repair, such as a pull_request run that tested a merge with an older base. */ export type RunVerdict = { ok: true; evidence: RunEvidence } | { ok: false; reason: RunRefusal; detail: string; fix?: string }; @@ -243,12 +244,16 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof if (rec.files !== undefined && (!Array.isArray(rec.files) || rec.files.some(f => typeof f !== 'string'))) return fail('artifact_mismatch', `${at}'s ${p.artifact} files is not a list of paths`); if (rec.tests !== undefined && (!Number.isInteger(rec.tests) || (rec.tests as number) < 0)) return fail('artifact_mismatch', `${at}'s ${p.artifact} tests is not a non-negative integer`); if (rec.skipped_files !== undefined && (!Array.isArray(rec.skipped_files) || rec.skipped_files.some(f => typeof f !== 'string'))) return fail('artifact_mismatch', `${at}'s ${p.artifact} skipped_files is not a list of paths`); + if (rec.rerun_files !== undefined && (!Array.isArray(rec.rerun_files) || rec.rerun_files.some(f => typeof f !== 'string'))) return fail('rerun_malformed', `${at}'s ${p.artifact} rerun_files is not a list of paths (CI_CARRY_RERUN_MALFORMED)`); + if (rec.attempts !== undefined && (!Array.isArray(rec.attempts) || rec.attempts.some(n => !Number.isInteger(n) || (n as number) < 1))) return fail('rerun_malformed', `${at}'s ${p.artifact} attempts is not a list of attempt numbers (CI_CARRY_RERUN_MALFORMED)`); const files = ((rec.files ?? []) as string[]).map(normalPath); const tests = rec.tests === undefined ? null : (rec.tests as number); const changed = runChanged(deps, id, run.run_attempt); if (changed) return fail('run_changed', changed); const skipped = rec.skipped_files === undefined ? {} : { skipped: (rec.skipped_files as string[]).map(normalPath) }; - return { ok: true, evidence: { run_id: id, run_attempt: run.run_attempt, head_sha: run.head_sha, tree: headTree, profile: o.profileName, event: run.event, finished_at: finishedAt!, files, tests, ...skipped, ...(skippedJobs.length ? { skipped_jobs: skippedJobs } : {}) } }; + const rerun = ((rec.rerun_files ?? []) as string[]).map(normalPath); + const reran = rerun.length ? { rerun, attempts: [...new Set((rec.attempts ?? []) as number[])].sort((a, b) => a - b) } : {}; + return { ok: true, evidence: { run_id: id, run_attempt: run.run_attempt, head_sha: run.head_sha, tree: headTree, profile: o.profileName, event: run.event, finished_at: finishedAt!, files, tests, ...skipped, ...(skippedJobs.length ? { skipped_jobs: skippedJobs } : {}), ...reran } }; } catch (e: any) { return fail('api_error', `reading run ${id} failed: ${String(e?.message ?? e).slice(0, 300)}`); } @@ -322,9 +327,34 @@ export function verifyFromRun(src: RunSource, ref: string, names: string[], tree return refusals.length === 1 ? { ok: false, ...refusals[0]! } : { ok: false, reason: 'no_verified_run', detail: refusals.map(r => `${r.detail} (${r.reason})`).join(' | ') }; } -/** The receipt suffix a run clear records: runs, profiles and the verification time, read back by `parseRunReceipt`. */ +/** + * The receipt suffix a run clear records: runs, profiles and the verification time, read back by `parseRunReceipt`. A run some of + * whose files passed only on a re-run adds `rerun= attempts=` (GST-3054): allowed, but never shown as a clean pass. + */ export function runReceiptFields(evidence: RunEvidence[], now: Date): string { - return `runs=${evidence.map(e => `${e.run_id}/${e.run_attempt}`).join(',')} profiles=${evidence.map(e => e.profile).join(',')} heads=${[...new Set(evidence.map(e => e.head_sha.slice(0, 12)))].join(',')} verified=${now.toISOString()}`; + return `runs=${evidence.map(e => `${e.run_id}/${e.run_attempt}`).join(',')} profiles=${evidence.map(e => e.profile).join(',')} heads=${[...new Set(evidence.map(e => e.head_sha.slice(0, 12)))].join(',')}${rerunMarker([...rerunByRun(evidence).values()])} verified=${now.toISOString()}`; +} + +/** The re-run record per carried run (`/`): the distinct re-run files and attempts across every profile that read that run, so two profiles on one run count each file once. */ +export function rerunByRun(evidence: Array>): Map { + const runs = new Map; attempts: Set }>(); + for (const e of evidence) { + if (!e.rerun?.length) continue; + const key = `${e.run_id}/${e.run_attempt}`; + const run = runs.get(key) ?? { files: new Set(), attempts: new Set() }; + for (const f of e.rerun) run.files.add(f); + for (const n of e.attempts ?? []) run.attempts.add(n); + runs.set(key, run); + } + return new Map([...runs].map(([key, r]) => [key, { files: r.files.size, attempts: [...r.attempts].sort((a, b) => a - b) }])); +} + +/** ` rerun= attempts=` over the runs with re-run files, each run counted once, or '' when every run passed first time (GST-3054). */ +export function rerunMarker(runs: Array<{ files: number; attempts: number[] }>): string { + const reran = runs.filter(r => r.files > 0); + if (!reran.length) return ''; + const attempts = [...new Set(reran.flatMap(r => r.attempts))].sort((a, b) => a - b); + return ` rerun=${reran.reduce((n, r) => n + r.files, 0)} attempts=${attempts.length ? attempts.join(',') : 'unrecorded'}`; } export function parseRunReceipt(receipt: string): { runs: string[]; profiles: string[]; tree: string | null } | null { diff --git a/lib/ship-receipt.ts b/lib/ship-receipt.ts index a65895b27..4c8aee226 100644 --- a/lib/ship-receipt.ts +++ b/lib/ship-receipt.ts @@ -16,6 +16,7 @@ import { matchesAny } from './ship-policy'; import { ruledNoteText } from './ruled-notes'; import { gateDispositionText } from './gate-dispositions'; import { laneExceptionBodyLines } from './test-lane-exception'; +import { rerunMarker } from './run-evidence'; import { stampFileKind, versionStampOnly } from './pregate/stamp-only'; import { stampOnlyLine, type BranchSide } from './stamp-only'; @@ -207,7 +208,15 @@ export function receiptSummaryLine(r: ShipReceipt): string { if (loop) parts.push(`passes=${loop.passes} blocking_found=${loop.blocking_found} deferred=${loop.deferred}` + (loop.fix_checks ? ` fix_checks=${loop.fix_checks}` : '') + (loop.rounds !== undefined ? ` rounds=${loop.rounds}` : '') + (loop.dispatches ? ` dispatches=${Object.values(loop.dispatches).reduce((a, c) => a + c, 0)}` : '')); if (r.carry && typeof r.carry.from === 'string') parts.push(`carried_from=${r.carry.from.slice(0, 12)}`); - if (Array.isArray(r.ci_carried) && r.ci_carried.length) parts.push(`ci_carried=${r.ci_carried.map(c => `${c.lane}:${c.run}/${c.attempt}`).join(',')}`); + if (Array.isArray(r.ci_carried) && r.ci_carried.length) { + const runs = new Map(); + for (const c of r.ci_carried) { + const key = `${c.run}/${c.attempt}`; + const seen = runs.get(key) ?? { files: 0, attempts: [] }; + runs.set(key, { files: Math.max(seen.files, typeof c.rerun === 'number' ? c.rerun : 0), attempts: [...seen.attempts, ...(Array.isArray(c.attempts) ? c.attempts : [])] }); + } + parts.push(`ci_carried=${r.ci_carried.map(c => `${c.lane}:${c.run}/${c.attempt}`).join(',')}${rerunMarker([...runs.values()])}`); + } if (r.base_red && typeof r.base_red.base === 'string') parts.push(`base_red=${r.base_red.base.slice(0, 12)}`); return `SHIP_RECEIPT: ${oneLine(parts.join(' '))}`; } diff --git a/test/ci-carry-rerun.test.ts b/test/ci-carry-rerun.test.ts new file mode 100644 index 000000000..bf82b7cb1 --- /dev/null +++ b/test/ci-carry-rerun.test.ts @@ -0,0 +1,98 @@ +/** + * GST-3054: a carried CI run whose tree.json says some files passed only on a re-run attempt (gbrain#9690's + * `rerun_files` and `attempts`) still carries, but every carry receipt and the SHIP_RECEIPT line say so + * (`rerun= attempts=`), and a malformed record fails closed. The dependencies are fakes; nothing reaches GitHub. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { findCarry } from '../lib/ci-carry'; +import { RESULT_CODES } from '../lib/result-codes'; +import { receiptSummaryLine, validateReceipt } from '../lib/ship-receipt'; +import { rerunByRun, runReceiptFields, type ApiRun, type RunDeps, type RunProfile, type RunSource, type TreeRecord } from '../lib/run-evidence'; + +const REPO = 'garrytan/gstack'; +const HEAD = 'a'.repeat(40); +const TREE = 'b'.repeat(40); +const NOW = new Date('2026-11-02T12:00:00Z'); +const PROFILE: RunProfile = { workflow: 'free-tests.yml', events: ['workflow_dispatch'], jobs: ['free-tests'], artifact: 'tested-tree', os: 'Linux' }; +const FIXTURES = path.join(import.meta.dir, 'fixtures', 'ci-carry-rerun'); +const fixture = (name: string): TreeRecord => JSON.parse(fs.readFileSync(path.join(FIXTURES, name), 'utf8')); +const INVENTORY = ['test/a.test.ts', 'test/b.test.ts', 'test/c.test.ts']; + +function source(rec: TreeRecord): RunSource { + const r: ApiRun = { id: rec.run_id as number, run_attempt: rec.run_attempt as number, status: 'completed', conclusion: 'success', event: 'workflow_dispatch', head_sha: HEAD, + path: '.github/workflows/free-tests.yml', created_at: '2026-11-02T10:00:00Z', updated_at: '2026-11-02T11:30:00Z', repository: { full_name: REPO }, head_repository: { full_name: REPO } }; + const deps: RunDeps = { + run: () => r, + runsForHead: (_wf, sha) => (sha === HEAD ? [r] : []), + jobs: () => [{ name: 'free-tests', conclusion: 'success', completed_at: '2026-11-02T11:30:00Z' }], + artifact: () => rec, + commitTree: sha => (sha === HEAD ? TREE : null), + }; + return { repo: REPO, deps, profiles: { 'ci-reuse/free-tests': PROFILE }, now: NOW }; +} +const carry = (rec: TreeRecord, inventory = true) => findCarry(source(rec), { profiles: ['ci-reuse/free-tests'], head: HEAD, tree: TREE, wtree: () => TREE, ...(inventory ? { inventory: () => INVENTORY } : {}) }); +const receipt = (ci_carried: unknown) => ({ schema_version: 1 as const, head: HEAD, base: 'main', ci_carried } as Parameters[0]); + +describe('a carried run that passed only after a re-run (GST-3054)', () => { + // Value: protects=a tree.json without rerun_files carries exactly as before; fails_when=the marker appears on a clean first-attempt run + test('a tree.json without rerun_files or attempts carries with no rerun marker', () => { + for (const inventory of [true, false]) { + const v = carry(fixture('no-fields.tree.json'), inventory); + expect(v.ok).toBe(true); + if (!v.ok) return; + expect(v.receipt).not.toMatch(/ rerun=| attempts=/); + expect(v.runs[0]!.rerun).toBeUndefined(); + expect(v.runs[0]!.attempts).toBeUndefined(); + } + const empty = carry({ ...fixture('no-fields.tree.json'), rerun_files: [], attempts: [1] }); + expect(empty.ok).toBe(true); + expect(empty.ok ? empty.receipt : '').not.toContain('rerun='); + const line = receiptSummaryLine(receipt([{ lane: 'free', run: 4, attempt: 1, tree: TREE }])); + expect(line).toContain('ci_carried=free:4/1'); + expect(line).not.toMatch(/ rerun=| attempts=/); + }); + + // Value: protects=a re-run pass is marked, never presented as clean; fails_when=rerun_files is ignored again or the carry refuses it + test('rerun_files carries, and the carry receipt names the re-run count and the attempts', () => { + for (const inventory of [true, false]) { + const v = carry(fixture('rerun.tree.json'), inventory); + expect(v.ok).toBe(true); + if (!v.ok) return; + expect(v.receipt).toContain(' runs=5/2 '); + expect(v.receipt).toContain(' rerun=2 attempts=1,2 '); + expect([v.runs[0]!.rerun, v.runs[0]!.attempts]).toEqual([['test/b.test.ts', 'test/c.test.ts'], [1, 2]]); + } + }); + + // Value: protects=the SHIP_RECEIPT line ready and land print shows the marker; fails_when=the receipt drops it or it fails schema validation + test('the receipt records rerun and attempts per carried run, and SHIP_RECEIPT shows them after ci_carried', () => { + const r = receipt([{ lane: 'free', run: 5, attempt: 2, tree: TREE, rerun: 2, attempts: [1, 2] }, { lane: 'lint', run: 5, attempt: 2, tree: TREE, rerun: 2, attempts: [1, 2] }]); + expect(validateReceipt(r)).toEqual([]); + expect(receiptSummaryLine(r)).toContain('ci_carried=free:5/2,lint:5/2 rerun=2 attempts=1,2'); + expect(validateReceipt(receipt([{ lane: 'free', run: 5, attempt: 2, tree: TREE, rerun: '2', attempts: 'x' }])).length).toBeGreaterThan(0); + }); + + // Value: protects=an unmarked lane on the same run never hides the marker, and two profiles on one run count each file once; fails_when=the last entry or profile overwrites or double-counts + test('lanes and profiles that share a run keep the marker and count each re-run file once', () => { + expect(receiptSummaryLine(receipt([{ lane: 'free', run: 5, attempt: 2, tree: TREE, rerun: 1, attempts: [1, 2] }, { lane: 'lint', run: 5, attempt: 2, tree: TREE }]))).toContain('ci_carried=free:5/2,lint:5/2 rerun=1 attempts=1,2'); + const run = { run_id: 5, run_attempt: 2, head_sha: HEAD, tree: TREE, event: 'workflow_dispatch', finished_at: '2026-11-02T11:30:00Z', files: INVENTORY, tests: 3, rerun: ['test/b.test.ts'], attempts: [2, 1, 2] }; + expect(runReceiptFields([{ ...run, profile: 'one' }, { ...run, profile: 'two' }], NOW)).toContain(' rerun=1 attempts=1,2 '); + expect([...rerunByRun([run, { ...run, rerun: ['test/c.test.ts'] }])]).toEqual([['5/2', { files: 2, attempts: [1, 2] }]]); + }); + + // Value: protects=fail closed on a malformed re-run record; fails_when=a non-array or non-string rerun_files, or bad attempts, carries + test('a malformed rerun_files or attempts refuses with CI_CARRY_RERUN_MALFORMED', () => { + const bad: Array> = [{}, { rerun_files: [1] }, { rerun_files: ['test/b.test.ts'], attempts: '1,2' }, { rerun_files: ['test/b.test.ts'], attempts: [0, 2] }, { rerun_files: ['test/b.test.ts'], attempts: ['2'] }]; + for (const over of bad) { + const v = carry({ ...fixture('malformed.tree.json'), ...over }); + expect(v.ok).toBe(false); + if (v.ok) return; + expect(v.reason).toBe('no_verified_run'); + expect(v.detail).toContain('rerun_malformed'); + expect(v.detail).toContain('(CI_CARRY_RERUN_MALFORMED)'); + } + expect(RESULT_CODES.CI_CARRY_RERUN_MALFORMED.anchor).toBe('ci-carry-rerun-malformed'); + }); +}); diff --git a/test/evidence-reuse.test.ts b/test/evidence-reuse.test.ts index 4011dcd70..273adc821 100644 --- a/test/evidence-reuse.test.ts +++ b/test/evidence-reuse.test.ts @@ -128,6 +128,18 @@ describe('gstack-evidence record-ci and carried (A4)', () => { expect(JSON.parse(run(['carried', '--json']).stdout)).toEqual([]); }); + // Value: protects=GST-3054, a carried run that passed only on a re-run reaches the receipt's ci_carried entry marked; fails_when=the + // ledger's rerun record is dropped on the way to the receipt, or a forged one with bad attempts is copied into it + test('carried copies a CI row\'s re-run record into its ci_carried entry, and drops a malformed one', () => { + expect(run(['run', '--label', 'unit', '--', lane(2)]).status).toBe(0); + fs.appendFileSync(ledger(), JSON.stringify({ ...ciRow(), rerun: { '41/2': { files: 290, attempts: [1, 2] } } }) + '\n'); + expect(JSON.parse(run(['carried', '--base', 'main', '--json']).stdout)).toEqual([{ lane: 'free', run: 41, attempt: 2, tree: wtree(), rerun: 290, attempts: [1, 2] }]); + expect(run(['carried', '--base', 'main']).stdout).toContain('CI_CARRIED: free run=41/2 tree='); + expect(run(['carried', '--base', 'main']).stdout).toContain(' rerun=290 attempts=1,2'); + fs.appendFileSync(ledger(), JSON.stringify({ ...ciRow(), rerun: { '41/2': { files: 290, attempts: '1,2' } } }) + '\n'); + expect(JSON.parse(run(['carried', '--base', 'main', '--json']).stdout)).toEqual([{ lane: 'free', run: 41, attempt: 2, tree: wtree() }]); + }); + // Value: protects=record-ci finds and verifies the CI run itself, so no caller supplies run ids, a test count or a tree (/ship pass 1 red-team#1); // fails_when=the forged call from that finding writes a FRESH carried row; seam=no origin, so the base's ship_lanes_from is unreadable test('record-ci takes no run facts from its caller and records nothing without a verified run', () => { diff --git a/test/fixtures/ci-carry-rerun/malformed.tree.json b/test/fixtures/ci-carry-rerun/malformed.tree.json new file mode 100644 index 000000000..11d69c3c9 --- /dev/null +++ b/test/fixtures/ci-carry-rerun/malformed.tree.json @@ -0,0 +1,19 @@ +{ + "run_id": 6, + "run_attempt": 2, + "event": "workflow_dispatch", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "tree": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "files": [ + "test/a.test.ts", + "test/b.test.ts", + "test/c.test.ts" + ], + "tests": 60, + "os": "Linux", + "rerun_files": "test/b.test.ts", + "attempts": [ + 1, + 2 + ] +} diff --git a/test/fixtures/ci-carry-rerun/no-fields.tree.json b/test/fixtures/ci-carry-rerun/no-fields.tree.json new file mode 100644 index 000000000..08b4760b5 --- /dev/null +++ b/test/fixtures/ci-carry-rerun/no-fields.tree.json @@ -0,0 +1,14 @@ +{ + "run_id": 4, + "run_attempt": 1, + "event": "workflow_dispatch", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "tree": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "files": [ + "test/a.test.ts", + "test/b.test.ts", + "test/c.test.ts" + ], + "tests": 60, + "os": "Linux" +} diff --git a/test/fixtures/ci-carry-rerun/rerun.tree.json b/test/fixtures/ci-carry-rerun/rerun.tree.json new file mode 100644 index 000000000..52bf1091c --- /dev/null +++ b/test/fixtures/ci-carry-rerun/rerun.tree.json @@ -0,0 +1,22 @@ +{ + "run_id": 5, + "run_attempt": 2, + "event": "workflow_dispatch", + "sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "tree": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "files": [ + "test/a.test.ts", + "test/b.test.ts", + "test/c.test.ts" + ], + "tests": 60, + "os": "Linux", + "rerun_files": [ + "test/b.test.ts", + "test/c.test.ts" + ], + "attempts": [ + 1, + 2 + ] +} diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index e8a831859..f0ba12010 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -51,6 +51,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/ceo-split-collection-3638.json': ['test/ceo-split-collection.test.ts'], 'test/fixtures/ceo-split-wording-variants.json': ['test/ceo-split-collection.test.ts'], 'test/fixtures/ceo-stale-fill-decision/**': ['test/ceo-stale-fill-decision.test.ts'], + 'test/fixtures/ci-carry-rerun/**': ['test/ci-carry-rerun.test.ts'], 'test/fixtures/ci-gate/**': ['test/gstack-ci-gate.test.ts'], 'test/fixtures/claude-agent-payload-2.1.292.json': ['test/autoplan-publication-payload.test.ts', 'test/autoplan-schema-canary-compare.test.ts'], 'test/fixtures/claude-native-journal-roots-2.1.284.json': ['test/autoplan-publication-hook.test.ts', 'test/plan-count-transcript.test.ts'],