diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 82650dce9..975b2e1e1 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -338,6 +338,7 @@ gstack_result_anchor() { PROSE_LINT_FOUND) printf '%s' 'prose-lint-found' ;; RECEIPT_BLOCKER_OPEN) printf '%s' 'receipt-blocker-open' ;; REVIEW_LOOP_EXCEPTION_INVALID) printf '%s' 'review-loop-exception-invalid' ;; + CI_CARRY_TREE_LOG_UNBOUND) printf '%s' 'ci-carry-tree-log-unbound' ;; CI_CARRY_RERUN_MALFORMED) printf '%s' 'ci-carry-rerun-malformed' ;; RECEIPT_CARRY_TREE_MISMATCH) printf '%s' 'receipt-carry-tree-mismatch' ;; RECEIPT_CARRY_PRODUCT_CHANGED) printf '%s' 'receipt-carry-product-changed' ;; diff --git a/bin/gstack-evidence b/bin/gstack-evidence index 02dcb3fa1..1aeee28b7 100755 --- a/bin/gstack-evidence +++ b/bin/gstack-evidence @@ -151,6 +151,8 @@ interface EvidenceRecord { exception?: BoundLaneException; /** Per carried run (`/`) whose tree.json lists files that passed only on a re-run: how many, and the run's attempts (record-ci, GST-3054). */ rerun?: Record; + /** Per carried run (`/`) whose tree.json came from its job log because it held no artifact (HARN-118). */ + tree_source?: Record; } /** Bytes of streamed output kept for the tests_ran parse (the 2MB log would truncate the summary). */ @@ -716,7 +718,7 @@ function reused(file: string, label: string, cmdSha: string, maxAgeHours: number /** record-ci's record line, the one lib/ci-carry.ts recordCiResult reads. */ const recordedCiLine = (r: EvidenceRecord): string => - `gstack-evidence: recorded label=${r.label} exit=0 tests_ran=${r.tests_ran}${r.tests_unit ? ` unit=${r.tests_unit}` : ""} log=- source=ci runs=${r.runs!.join(",")}${rerunMarker(Object.values(r.rerun ?? {}))} policy=${r.policy}${r.skipped_here ? ` skipped_here=${r.skipped_here.files.length} tests_ran_here=${r.skipped_here.tests_ran}${r.skipped_here.not_owed ? ` not_owed_here=${r.skipped_here.not_owed.length}` : ""}` : ""}`; + `gstack-evidence: recorded label=${r.label} exit=0 tests_ran=${r.tests_ran}${r.tests_unit ? ` unit=${r.tests_unit}` : ""} log=- source=ci runs=${r.runs!.join(",")}${rerunMarker(Object.values(r.rerun ?? {}))}${Object.keys(r.tree_source ?? {}).length ? " tree_source=job-log" : ""} policy=${r.policy}${r.skipped_here ? ` skipped_here=${r.skipped_here.files.length} tests_ran_here=${r.skipped_here.tests_ran}${r.skipped_here.not_owed ? ` not_owed_here=${r.skipped_here.not_owed.length}` : ""}` : ""}`; /** * record-ci's reuse (GST-1795): the label's newest ledger row stands only when it is the CI row this record-ci would write again @@ -806,7 +808,9 @@ async function cmdRecordCi(argv: string[]): Promise { const runs = v.runs.map((r) => `${r.run_id}/${r.run_attempt}`); const reran = rerunByRun(v.runs); const rerun = reran.size ? Object.fromEntries(reran) : undefined; - const prior = v.excepted?.length ? null : reusableCiRow(label, { commit: head, wtree: tree, cmd_sha256: sha256(command), runs, profiles: m.profiles, policy: loaded.label, jobs: m.jobs, tests_ran: v.tests, tests_unit: v.unit, not_owed: v.notOwed, rerun }, v.remainder ?? [], m.per_file); + const logged = v.runs.filter((r) => r.tree_source === "job-log"); + const tree_source = logged.length ? Object.fromEntries(logged.map((r) => [`${r.run_id}/${r.run_attempt}`, "job-log" as const])) : undefined; + const prior = v.excepted?.length ? null : reusableCiRow(label, { commit: head, wtree: tree, cmd_sha256: sha256(command), runs, profiles: m.profiles, policy: loaded.label, jobs: m.jobs, tests_ran: v.tests, tests_unit: v.unit, not_owed: v.notOwed, rerun, tree_source }, v.remainder ?? [], m.per_file); if (prior) { console.error(recordedCiLine(prior)); if (prior.not_owed) console.error(`gstack-evidence: not owed: ${prior.not_owed}`); @@ -871,7 +875,7 @@ async function cmdRecordCi(argv: string[]): Promise { } const rec: EvidenceRecord = { ts: new Date(v.runs.map((r) => r.finished_at).sort()[0]!).toISOString(), label, command, cmd_sha256: sha256(command), exit: 0, duration_s: 0, - commit: head, tree, dirty: false, wtree: tree, tests_ran: testsRan, source: "ci", runs, profiles: m.profiles, policy: loaded.label, ...(m.jobs ? { jobs: m.jobs } : {}), ...(v.unit ? { tests_unit: v.unit } : {}), ...(v.notOwed ? { not_owed: v.notOwed } : {}), ...(here ? { skipped_here: here } : {}), ...(exception ? { exception } : {}), ...(rerun ? { rerun } : {}), + commit: head, tree, dirty: false, wtree: tree, tests_ran: testsRan, source: "ci", runs, profiles: m.profiles, policy: loaded.label, ...(m.jobs ? { jobs: m.jobs } : {}), ...(v.unit ? { tests_unit: v.unit } : {}), ...(v.notOwed ? { not_owed: v.notOwed } : {}), ...(here ? { skipped_here: here } : {}), ...(exception ? { exception } : {}), ...(rerun ? { rerun } : {}), ...(tree_source ? { tree_source } : {}), }; appendJsonl(paths.file, rec, { mode: 0o600 }); console.error(recordedCiLine(rec)); @@ -905,10 +909,10 @@ function cmdCarried(argv: string[]): number { const [run, attempt] = ref.split("/").map(Number); const re = r.rerun?.[ref]; const reran = re && Number.isInteger(re.files) && re.files > 0 && Array.isArray(re.attempts) && re.attempts.every((n) => Number.isInteger(n) && n > 0) ? { rerun: re.files, attempts: re.attempts } : {}; - return { lane: r.label, run, attempt, tree: r.wtree!, ...reran, ...(r.exception ? { exception: r.exception } : {}) }; + return { lane: r.label, run, attempt, tree: r.wtree!, ...reran, ...(r.tree_source?.[ref] === "job-log" ? { tree_source: "job-log" as const } : {}), ...(r.exception ? { exception: r.exception } : {}) }; })); if (argv.includes("--json")) console.log(JSON.stringify(carried)); - else for (const c of carried) console.log(`CI_CARRIED: ${c.lane} run=${c.run}/${c.attempt} tree=${c.tree.slice(0, 12)}${"rerun" in c ? rerunMarker([{ files: c.rerun!, attempts: c.attempts! }]) : ""}`); + else for (const c of carried) console.log(`CI_CARRIED: ${c.lane} run=${c.run}/${c.attempt} tree=${c.tree.slice(0, 12)}${"rerun" in c ? rerunMarker([{ files: c.rerun!, attempts: c.attempts! }]) : ""}${"tree_source" in c ? " tree_source=job-log" : ""}`); return 0; } diff --git a/changelog.d/ci-results-count-when-artifact-storage-is-full.md b/changelog.d/ci-results-count-when-artifact-storage-is-full.md new file mode 100644 index 000000000..28c2adc1d --- /dev/null +++ b/changelog.d/ci-results-count-when-artifact-storage-is-full.md @@ -0,0 +1,4 @@ +--- +important: blocker +--- +- **A green CI run still counts when the repository's artifact storage is full.** When GitHub refuses `upload-artifact` with "Artifact storage quota has been hit", the run had no `tree.json` and `accept`, `record-ci`, `gstack-ci-carry` and the tests stage refused it as `artifact_missing`. A run profile can now set `tree_log_step` to the step that prints `GSTACK_TREE_JSON `, and gstack reads that one line from the job's log, only when the run has no artifact, only when the runner stamped it inside that step, a second or more clear of the steps before and after it, and with every `tree.json` check unchanged. A line printed anywhere else, a test's included, refuses with `CI_CARRY_TREE_LOG_UNBOUND`. A run carried this way says `tree_source=job-log` on its `CI_CARRY:`, `LANE:` and `SHIP_RECEIPT:` lines and in the receipt. Profiles without the key behave exactly as before. diff --git a/docs/pregate.md b/docs/pregate.md index 2dfc4058a..488063988 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -105,7 +105,7 @@ the byte-pinned `/ship` renders in `test/fixtures/golden/`). | `stall_timeout_s` | seconds a test command may print nothing while using under a tenth of one CPU before its process group is killed and the check is `incomplete` (`PREGATE_STALLED`); default 300, `0` disables; `--stall-timeout` overrides; the `--remote` gate is exempt (see [supervision](#supervision)) | | `timeout_s` | seconds each runner process may take (the unit-file group, each serial file, each guard without its own `timeout_s`, and the remote gate's deadline) when `--timeout` is not passed; 1 to 3600, default 600. `gstack-ship-preflight` forwards its own `--timeout`; without it, and in the `finish` fallback, this is how a repo with long lanes sets theirs. The whole pregate under `gstack-ship-preflight` has a separate budget, `--step-timeout ` (1 to 86400, default 3600): a pregate stopped there prints `PREGATE: time limit hit` and a STOP naming the way out, push and let CI's full run at the tree carry the tests ([CI carry](#ci-carry)), or a larger `--step-timeout` | | `remote_gate` | the repo's remote test gate command; under 8 cores the local test tier becomes a `requires-remote` obligation naming it, and `--remote` runs it (see [remote gate](#remote-gate)) | -| `run_profiles` | a map from each profile name to `{ workflow, events, jobs, artifact, max_age_hours?, os?, skippable? }`: the GitHub Actions runs `clear --run` verifies as evidence on the identical tree; a profile named after a lane (`free-tests.yml/macos-named-regressions`) clears that lane (see [run evidence](#run-evidence)). `--from-run` and the reuse job verify runs against them too ([from-run](#from-run), [CI reuse](#ci-reuse)) | +| `run_profiles` | a map from each profile name to `{ workflow, events, jobs, artifact, max_age_hours?, os?, skippable?, tree_log_step? }`: the GitHub Actions runs `clear --run` verifies as evidence on the identical tree; a profile named after a lane (`free-tests.yml/macos-named-regressions`) clears that lane (see [run evidence](#run-evidence)). `--from-run` and the reuse job verify runs against them too ([from-run](#from-run), [CI reuse](#ci-reuse)) | | `remote_gate_runs` | `{ profiles, inventory, skipped? }`: the run profiles that together clear `pregate.json/remote_gate`, the command that prints the remote gate's test files, one per line, which their runs must cover, and an optional command that prints which selected files the gate itself never runs (see [run evidence](#run-evidence)) | | `ship_lanes_from` | maps each exact /ship lane command to `{ profiles, inventory }` or `{ profiles, jobs }`, with optional `aliases` (and, on an inventory entry, an optional `per_file` runner for files CI skipped entirely, see [skipped files](#skipped-files)): the run profiles whose verified runs at HEAD's clean tree carry that lane instead of running it, and either the command listing its test files, which their runs must cover, or the CI jobs that must have succeeded (see [CI carry](#ci-carry)); read from the base branch only | | `allowlist` | `[{ finding, paths?, pattern?, reason }]`: benign findings the `secrets` check accepts and names as allowlisted, by gstack-redact finding id or non-secret category, path globs and a pattern anchored at the finding; never `env.value` (see [allowlist](#allowlist)) | @@ -843,6 +843,47 @@ its own job: `run_id`, `run_attempt`, `event`, `sha` (or `head_sha`), run executed), `tests` (the count) and, with `os`, the runner OS or a list of every OS the run's test jobs ran on (the profile's `os` must be one of them). +**When artifact storage is full: `tree_log_step` (HARN-118).** A +private repository whose GitHub Actions storage quota is spent still runs its +jobs green, but `upload-artifact` fails with "Artifact storage quota has been +hit", so the run holds no `tree.json` and every carry refuses it as +`artifact_missing`. A profile that sets `tree_log_step` to the name of one step +in its job lets gstack read `tree.json` from that job's log instead, only when +the run holds no unexpired artifact by the profile's name at all (an artifact +always wins). The step prints one line, after the test step, a second after +that step ends and a second before the next step starts: + +```yaml +- name: Print tree.json + run: sleep 1 && echo "GSTACK_TREE_JSON checks-tree $(base64 -w0 tree.json)" && sleep 2 +``` + +gstack reads the log with `gh api --allow-escape-sequences +repos//actions/jobs//logs` for every profile job at the attempt +that ran, and counts lines matching +`^\S+Z GSTACK_TREE_JSON (\S+) ([A-Za-z0-9+/]+={0,2})\r?$` that name the profile's +artifact. Exactly one must exist across those jobs; zero, two or more refuse as +`artifact_missing` with the count. The first field of every log line is the +timestamp the runner stamps on output as it arrives, which nothing a step +prints can set, so the line binds to the named step by that timestamp: inside +the step's start and finish (as the jobs API reports them, to the second), at +least one second after every earlier step that ran ended and one second before +every later step started, in a job and step that succeeded. A step's output, +background processes included, stays in that step: GitHub holds the step open +until its output closes. A line a test prints, even a well-formed one with a +real `tree.json`, is either a second line or a line outside the step, and +refuses +([`CI_CARRY_TREE_LOG_UNBOUND`](troubleshooting.md#ci-carry-tree-log-unbound)). +The decoded `tree.json` is capped at 8 MiB like an artifact, and every check +below runs on it unchanged. The step prints the `tree.json` the job wrote, so +it trusts that file exactly as much as the upload step does. The carry then says where `tree.json` came from: +`tree_source=job-log` on the `CI_CARRY:`, `LANE:` and `CI_CARRIED:` lines, and +`tree_source: "job-log"` in the receipt's `ci_carried` entry with +`tree_source=job-log` after `ci_carried=` on the `SHIP_RECEIPT:` line. A profile +without the key never reads a log. An older gstack refuses the key as unknown, +so a repository that sets it raises `min_gstack_version` to the release that +ships it. A release asset is no substitute: nothing binds it to the run. + **What `clear --run` checks** (`lib/run-evidence.ts`), for every run, online, once: @@ -1356,7 +1397,9 @@ the same run. `gstack-ship-receipt write` records the carried runs as them (`ci_carried=:/`). A run whose `tree.json` lists `rerun_files` adds `rerun` and `attempts` to its entries and `rerun= attempts=` after `ci_carried=`, as its `CI_CARRY:` line does -([ship-policy.md](ship-policy.md#ci-carry-rerun)). Each `record-ci` gets the base +([ship-policy.md](ship-policy.md#ci-carry-rerun)). A run whose `tree.json` came +from its job log adds `tree_source: "job-log"` to its entries and +`tree_source=job-log` after them ([tree_log_step](#tree-log-step)). Each `record-ci` gets the base policy's [`ci_record_timeout_s`](ship-policy.md#ci-record-timeout) (600 s by default, 3600 s at most), and accept prints it as `CI_RECORD_TIMEOUT:`. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e4bff3d3a..36e14b707 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7506,6 +7506,26 @@ pass the `gstack-review-scope summary` file as `--review-loop`, put the printed line in the PR body, and write again ([ship-policy.md](ship-policy.md#ship-receipt)). + +### `CI_CARRY_TREE_LOG_UNBOUND` + +**Meaning.** A CI run holds no `tree.json` artifact (its repository's artifact +storage is full), its run profile sets `tree_log_step`, and its job log has +exactly one `GSTACK_TREE_JSON` line for the profile's artifact, but gstack +cannot prove that line came from the step `tree_log_step` names. The runner's +timestamp on the line falls outside that step, less than a second after an +earlier step (the test step) ended, or less than a second before a later step +started; or the job has no step by that name, has +two, or the step or job did not succeed; or a step before or after it has no start +and finish time. A line printed anywhere else, by a test for one, is never +evidence, so the run is refused (`tree_log_unbound` in the `CI_CARRY: none` +or `no_verified_run` detail) and the lane runs here instead +([pregate.md](pregate.md#tree-log-step)). + +**Fix.** Print the line only from the named step, after the test step, with +`sleep 1` before it and `sleep 2` after it, and make sure no test prints `GSTACK_TREE_JSON`. Then +dispatch a fresh run at the same head and rerun `gstack-ci-carry wait`. + ### `CI_CARRY_RERUN_MALFORMED` diff --git a/lib/ci-carry.ts b/lib/ci-carry.ts index 15a3c5d1f..5a83ffd0e 100644 --- a/lib/ci-carry.ts +++ b/lib/ci-carry.ts @@ -83,7 +83,7 @@ function newestVerified(src: RunSource, name: string, profile: RunProfile, heads for (const run of listed) { const v = verifyRun(src.deps, { repo: src.repo, runRef: String(run.id), profileName: name, profile, vetoHeads: heads, now: src.now, unchanged: src.unchanged, docsOnly }); if (v.ok) return v; - refusals.push(`run ${run.id}: ${v.reason}${v.reason === 'rerun_malformed' ? ` (${v.detail})` : ''}${v.fix ? ` (the pull request's base moved after the run; fix: ${v.fix})` : ''}`); + refusals.push(`run ${run.id}: ${v.reason}${v.reason === 'rerun_malformed' || v.reason === 'tree_log_unbound' || (v.reason === 'artifact_missing' && profile.tree_log_step) ? ` (${v.detail})` : ''}${v.fix ? ` (the pull request's base moved after the run; fix: ${v.fix})` : ''}`); } return no('no_verified_run', `${name}: ${refusals.join('; ')}`); } @@ -237,7 +237,7 @@ export function importedTestLines(cwd: string, missing: string[], candidates: st * not-owed reason, or the last CARRY_NONE reason. A lane the cleared remote gate stood in for (--remote-gate-stands-in, GST-1688) * reads as its receipt, its count and why the CI carry refused. */ -export function recordCiResult(stderr: string): { runs?: string; tests?: string; files?: boolean; notOwed?: string; here?: string; exception?: string; reused?: string; from?: string; none?: string; diagnosis?: string[]; remoteGate?: string; standIn?: string } { +export function recordCiResult(stderr: string): { runs?: string; treeSource?: string; tests?: string; files?: boolean; notOwed?: string; here?: string; exception?: string; reused?: string; from?: string; none?: string; diagnosis?: string[]; remoteGate?: string; standIn?: string } { const lines = stderr.split('\n'); const last = lines.findLast(x => x.startsWith('gstack-evidence: recorded label=')); const gateRow = last && / tests_ran=(\d+) log=- source=remote_gate receipt=(.+)$/.exec(last); @@ -249,12 +249,13 @@ export function recordCiResult(stderr: string): { runs?: string; tests?: string; if (at < 0) return none ? { none, ...(diagnosis.length ? { diagnosis } : {}) } : {}; const exception = lines.slice(at + 1).find(x => x.startsWith('TEST_LANE_EXCEPTION: '))?.slice('TEST_LANE_EXCEPTION: '.length); const got = / tests_ran=(\d+|n\/a)( unit=files)? log=- source=ci runs=(\S+)/.exec(lines[at]!); + const treeSource = / tree_source=(job-log)(?= |$)/.exec(lines[at]!)?.[1]; const here = / skipped_here=(\d+) tests_ran_here=(\S+)( not_owed_here=\d+)?$/.exec(lines[at]!); const notOwed = lines.slice(at + 1).find(x => x.startsWith('gstack-evidence: not owed: '))?.slice('gstack-evidence: not owed: '.length); const reused = lines.slice(at + 1).map(x => /^CI_RECORD_REUSED: .* tree=([0-9a-f]+) /.exec(x)?.[1]).find(Boolean); const fromLine = lines.slice(at + 1).find(x => x.startsWith('CI_CARRY_FROM: ')); const from = fromLine ? [...new Set([...fromLine.matchAll(/ at commit ([0-9a-f]{12})/g)].map(m => m[1]!))].join(',') : ''; - return { runs: got?.[3], tests: got?.[1], files: !!got?.[2], ...(notOwed ? { notOwed } : {}), ...(here ? { here: `skipped_here=${here[1]} file(s) tests_ran_here=${here[2]}${here[3] ?? ''}` } : {}), ...(exception ? { exception } : {}), ...(reused ? { reused } : {}), ...(from ? { from } : {}) }; + return { runs: got?.[3], tests: got?.[1], files: !!got?.[2], ...(treeSource ? { treeSource } : {}), ...(notOwed ? { notOwed } : {}), ...(here ? { here: `skipped_here=${here[1]} file(s) tests_ran_here=${here[2]}${here[3] ?? ''}` } : {}), ...(exception ? { exception } : {}), ...(reused ? { reused } : {}), ...(from ? { from } : {}) }; } /** The per_file runner command for `files`: `{files}` replaced by the paths, each anchored with ./ (Bun reads a bare relative path as a substring filter) and single-quoted for bash. */ diff --git a/lib/headless-artifacts.ts b/lib/headless-artifacts.ts index 97ca41146..737bce565 100644 --- a/lib/headless-artifacts.ts +++ b/lib/headless-artifacts.ts @@ -104,8 +104,8 @@ export interface ShipReceipt { gate?: Record; ci?: Record; spend_usd?: number | string; predecessor?: string; /** The run's calls from the call log (lib/ship-clock.ts callsSince: gstack-command calls plus keep-alives), so a 200-call run shows at the receipt; absent on older receipts. */ calls?: number; - /** GSTA-258: the lanes CI carried at this head's tree, one entry per run (gstack-evidence carried); `rerun` and `attempts` when files passed only on a re-run (GST-3054). */ - ci_carried?: Array<{ lane: string; run: number; attempt: number; tree: string; rerun?: number; attempts?: number[]; exception?: BoundLaneException }>; + /** GSTA-258: the lanes CI carried at this head's tree, one entry per run (gstack-evidence carried); `rerun` and `attempts` when files passed only on a re-run (GST-3054); `tree_source: job-log` when the run's tree.json came from its job log (HARN-118). */ + ci_carried?: Array<{ lane: string; run: number; attempt: number; tree: string; rerun?: number; attempts?: number[]; tree_source?: 'job-log'; exception?: BoundLaneException }>; session_kind?: string; artifacts_consumed?: string; run?: string; // C-stamp (C3/C5/C7): the tree receipt's verdicts and the policy provenance, all optional. thread?: string; tree?: string; gate_reuse?: string; policy?: string; queue_mode?: string; preregistration?: string; history?: string; @@ -267,7 +267,7 @@ export const ARTIFACT_SCHEMAS = { schema_version: { const: 1 }, pr: { type: 'integer' }, head: str, version: str, base: str, gated_tree: str, gate: { type: 'object' }, ci: { type: 'object' }, spend_usd: { type: ['number', 'string'] }, calls: { type: 'integer', minimum: 0 }, predecessor: str, ci_carried: { type: 'array', items: { type: 'object', required: ['lane', 'run', 'attempt', 'tree'], properties: { lane: str, run: { type: 'integer', minimum: 1 }, attempt: { type: 'integer', minimum: 1 }, tree: str, - rerun: { type: 'integer', minimum: 1 }, attempts: { type: 'array', items: { type: 'integer', minimum: 1 } }, + rerun: { type: 'integer', minimum: 1 }, attempts: { type: 'array', items: { type: 'integer', minimum: 1 } }, tree_source: { enum: ['job-log'] }, exception: { type: 'object', required: ['head', 'files', 'by', 'reason', 'at', 'approval', 'preflight'], properties: { head: str, files: strList, by: str, reason: str, at: str, approval: str, preflight: { type: 'object', required: ['label', 'tests_ran'], properties: { label: str, tests_ran: { type: 'integer', minimum: 1 } } } } } } } }, session_kind: str, artifacts_consumed: { ...str, description: '/' }, run: str, diff --git a/lib/result-codes-release.ts b/lib/result-codes-release.ts index b7a30ea79..562b45342 100644 --- a/lib/result-codes-release.ts +++ b/lib/result-codes-release.ts @@ -425,6 +425,11 @@ export const RELEASE_RESULT_CODES = { summary: 'write --review-loop-exception got an exception that is malformed, accepts a stop the review loop does not show (a fix_check with review_loop.fix_checks 0 or missing, or a second_pass with fewer than two passes or no deferred finding), names a head that is not the receipt head or its ancestor, or is missing from the PR body', fix: 'pass { accepted_by, date, via, reason } plus fix_check: { head, findings } or second_pass: { head, deferred: [] } with the owner’s words, the --review-loop summary that shows that stop, and the printed "Review-loop exception: …" line in the body', }, + CI_CARRY_TREE_LOG_UNBOUND: { + anchor: 'ci-carry-tree-log-unbound', + summary: 'a CI run with no tree.json artifact (artifact storage full) has one GSTACK_TREE_JSON line in its job log, but the runner did not stamp it inside the step the run profile names in tree_log_step, at least a second after every earlier step ended and a second before every later one started, or that step is missing, repeated or did not succeed; a line printed anywhere else (a test, for one) is never evidence', + fix: 'print the line only from the step tree_log_step names, after the test step, with sleep 1 before it and sleep 2 after it (docs/pregate.md#tree-log-step); then dispatch a fresh run at the same head and rerun gstack-ci-carry wait', + }, CI_CARRY_RERUN_MALFORMED: { anchor: 'ci-carry-rerun-malformed', summary: 'a CI run\'s tree.json carries rerun_files or attempts in a shape gstack cannot read (rerun_files not a list of paths, attempts not a list of attempt numbers), so the run is not evidence and no lane carries from it', diff --git a/lib/run-evidence.ts b/lib/run-evidence.ts index 308c65b8c..64b44e2b2 100644 --- a/lib/run-evidence.ts +++ b/lib/run-evidence.ts @@ -11,7 +11,9 @@ * the run belongs to this repository and ran for a same-repository head; * its workflow file and event are the profile's; it is the newest attempt, * completed with success; every job the profile names appears exactly once - * in that attempt and succeeded; its artifact names this run, attempt, + * in that attempt and succeeded; its artifact (or, for a profile with + * tree_log_step and a run with no artifact at all, the one tree.json line + * the runner logged inside that step; treeFromLog) names this run, attempt, * event and head (the API always answers with the newest attempt, so an * older green attempt under a newer red one is refused), is not a reuse * record, records the tree git computes for the head, and holds well-formed @@ -47,14 +49,19 @@ import { runBashSync } from './script-invocation'; * `skippable` (GST-1190): named jobs the workflow may skip by path; a skip counts only in a pull_request run whose `gate` job * succeeded, with the workflow, the declared `scripts` and any path the gate's steps name matching origin/ byte for byte. */ -export interface RunProfile { workflow: string; events: string[]; jobs: string[]; artifact: string; max_age_hours?: number; os?: string; skippable?: { jobs: string[]; gate: string; scripts: string[] } } +/** + * `tree_log_step` (HARN-118): the opt-in for a run with no unexpired artifact named `artifact` (artifact storage full): its + * tree.json is read from the one `GSTACK_TREE_JSON ` line the runner logged inside this step of a profile job. + */ +export interface RunProfile { workflow: string; events: string[]; jobs: string[]; artifact: string; max_age_hours?: number; os?: string; skippable?: { jobs: string[]; gate: string; scripts: string[] }; tree_log_step?: string } export interface RemoteGateRuns { profiles: string[]; inventory: string; skipped?: string } export interface ApiRun { id: number; run_attempt: number; status: string; conclusion: string | null; event: string; head_sha: string; path?: string; created_at?: string; run_started_at?: string; updated_at?: string; repository?: { full_name: string } | null; head_repository?: { full_name: string } | null; head_branch?: string | null; } -export interface ApiJob { name: string; conclusion: string | null; started_at?: string | null; completed_at?: string | null } +export interface ApiStep { number: number; name: string; conclusion: string | null; started_at?: string | null; completed_at?: string | null } +export interface ApiJob { id?: number; name: string; conclusion: string | null; started_at?: string | null; completed_at?: string | null; steps?: ApiStep[] } export interface TreeRecord { run_id?: unknown; run_attempt?: unknown; event?: unknown; sha?: unknown; head_sha?: unknown; tree?: unknown; reused?: unknown; reuse?: unknown; sizing_experiment?: unknown; os?: unknown; files?: unknown; tests?: unknown; skipped_files?: unknown; rerun_files?: unknown; attempts?: unknown } /** What a red run's reader needs beyond RunDeps (lib/pregate/flaky-exception.ts): every file of one artifact, and a pull request's head. */ @@ -64,6 +71,9 @@ export interface RunArtifactDeps { pull(n: number): { head_sha: string; state: string; head_repo: string | null }; } +/** What `RunDeps.artifact` throws when the run holds no unexpired artifact by that name at all: the only case a job log may stand in. */ +export class NoArtifactError extends Error {} + /** Everything verifyRun reads, injectable so every refusal has a fixture. */ export interface RunDeps { run(id: number): ApiRun; @@ -71,6 +81,8 @@ export interface RunDeps { jobs(id: number, attempt: number): ApiJob[]; /** The tree.json of the run's unexpired artifact `name`: the only one, or with one per rerun attempt of its job, the one that records run_attempt `attempt`. */ artifact(id: number, name: string, attempt: number): TreeRecord; + /** One job's whole log as the runner wrote it, each line led by the runner's own timestamp (HARN-118's fallback only). */ + jobLog?(jobId: number): string; commitTree(sha: string): string | null; /** `sha` and its ancestors, newest first, at most `max` (git rev-list): where a commit with `sha`'s exact tree is looked for (GST-3019). */ ancestors?(sha: string, max: number): string[]; @@ -78,17 +90,18 @@ export interface RunDeps { export type RunRefusal = 'run_ref_invalid' | 'other_repository' | 'fork_run' | 'workflow_mismatch' | 'event_mismatch' | 'run_in_progress' | 'run_not_success' | 'job_missing' | 'job_ambiguous' | 'job_skipped' | 'job_not_success' | 'artifact_missing' | 'artifact_mismatch' - | 'artifact_reused' | 'sizing_experiment' | 'tree_mismatch' | 'newer_run_vetoes' | 'run_changed' | 'run_stale' | 'os_mismatch' | 'rerun_malformed' | 'api_error'; + | 'artifact_reused' | 'sizing_experiment' | 'tree_mismatch' | 'newer_run_vetoes' | 'run_changed' | 'run_stale' | 'os_mismatch' | 'rerun_malformed' | 'tree_log_unbound' | 'api_error'; /** `skipped`: the files whose every test skipped on the runner (tree.json `skipped_files`, GST-1190); undefined when an older tree.json does not say. */ /** `docs_only`: a pull_request run whose gate skipped every skippable job on a change lib/docs-only.ts calls human docs only; it ran and recorded no tests (R81). */ +/** `tree_source`: `job-log` when tree.json came from the job log because the run held no artifact (HARN-118); absent for an artifact. */ /** `rerun`: the files whose newest result came from a re-run attempt (tree.json `rerun_files`, gbrain#9690), with the run's `attempts`; set only when a file passed on a re-run (GST-3054). */ -export interface RunEvidence { run_id: number; run_attempt: number; head_sha: string; tree: string; profile: string; event: string; finished_at: string; files: string[]; tests: number | null; skipped?: string[]; skipped_jobs?: string[]; docs_only?: true; rerun?: string[]; attempts?: number[] } +export interface RunEvidence { run_id: number; run_attempt: number; head_sha: string; tree: string; profile: string; event: string; finished_at: string; files: string[]; tests: number | null; skipped?: string[]; skipped_jobs?: string[]; docs_only?: true; rerun?: string[]; attempts?: number[]; tree_source?: 'job-log' } /** `fix`: the one command that clears a refusal the caller can repair, such as a pull_request run that tested a merge with an older base. */ export type RunVerdict = { ok: true; evidence: RunEvidence } | { ok: false; reason: RunRefusal; detail: string; fix?: string }; const SHA = /^[0-9a-f]{40}$/; const RUN_EVENTS = ['push', 'workflow_dispatch', 'pull_request']; -const PROFILE_KEYS = ['workflow', 'events', 'jobs', 'artifact', 'max_age_hours', 'os', 'skippable']; +const PROFILE_KEYS = ['workflow', 'events', 'jobs', 'artifact', 'max_age_hours', 'os', 'skippable', 'tree_log_step']; const WORKFLOW_FILE = /^[A-Za-z0-9_.-]+\.ya?ml$/; const NAME = /^[A-Za-z0-9_.\/-]{1,128}$/; export const DEFAULT_MAX_AGE_HOURS = 24; @@ -97,6 +110,9 @@ const MAX_ARTIFACT_BYTES = 8 * 1024 * 1024; const MAX_ARTIFACT_ATTEMPTS = 10; /** Shard logs are the whole test stream, so a red run's reader allows more than a tree.json. */ const MAX_LOG_ARTIFACT_BYTES = 64 * 1024 * 1024; +/** The one log line that carries a tree.json when a run holds no artifact (HARN-118): the runner's timestamp, the marker, the artifact name, base64. */ +const TREE_LOG_LINE = /^\S+Z GSTACK_TREE_JSON (\S+) ([A-Za-z0-9+/]+={0,2})\r?$/; +const LOG_STAMP = /^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)(?:\.(\d{1,9}))?Z$/; /** A numeric run id, or a run URL on this repository (`…/actions/runs/[/attempts/][/job/]`). */ export function parseRunRef(ref: string, repo: string): number | null { @@ -127,6 +143,7 @@ export function runProfileErrors(profiles: unknown, gateRuns: unknown): Array<{ if (typeof p.artifact !== 'string' || !NAME.test(p.artifact)) errors.push({ path: `${at}.artifact`, message: 'must be the artifact name holding tree.json' }); if ('max_age_hours' in p && (!Number.isInteger(p.max_age_hours) || (p.max_age_hours as number) < 1 || (p.max_age_hours as number) > 168)) errors.push({ path: `${at}.max_age_hours`, message: 'must be an integer from 1 to 168' }); if ('os' in p && (typeof p.os !== 'string' || !/^(Linux|macOS|Windows)$/.test(p.os))) errors.push({ path: `${at}.os`, message: 'must be Linux, macOS or Windows' }); + if ('tree_log_step' in p && (typeof p.tree_log_step !== 'string' || !p.tree_log_step.trim() || p.tree_log_step.length > 200 || /[\0\r\n]/.test(p.tree_log_step))) errors.push({ path: `${at}.tree_log_step`, message: 'must be the name of the step in a profile job that prints GSTACK_TREE_JSON , one line of at most 200 characters' }); if ('skippable' in p) { const s = p.skippable as Record | null; const names = (v: unknown) => Array.isArray(v) && v.length > 0 && v.length <= 32 && v.every(x => typeof x === 'string' && !!x.trim() && x.length <= 128); @@ -216,8 +233,15 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof } const ranIn = ownAttempt(deps, id, run.run_attempt, jobs, p.jobs.filter(n => !skippedJobs.includes(n))); let rec: TreeRecord; + let fromLog = false; try { rec = deps.artifact(id, p.artifact, ranIn); } - catch (e: any) { return fail('artifact_missing', `${at} has no readable ${p.artifact} tree.json: ${String(e?.message ?? e).slice(0, 200)}`); } + catch (e: any) { + if (!(e instanceof NoArtifactError) || !p.tree_log_step || !deps.jobLog) return fail('artifact_missing', `${at} has no readable ${p.artifact} tree.json: ${String(e?.message ?? e).slice(0, 200)}`); + const logged = treeFromLog(deps, id, ranIn, p, p.jobs.filter(n => !skippedJobs.includes(n)), at); + if (!logged.ok) return logged; + rec = logged.rec; + fromLog = true; + } if (!rec || typeof rec !== 'object') return fail('artifact_mismatch', `${at}'s ${p.artifact} tree.json is not a JSON object`); if (rec.reused === true || rec.reuse === true) return fail('artifact_reused', `${at} reused another run; a reuse is never evidence`); if (rec.sizing_experiment !== undefined) return fail('sizing_experiment', `${at} was a runner-sizing experiment (a dispatch that overrode the runner, shards or sub-shards); it is never a Free Tests result`); @@ -253,12 +277,60 @@ export function verifyRun(deps: RunDeps, o: { repo: string; runRef: string; prof const skipped = rec.skipped_files === undefined ? {} : { skipped: (rec.skipped_files as string[]).map(normalPath) }; const rerun = ((rec.rerun_files ?? []) as string[]).map(normalPath); const reran = rerun.length ? { rerun, attempts: [...new Set((rec.attempts ?? []) as number[])].sort((a, b) => a - b) } : {}; - return { ok: true, evidence: { run_id: id, run_attempt: run.run_attempt, head_sha: run.head_sha, tree: headTree, profile: o.profileName, event: run.event, finished_at: finishedAt!, files, tests, ...skipped, ...(skippedJobs.length ? { skipped_jobs: skippedJobs } : {}), ...reran } }; + return { ok: true, evidence: { run_id: id, run_attempt: run.run_attempt, head_sha: run.head_sha, tree: headTree, profile: o.profileName, event: run.event, finished_at: finishedAt!, files, tests, ...skipped, ...(skippedJobs.length ? { skipped_jobs: skippedJobs } : {}), ...reran, ...(fromLog ? { tree_source: 'job-log' as const } : {}) } }; } catch (e: any) { return fail('api_error', `reading run ${id} failed: ${String(e?.message ?? e).slice(0, 300)}`); } } +/** + * The tree.json run `id` logged at `attempt` for a profile that opted in with `tree_log_step`, read only when the run holds no + * artifact (HARN-118). Exactly one TREE_LOG_LINE naming the profile's artifact across the profile's `names` jobs counts, and it + * binds only by the runner's own timestamp, which a step's output cannot set: inside the named step (one step of that name, + * a success), at least a second after every earlier step that ran ended and a second before every later one started, + * since the API times steps only to the second. A test + * that prints a well-formed line therefore adds a second line, or stands alone outside the step, and refuses either way. + * The decoded tree.json is capped at MAX_ARTIFACT_BYTES; verifyRun then checks it exactly as it checks an artifact. + */ +function treeFromLog(deps: RunDeps, id: number, attempt: number, p: RunProfile, names: string[], at: string): { ok: true; rec: TreeRecord } | { ok: false; reason: RunRefusal; detail: string } { + const missing = (detail: string) => ({ ok: false as const, reason: 'artifact_missing' as const, detail: `${at} has no unexpired ${p.artifact} artifact and ${detail}` }); + const hits: Array<{ job: ApiJob; stamp: string; data: string }> = []; + for (const job of deps.jobs(id, attempt).filter(j => names.includes(j.name))) { + if (!Number.isSafeInteger(job.id)) return missing(`its ${job.name} job has no numeric id, so its log cannot be read`); + let text: string; + try { text = deps.jobLog!(job.id!); } catch (e: any) { return missing(`its ${job.name} job's log cannot be read: ${String(e?.message ?? e).slice(0, 200)}`); } + for (const raw of text.replace(/^\uFEFF/, '').split('\n')) { + const m = TREE_LOG_LINE.exec(raw); + if (m && m[1] === p.artifact) hits.push({ job, stamp: raw.slice(0, raw.indexOf(' ')), data: m[2]! }); + } + } + if (hits.length !== 1) return missing(`${hits.length} GSTACK_TREE_JSON lines name it in the logs of ${names.join(', ')} at attempt ${attempt}; exactly one counts`); + const { job, stamp, data } = hits[0]!; + const step = p.tree_log_step!; + const unbound = (why: string) => ({ ok: false as const, reason: 'tree_log_unbound' as const, detail: `${at}'s ${p.artifact} line in the ${job.name} job ${why} (CI_CARRY_TREE_LOG_UNBOUND)` }); + if (job.conclusion !== 'success') return unbound(`is from a job that concluded ${job.conclusion}`); + const named = (job.steps ?? []).filter(s => s.name === step); + if (named.length !== 1) return unbound(named.length ? `cannot be bound: the job has ${named.length} steps named "${step}"` : `cannot be bound: the job has no step named "${step}"`); + const s = named[0]!; + if (s.conclusion !== 'success') return unbound(`cannot be bound: step "${step}" concluded ${s.conclusion}`); + const ran = (job.steps ?? []).filter(x => x.number < s.number && x.conclusion !== 'skipped'); + const later = (job.steps ?? []).filter(x => x.number > s.number && x.conclusion !== 'skipped'); + const untimed = [s, ...ran, ...later].find(x => !Number.isFinite(Date.parse(x.started_at ?? '')) || !Number.isFinite(Date.parse(x.completed_at ?? ''))); + if (untimed) return unbound(`cannot be bound: step "${untimed.name}" has no start and finish time`); + const t = LOG_STAMP.exec(stamp); + const when = t ? Date.parse(`${t[1]}.${(t[2] ?? '').padEnd(3, '0').slice(0, 3)}Z`) : NaN; + const from = Math.max(Date.parse(s.started_at!), ...ran.map(x => Date.parse(x.completed_at!) + 1000)); + const until = Math.min(Date.parse(s.completed_at!) + 1000, ...later.map(x => Date.parse(x.started_at!) - 1000)); + if (!(when >= from && when < until)) return unbound(`was stamped ${stamp.slice(0, 40)}, not inside step "${step}" (${s.started_at} to ${s.completed_at}) at least a second after every earlier step ended and before every later step started`); + if (data.length > Math.ceil(MAX_ARTIFACT_BYTES / 3) * 4) return missing(`its line holds a tree.json over ${MAX_ARTIFACT_BYTES} bytes`); + const bytes = Buffer.from(data, 'base64'); + if (bytes.length > MAX_ARTIFACT_BYTES) return missing(`its line holds a tree.json over ${MAX_ARTIFACT_BYTES} bytes`); + const bad = { ok: false as const, reason: 'artifact_mismatch' as const }; + if (bytes.toString('base64') !== data) return { ...bad, detail: `${at}'s ${p.artifact} line is not canonical base64` }; + try { return { ok: true, rec: JSON.parse(bytes.toString('utf8')) as TreeRecord }; } + catch { return { ...bad, detail: `${at}'s ${p.artifact} line does not decode to JSON` }; } +} + /** * The newest attempt that ran any of `names` itself (GST-2284). A job-level rerun relists every job it did not rerun under the * new attempt with its old start and finish times, so a job listed with the same times and conclusion one attempt earlier @@ -332,7 +404,7 @@ export function verifyFromRun(src: RunSource, ref: string, names: string[], tree * whose files passed only on a re-run adds `rerun= attempts=` (GST-3054): allowed, but never shown as a clean pass. */ export function runReceiptFields(evidence: RunEvidence[], now: Date): string { - return `runs=${evidence.map(e => `${e.run_id}/${e.run_attempt}`).join(',')} profiles=${evidence.map(e => e.profile).join(',')} heads=${[...new Set(evidence.map(e => e.head_sha.slice(0, 12)))].join(',')}${rerunMarker([...rerunByRun(evidence).values()])} verified=${now.toISOString()}`; + return `runs=${evidence.map(e => `${e.run_id}/${e.run_attempt}`).join(',')} profiles=${evidence.map(e => e.profile).join(',')} heads=${[...new Set(evidence.map(e => e.head_sha.slice(0, 12)))].join(',')}${rerunMarker([...rerunByRun(evidence).values()])}${evidence.some(e => e.tree_source === 'job-log') ? ' tree_source=job-log' : ''} verified=${now.toISOString()}`; } /** The re-run record per carried run (`/`): the distinct re-run files and attempts across every profile that read that run, so two profiles on one run count each file once. */ @@ -394,7 +466,8 @@ export function ghRunDeps(repo: string, repoRoot: string): RunDeps { // A rerun of the uploading job leaves one artifact per attempt (GST-2284): read each by id and keep the one from `attempt`. artifact: (id, name, attempt) => { const listed = all<{ id: number; name: string; size_in_bytes: number; expired: boolean }>(`repos/${repo}/actions/runs/${id}/artifacts?name=${encodeURIComponent(name)}&per_page=100`, 'artifacts').filter(a => a.name === name && !a.expired); - if (listed.length === 0 || listed.length > MAX_ARTIFACT_ATTEMPTS) throw new Error(`run ${id} has ${listed.length} unexpired artifacts named ${name}`); + if (listed.length === 0) throw new NoArtifactError(`run ${id} has 0 unexpired artifacts named ${name}`); + if (listed.length > MAX_ARTIFACT_ATTEMPTS) throw new Error(`run ${id} has ${listed.length} unexpired artifacts named ${name}`); for (const a of listed) if (!(a.size_in_bytes <= MAX_ARTIFACT_BYTES)) throw new Error(`artifact ${name} is ${a.size_in_bytes} bytes, over ${MAX_ARTIFACT_BYTES}`); const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-run-evidence-')); try { @@ -417,6 +490,11 @@ export function ghRunDeps(repo: string, repoRoot: string): RunDeps { return hits[0]!; } finally { fs.rmSync(dir, { recursive: true, force: true }); } }, + // A job log holds the runner's escape sequences, which gh refuses to print without --allow-escape-sequences. + jobLog: jobId => { + if (!Number.isSafeInteger(jobId)) throw new Error(`job id ${jobId} is not a number`); + return gh(['api', '--allow-escape-sequences', `repos/${repo}/actions/jobs/${jobId}/logs`], 120_000); + }, commitTree: sha => { const read = () => spawnSync('git', ['rev-parse', '--verify', '--quiet', `${sha}^{tree}`], { cwd: repoRoot, encoding: 'utf8', timeout: 30_000 }); let r = read(); diff --git a/lib/ship-bin-common.ts b/lib/ship-bin-common.ts index 59f68722a..8dca0ba43 100644 --- a/lib/ship-bin-common.ts +++ b/lib/ship-bin-common.ts @@ -185,7 +185,7 @@ export function laneConcurrency(mode: string | undefined, cpus = os.availablePar return cpus <= 4 ? 1 : Infinity; } -export interface LaneResult { label: string; exit: number | null; testsRan: string; log: string; carried?: string; from?: string; held?: true; reused?: string; carryNone?: string; diagnosis?: string[]; notOwed?: string; here?: string; exception?: string; remoteGate?: string; standIn?: string } +export interface LaneResult { label: string; exit: number | null; testsRan: string; log: string; carried?: string; treeSource?: string; from?: string; held?: true; reused?: string; carryNone?: string; diagnosis?: string[]; notOwed?: string; here?: string; exception?: string; remoteGate?: string; standIn?: string } /** The base branch's `ship_lanes_from` entry for each lane whose command is a key or a declared alias, byte for byte (the ledger binds its sha256). */ export function mappedLanes(cwd: string, base: string, list: { label: string; command: string }[]): Map { @@ -206,7 +206,7 @@ export function carryLane(cwd: string, base: string, l: { label: string; command const rec = bin(cwd, 'gstack-evidence', ['record-ci', '--label', l.label, '--base', base, '--remote-gate-stands-in', '--', l.command], timeoutS * 1000); const got = recordCiResult(rec.stderr); if (rec.status === 0 && got.remoteGate) return { label: l.label, exit: 0, testsRan: got.tests!, log: '-', remoteGate: got.remoteGate, standIn: got.standIn ?? 'not carried' }; - if (rec.status === 0 && got.runs) return { label: l.label, exit: 0, testsRan: got.files ? `${got.tests} (files)` : got.tests!, log: '-', carried: got.runs, ...(got.notOwed ? { notOwed: got.notOwed } : {}), ...(got.here ? { here: got.here } : {}), ...(got.exception ? { exception: got.exception } : {}), ...(got.reused ? { reused: got.reused } : {}), ...(got.from ? { from: got.from } : {}) }; + if (rec.status === 0 && got.runs) return { label: l.label, exit: 0, testsRan: got.files ? `${got.tests} (files)` : got.tests!, log: '-', carried: got.runs, ...(got.treeSource ? { treeSource: got.treeSource } : {}), ...(got.notOwed ? { notOwed: got.notOwed } : {}), ...(got.here ? { here: got.here } : {}), ...(got.exception ? { exception: got.exception } : {}), ...(got.reused ? { reused: got.reused } : {}), ...(got.from ? { from: got.from } : {}) }; return { label: l.label, exit: null, testsRan: 'unknown', log: '-', carryNone: got.none ?? `record_failed: ${firstLine(rec.stderr)}`, ...(got.diagnosis ? { diagnosis: got.diagnosis } : {}) }; } @@ -257,7 +257,7 @@ export async function runLanes(cwd: string, list: { label: string; command: stri /** Step 5's lane verdict: a lane passes on its runner count, never exit 0 alone; a carried lane names its CI runs. */ export function laneLine(l: LaneResult): { line: string; ok: boolean } { if (l.notOwed) return { line: `LANE: ${l.label} not owed: ${l.notOwed} run=${l.carried}`, ok: true }; - if (l.carried) return { line: `LANE: ${l.label} carried run=${l.carried} tests_ran=${l.testsRan}${l.here ? ` ${l.here}` : ''}${l.from ? ` from=${l.from} (same tree as HEAD)` : ''}${l.reused ? ` (reused the CI record at tree ${l.reused})` : ''}`, ok: true }; + if (l.carried) return { line: `LANE: ${l.label} carried run=${l.carried} tests_ran=${l.testsRan}${l.treeSource ? ` tree_source=${l.treeSource}` : ''}${l.here ? ` ${l.here}` : ''}${l.from ? ` from=${l.from} (same tree as HEAD)` : ''}${l.reused ? ` (reused the CI record at tree ${l.reused})` : ''}`, ok: true }; if (l.remoteGate) return { line: `LANE: ${l.label} satisfied by remote_gate receipt=${l.remoteGate}`, ok: true }; if (l.held) return { line: `LANE: ${l.label} held: not run`, ok: false }; const zero = l.exit === 0 && l.testsRan === '0'; diff --git a/lib/ship-preflight.ts b/lib/ship-preflight.ts index 6fa322fe9..493f9cbf4 100644 --- a/lib/ship-preflight.ts +++ b/lib/ship-preflight.ts @@ -267,7 +267,7 @@ function simulate(cwd: string, base: string, list: { label: string; command: str const v = findCarry(check, { profiles: m.profiles, head, tree, wtree, ...mappingCoverage(m, cwd) }); if (!v.ok) { say(`CI_CARRY: none label=${l.label} (${v.reason}: ${v.detail})`); continue; } carried++; - say(`LANE: ${l.label} would carry run=${v.runs.map(e => `${e.run_id}/${e.run_attempt}`).join(',')} tests_ran=${v.tests}${v.unit ? ` unit=${v.unit}` : ''}${v.remainder ? ` and would run ${v.remainder.length} skipped file(s) here with per_file` : ''}`); + say(`LANE: ${l.label} would carry run=${v.runs.map(e => `${e.run_id}/${e.run_attempt}`).join(',')} tests_ran=${v.tests}${v.unit ? ` unit=${v.unit}` : ''}${v.runs.some(e => e.tree_source === 'job-log') ? ' tree_source=job-log' : ''}${v.remainder ? ` and would run ${v.remainder.length} skipped file(s) here with per_file` : ''}`); } say(`${carried} of ${mapped} mapped lane(s) would carry once origin/${base} holds this map`); return done(mapped > 0 && carried === mapped ? 0 : 1); diff --git a/lib/ship-receipt.ts b/lib/ship-receipt.ts index eb2c17bd4..8c6ae22f3 100644 --- a/lib/ship-receipt.ts +++ b/lib/ship-receipt.ts @@ -220,7 +220,7 @@ export function receiptSummaryLine(r: ShipReceipt): string { const seen = runs.get(key) ?? { files: 0, attempts: [] }; runs.set(key, { files: Math.max(seen.files, typeof c.rerun === 'number' ? c.rerun : 0), attempts: [...seen.attempts, ...(Array.isArray(c.attempts) ? c.attempts : [])] }); } - parts.push(`ci_carried=${r.ci_carried.map(c => `${c.lane}:${c.run}/${c.attempt}`).join(',')}${rerunMarker([...runs.values()])}`); + parts.push(`ci_carried=${r.ci_carried.map(c => `${c.lane}:${c.run}/${c.attempt}`).join(',')}${rerunMarker([...runs.values()])}${r.ci_carried.some(c => c.tree_source === 'job-log') ? ' tree_source=job-log' : ''}`); } if (r.base_red && typeof r.base_red.base === 'string') parts.push(`base_red=${r.base_red.base.slice(0, 12)}`); return `SHIP_RECEIPT: ${oneLine(parts.join(' '))}`; diff --git a/test/helpers/fake-gh-runs.ts b/test/helpers/fake-gh-runs.ts index d7facc95f..37a1129c1 100644 --- a/test/helpers/fake-gh-runs.ts +++ b/test/helpers/fake-gh-runs.ts @@ -8,8 +8,10 @@ import * as path from 'node:path'; export interface FakeWorld { runs: Array & { id: number; run_attempt: number; head_sha: string; path: string }>; /** Keyed `/` for one attempt's jobs, else `` for every attempt. */ - jobs: Record>; + jobs: Record>>; trees: Record>; + /** Job id → that job's raw log (HARN-118's tree.json line); a run with no `trees` entry holds no artifact. */ + logs?: Record; /** Once `marker` exists, the next run read writes `content` to `file` (a working-tree edit inside a run recheck). */ editWhenArmed?: { marker: string; file: string; content: string }; } @@ -23,7 +25,7 @@ if (import.meta.main) { const lines = (list: unknown[]) => process.stdout.write(list.map(x => JSON.stringify(x)).join('\n') + (list.length ? '\n' : '')); if (a[0] === 'api') { const jq = a.indexOf('--jq'); - const p = a.filter((x, i) => i > 0 && x !== '--paginate' && i !== jq && i !== jq + 1)[0] ?? ''; + const p = a.filter((x, i) => i > 0 && x !== '--paginate' && x !== '--allow-escape-sequences' && i !== jq && i !== jq + 1)[0] ?? ''; const [route, query = ''] = p.split('?'); const q = new URLSearchParams(query); let m: RegExpExecArray | null; @@ -37,6 +39,10 @@ if (import.meta.main) { lines(world.runs.filter(r => r.head_sha === q.get('head_sha') && r.path === `.github/workflows/${m![1]}`)); } else if ((m = /\/actions\/runs\/(\d+)\/attempts\/(\d+)\/jobs$/.exec(route!))) { lines(world.jobs[`${m[1]}/${m[2]}`] ?? world.jobs[m[1]!] ?? []); + } else if ((m = /\/actions\/jobs\/(\d+)\/logs$/.exec(route!))) { + const text = world.logs?.[m[1]!]; + if (text === undefined || !a.includes('--allow-escape-sequences')) { process.stderr.write('HTTP 404'); process.exit(1); } + process.stdout.write(text); } else if ((m = /\/actions\/runs\/(\d+)\/artifacts$/.exec(route!))) { lines(world.trees[m[1]!] ? [{ name: q.get('name'), size_in_bytes: 200, expired: false }] : []); } else { process.stderr.write(`fake gh: no route for ${p}`); process.exit(1); } diff --git a/test/run-evidence-job-log.test.ts b/test/run-evidence-job-log.test.ts new file mode 100644 index 000000000..e53872cec --- /dev/null +++ b/test/run-evidence-job-log.test.ts @@ -0,0 +1,222 @@ +/** + * HARN-118 / GST-3231: a private repository whose artifact storage is full runs green but uploads no tree.json, so a run + * profile that opts in with `tree_log_step` reads tree.json from one `GSTACK_TREE_JSON ` line in the + * log of its job instead. The line counts only when it is the one such line across the profile's jobs at the attempt that + * ran, the runner stamped it inside the named step at least a second after every earlier step ended, and that step + * succeeded. An artifact always wins, a profile without the key never reads a log, and every tree.json check still runs. + * The dependencies are fakes; nothing reaches GitHub. + */ +import { describe, expect, test } from 'bun:test'; +import { findCarry, recordCiResult } from '../lib/ci-carry'; +import { RESULT_CODES } from '../lib/result-codes'; +import { laneLine } from '../lib/ship-bin-common'; +import { receiptSummaryLine, validateReceipt } from '../lib/ship-receipt'; +import { NoArtifactError, runProfileErrors, runReceiptFields, verifyRun, type ApiJob, type ApiRun, type ApiStep, type RunDeps, type RunProfile, type TreeRecord } from '../lib/run-evidence'; + +const REPO = 'garrytan/harness-evals'; +const HEAD = 'a'.repeat(40); +const TREE = 'b'.repeat(40); +const NOW = new Date('2026-11-19T12:00:00Z'); +const RUN_ID = 44577938128; +const JOB_ID = 135750428163; +const STEP = 'Print tree.json for gstack'; +const PROFILE: RunProfile = { workflow: 'ci.yml', events: ['push'], jobs: ['checks'], artifact: 'checks-tree', os: 'Linux', tree_log_step: STEP }; +const REC: TreeRecord = { run_id: RUN_ID, run_attempt: 1, event: 'push', sha: HEAD, tree: TREE, os: 'Linux', files: ['tests/a.test.ts', 'tests/b.test.ts'], tests: 9 }; +const b64 = (rec: unknown) => Buffer.from(JSON.stringify(rec)).toString('base64'); +/** One log line as the runner writes it: its own timestamp, then what the step printed. */ +const line = (at: string, printed: string) => `${at} ${printed}`; +const treeLine = (at: string, rec: unknown = REC, name = 'checks-tree') => line(at, `GSTACK_TREE_JSON ${name} ${b64(rec)}`); +/** The step prints the line 1.5 s after the test step ended (it sleeps a second first). */ +const GOOD_AT = '2026-11-19T11:20:01.5000000Z'; +/** A test inside the test step printing a well-formed line with a real-looking tree.json. */ +const FORGED_AT = '2026-11-19T11:10:00.1234567Z'; +const log = (...lines: string[]) => ['\uFEFF2026-11-19T11:00:00.0082632Z Current runner version: 2.338.2', ...lines, '2026-11-19T11:20:04.1000000Z Cleaning up orphan processes'].join('\n'); + +function steps(over: Record | null> = {}, extra: ApiStep[] = []): ApiStep[] { + const list: ApiStep[] = [ + { number: 1, name: 'Set up job', conclusion: 'success', started_at: '2026-11-19T11:00:00Z', completed_at: '2026-11-19T11:00:02Z' }, + { number: 2, name: 'Run tests', conclusion: 'success', started_at: '2026-11-19T11:00:02Z', completed_at: '2026-11-19T11:20:00Z' }, + { number: 3, name: STEP, conclusion: 'success', started_at: '2026-11-19T11:20:00Z', completed_at: '2026-11-19T11:20:03Z' }, + { number: 4, name: 'Complete job', conclusion: 'success', started_at: '2026-11-19T11:20:03Z', completed_at: '2026-11-19T11:20:04Z' }, + ]; + return [...list.filter(s => over[s.name] !== null).map(s => ({ ...s, ...over[s.name] })), ...extra]; +} +const job = (over: Partial = {}): ApiJob => ({ id: JOB_ID, name: 'checks', conclusion: 'success', started_at: '2026-11-19T11:00:00Z', completed_at: '2026-11-19T11:20:04Z', steps: steps(), ...over }); + +interface World { artifact?: TreeRecord | Error; logs: Record; jobs?: ApiJob[]; jobsByAttempt?: Record; attempt?: number } +function deps(w: World): RunDeps & { logReads: number[] } { + const run: ApiRun = { id: RUN_ID, run_attempt: w.attempt ?? 1, status: 'completed', conclusion: 'success', event: 'push', head_sha: HEAD, path: '.github/workflows/ci.yml', + created_at: '2026-11-19T11:00:00Z', updated_at: '2026-11-19T11:20:10Z', repository: { full_name: REPO }, head_repository: { full_name: REPO } }; + const logReads: number[] = []; + return { + logReads, + run: () => run, + runsForHead: (_wf, sha) => (sha === HEAD ? [run] : []), + jobs: (_id, attempt) => w.jobsByAttempt?.[attempt] ?? w.jobs ?? [job()], + artifact: (_id, name) => { if (!w.artifact) throw new NoArtifactError(`run ${RUN_ID} has 0 unexpired artifacts named ${name}`); if (w.artifact instanceof Error) throw w.artifact; return w.artifact; }, + jobLog: id => { logReads.push(id); const text = w.logs[id]; if (text === undefined) throw new Error('HTTP 404'); return text; }, + commitTree: sha => (sha === HEAD ? TREE : null), + }; +} +const check = (w: World, profile: RunProfile = PROFILE) => verifyRun(deps(w), { repo: REPO, runRef: String(RUN_ID), profileName: 'ci/checks', profile, now: NOW }); +const refusal = (w: World, profile?: RunProfile) => { const v = check(w, profile); return v.ok ? ['ok', ''] : [v.reason, v.detail]; }; + +describe('tree.json from the job log when the run has no artifact (HARN-118)', () => { + // Value: protects=a green run whose artifact upload hit the storage quota still carries; fails_when=the fallback is missing and artifact_missing refuses it + test('one line in the named step, with no artifact, is the run\'s tree.json, and every carry line says where it came from', () => { + const w: World = { logs: { [JOB_ID]: log(treeLine(GOOD_AT)) } }; + const d = deps(w); + const v = verifyRun(d, { repo: REPO, runRef: String(RUN_ID), profileName: 'ci/checks', profile: PROFILE, now: NOW }); + expect(v.ok).toBe(true); + if (!v.ok) return; + expect(v.evidence).toMatchObject({ run_id: RUN_ID, run_attempt: 1, tree: TREE, files: ['tests/a.test.ts', 'tests/b.test.ts'], tests: 9, tree_source: 'job-log' }); + expect(d.logReads).toEqual([JOB_ID]); + expect(runReceiptFields([v.evidence], NOW)).toContain(`runs=${RUN_ID}/1 profiles=ci/checks heads=${HEAD.slice(0, 12)} tree_source=job-log verified=`); + const carry = findCarry({ repo: REPO, deps: deps(w), profiles: { 'ci/checks': PROFILE }, now: NOW }, { profiles: ['ci/checks'], head: HEAD, tree: TREE, wtree: () => TREE, inventory: () => ['tests/a.test.ts', 'tests/b.test.ts'] }); + expect(carry.ok ? carry.receipt : carry.detail).toContain(' tree_source=job-log '); + expect(log(treeLine(GOOD_AT)).replace(/\n/g, '\r\n')).toContain('\r\n'); + expect(check({ logs: { [JOB_ID]: log(treeLine(GOOD_AT)).replace(/\n/g, '\r\n') } }).ok).toBe(true); + }); + + // Value: protects=an artifact always wins and the log is never read beside it; fails_when=a log line overrides or is consulted next to an uploaded tree.json + test('an artifact wins: the log is not read, and the evidence names no tree_source', () => { + const w: World = { artifact: REC, logs: { [JOB_ID]: log(treeLine(GOOD_AT, { ...REC, tree: 'c'.repeat(40) }), treeLine(GOOD_AT)) } }; + const d = deps(w); + const v = verifyRun(d, { repo: REPO, runRef: String(RUN_ID), profileName: 'ci/checks', profile: PROFILE, now: NOW }); + expect(v.ok).toBe(true); + expect(v.ok && v.evidence.tree_source).toBeUndefined(); + expect(d.logReads).toEqual([]); + expect(v.ok ? runReceiptFields([v.evidence], NOW) : '').not.toContain('tree_source'); + }); + + // Value: protects=rule 3: only "no artifact at all" opens the log; fails_when=an unreadable, oversized or ambiguous artifact falls back to a log line + test('an artifact that exists but cannot be read refuses as artifact_missing and never reads the log', () => { + for (const e of [new Error('run 1 has 2 unexpired artifacts named checks-tree and 0 of them record run_attempt 1'), new Error('artifact checks-tree is 9000000 bytes, over 8388608')]) { + const d = deps({ artifact: e, logs: { [JOB_ID]: log(treeLine(GOOD_AT)) } }); + const v = verifyRun(d, { repo: REPO, runRef: String(RUN_ID), profileName: 'ci/checks', profile: PROFILE, now: NOW }); + expect(v.ok ? 'ok' : v.reason).toBe('artifact_missing'); + expect(d.logReads).toEqual([]); + } + }); + + // Value: protects=the log of the attempt that ran (ranIn), not the newest attempt's relisted job; fails_when=a rerun of another job makes gstack read the new attempt's job id + test('after a rerun of another job, the log read is the profile job\'s own attempt', () => { + const first = job({ id: 501 }); + const relisted = job({ id: 502 }); + const d = deps({ attempt: 2, jobsByAttempt: { 1: [first], 2: [relisted, { name: 'lint', conclusion: 'success', started_at: '2026-11-19T11:30:00Z', completed_at: '2026-11-19T11:31:00Z' }] }, logs: { 501: log(treeLine(GOOD_AT)), 502: log(treeLine(GOOD_AT, { ...REC, run_attempt: 2 })) } }); + const v = verifyRun(d, { repo: REPO, runRef: String(RUN_ID), profileName: 'ci/checks', profile: PROFILE, now: NOW }); + expect(v.ok ? [v.evidence.run_attempt, v.evidence.tree_source] : v.detail).toEqual([2, 'job-log']); + expect(d.logReads).toEqual([501]); + }); + + // Value: protects=rule 1 fails closed on ambiguity, across every profile job; fails_when=the first or last of several lines is taken + test('two lines refuse as artifact_missing with the count, in one job or across the profile\'s jobs', () => { + expect(refusal({ logs: { [JOB_ID]: log(treeLine(GOOD_AT), treeLine('2026-11-19T11:20:02.0000000Z')) } })).toEqual(['artifact_missing', expect.stringContaining('2 GSTACK_TREE_JSON lines')]); + const two: ApiJob[] = [job(), job({ id: JOB_ID + 1, name: 'docker-e2e' })]; + const both = { ...PROFILE, jobs: ['checks', 'docker-e2e'] }; + expect(refusal({ jobs: two, logs: { [JOB_ID]: log(treeLine(GOOD_AT)), [JOB_ID + 1]: log(treeLine(GOOD_AT)) } }, both)).toEqual(['artifact_missing', expect.stringContaining('2 GSTACK_TREE_JSON lines')]); + expect(check({ jobs: two, logs: { [JOB_ID]: log(treeLine(GOOD_AT)), [JOB_ID + 1]: log() } }, both).ok).toBe(true); + }); + + // Value: protects=rule 1's zero case; fails_when=a run with no artifact and no line, or a line for another artifact, carries or errors as anything else + test('no line refuses as artifact_missing with the count; a line naming another artifact is not this profile\'s', () => { + expect(refusal({ logs: { [JOB_ID]: log() } })).toEqual(['artifact_missing', expect.stringContaining('0 GSTACK_TREE_JSON lines')]); + expect(refusal({ logs: { [JOB_ID]: log(treeLine(GOOD_AT, REC, 'other-tree')) } })).toEqual(['artifact_missing', expect.stringContaining('0 GSTACK_TREE_JSON lines')]); + expect(refusal({ logs: {} })[0]).toBe('artifact_missing'); + }); + + // Value: protects=the opt-in key: a repository that never prints the line can't be fooled by a test that prints one; fails_when=a profile without tree_log_step reads the log + test('a profile without tree_log_step never reads the log: no artifact stays artifact_missing', () => { + const { tree_log_step: _, ...plain } = PROFILE; + const d = deps({ logs: { [JOB_ID]: log(treeLine(GOOD_AT)) } }); + const v = verifyRun(d, { repo: REPO, runRef: String(RUN_ID), profileName: 'ci/checks', profile: plain, now: NOW }); + expect(v.ok ? 'ok' : v.reason).toBe('artifact_missing'); + expect(d.logReads).toEqual([]); + }); +}); + +describe('a forged line never stands in for the named step (HARN-118 design calls)', () => { + // Value: protects=a test that prints a well-formed line in an opted-in repository; fails_when=the only line, printed during the test step, is accepted + test('a well-formed line a test printed during the test step refuses as tree_log_unbound, alone or beside the real one', () => { + expect(refusal({ logs: { [JOB_ID]: log(treeLine(FORGED_AT)) } })).toEqual(['tree_log_unbound', expect.stringContaining(`not inside step "${STEP}"`)]); + expect(refusal({ logs: { [JOB_ID]: log(treeLine(FORGED_AT), treeLine(GOOD_AT)) } })).toEqual(['artifact_missing', expect.stringContaining('2 GSTACK_TREE_JSON lines')]); + }); + + // Value: protects=the runner's timestamp is the only one read; fails_when=a test that prints its own timestamp before the marker moves the line into the named step + test('a test that prints its own timestamp gets the runner\'s in front of it, so the line does not match', () => { + expect(refusal({ logs: { [JOB_ID]: log(line(FORGED_AT, `${GOOD_AT} GSTACK_TREE_JSON checks-tree ${b64(REC)}`)) } })).toEqual(['artifact_missing', expect.stringContaining('0 GSTACK_TREE_JSON lines')]); + expect(refusal({ logs: { [JOB_ID]: log(line(FORGED_AT, `x\r${GOOD_AT} GSTACK_TREE_JSON checks-tree ${b64(REC)}`)) } })).toEqual(['artifact_missing', expect.stringContaining('0 GSTACK_TREE_JSON lines')]); + }); + + // Value: protects=step binding at the API's one-second resolution; fails_when=a line in the second the test step ended, after the step, or from a failed, missing, repeated or untimed step counts + test('the line binds to the named step only: same second as the test step, after the step, or a step that is not one clean success all refuse', () => { + const at = (t: string) => refusal({ logs: { [JOB_ID]: log(treeLine(t)) } })[0]; + expect(at('2026-11-19T11:20:00.4000000Z')).toBe('tree_log_unbound'); + expect(at('2026-11-19T11:20:01.0000000Z')).toBe('ok'); + expect(at('2026-11-19T11:20:01.9990000Z')).toBe('ok'); + expect(at('2026-11-19T11:20:02.0000000Z')).toBe('tree_log_unbound'); + expect(at('2026-11-19T11:20:03.9000000Z')).toBe('tree_log_unbound'); + expect(at('2026-11-19T11:20:04.5000000Z')).toBe('tree_log_unbound'); + expect(refusal({ jobs: [job({ steps: steps({ 'Complete job': { started_at: '2026-11-19T11:20:05Z', completed_at: '2026-11-19T11:20:06Z' } }) })], logs: { [JOB_ID]: log(treeLine('2026-11-19T11:20:03.9000000Z')) } })[0]).toBe('ok'); + expect(at('not-a-timeZ')).toBe('tree_log_unbound'); + const withSteps = (s: ApiStep[]) => refusal({ jobs: [job({ steps: s })], logs: { [JOB_ID]: log(treeLine(GOOD_AT)) } }); + expect(withSteps(steps({ [STEP]: { conclusion: 'failure' } }))).toEqual(['tree_log_unbound', expect.stringContaining('concluded failure')]); + expect(withSteps(steps({ [STEP]: null }))).toEqual(['tree_log_unbound', expect.stringContaining(`no step named "${STEP}"`)]); + expect(withSteps(steps({}, [{ number: 5, name: STEP, conclusion: 'success', started_at: '2026-11-19T11:20:04Z', completed_at: '2026-11-19T11:20:05Z' }]))).toEqual(['tree_log_unbound', expect.stringContaining('2 steps named')]); + expect(withSteps(steps({ 'Run tests': { completed_at: null } }))).toEqual(['tree_log_unbound', expect.stringContaining('no start and finish time')]); + expect(withSteps(steps({ [STEP]: { started_at: null } }))[0]).toBe('tree_log_unbound'); + expect(withSteps(steps({ 'Complete job': { started_at: null } }))).toEqual(['tree_log_unbound', expect.stringContaining('"Complete job" has no start and finish time')]); + expect(withSteps(steps({}, [{ number: 5, name: 'Run more tests', conclusion: 'success', started_at: '2026-11-19T11:20:02Z', completed_at: '2026-11-19T11:25:00Z' }]))).toEqual(['tree_log_unbound', expect.stringContaining('before every later step started')]); + expect(refusal({ jobs: [job({ conclusion: 'failure' })], logs: { [JOB_ID]: log(treeLine(GOOD_AT)) } })[0]).toBe('job_not_success'); + expect(refusal({ jobs: [job({ id: undefined })], logs: { [JOB_ID]: log(treeLine(GOOD_AT)) } })[0]).toBe('artifact_missing'); + }); +}); + +describe('a log tree.json gets every check an artifact does', () => { + // Value: protects=rule 2's size cap before decoding or parsing; fails_when=an oversized line is decoded and accepted + test('a line whose tree.json is over 8 MiB refuses as artifact_missing', () => { + const big = Buffer.alloc(8 * 1024 * 1024 + 1, 0x20).toString('base64'); + expect(refusal({ logs: { [JOB_ID]: log(line(GOOD_AT, `GSTACK_TREE_JSON checks-tree ${big}`)) } })).toEqual(['artifact_missing', expect.stringContaining('over 8388608 bytes')]); + }); + + // Value: protects=rule 2's unchanged verifyRun checks; fails_when=a decoded record skips the run, attempt, tree, os or shape checks + test('a record naming another attempt, tree or OS, or a line that is not base64 JSON, refuses as an artifact would', () => { + const with_ = (rec: unknown) => refusal({ logs: { [JOB_ID]: log(treeLine(GOOD_AT, rec)) } })[0]; + expect(with_({ ...REC, run_attempt: 2 })).toBe('artifact_mismatch'); + expect(with_({ ...REC, run_id: 1 })).toBe('artifact_mismatch'); + expect(with_({ ...REC, tree: 'c'.repeat(40) })).toBe('tree_mismatch'); + expect(with_({ ...REC, os: 'macOS' })).toBe('os_mismatch'); + expect(with_({ ...REC, reused: true })).toBe('artifact_reused'); + expect(with_({ ...REC, files: 'tests/a.test.ts' })).toBe('artifact_mismatch'); + expect(refusal({ logs: { [JOB_ID]: log(line(GOOD_AT, `GSTACK_TREE_JSON checks-tree ${Buffer.from('{not json').toString('base64')}`)) } })[0]).toBe('artifact_mismatch'); + expect(refusal({ logs: { [JOB_ID]: log(line(GOOD_AT, 'GSTACK_TREE_JSON checks-tree eyJ')) } })[0]).toBe('artifact_mismatch'); + }); + + // Value: protects=the opt-in key's shape; fails_when=an empty, multi-line or non-string step name is accepted + test('tree_log_step must be one step name', () => { + const errs = (v: unknown) => runProfileErrors({ p: { ...PROFILE, tree_log_step: v } }, undefined).map(e => e.path); + expect(errs(STEP)).toEqual([]); + for (const bad of ['', ' ', 'a\nb', 7, 'x'.repeat(201)]) expect(errs(bad)).toEqual(['run_profiles["p"].tree_log_step']); + expect(RESULT_CODES.CI_CARRY_TREE_LOG_UNBOUND.anchor).toBe('ci-carry-tree-log-unbound'); + }); +}); + +describe('the receipt and the lane line say a tree.json came from a job log (rule 4)', () => { + const receipt = (ci_carried: unknown) => ({ schema_version: 1 as const, head: HEAD, base: 'main', ci_carried } as Parameters[0]); + // Value: protects=the receipt's ci_carried entry and SHIP_RECEIPT line record the source; fails_when=the field is dropped, unprinted or accepts another value + test('a ci_carried entry records tree_source job-log, and SHIP_RECEIPT prints it after ci_carried', () => { + const r = receipt([{ lane: 'tests', run: RUN_ID, attempt: 1, tree: TREE, tree_source: 'job-log' }]); + expect(validateReceipt(r)).toEqual([]); + expect(receiptSummaryLine(r)).toContain(`ci_carried=tests:${RUN_ID}/1 tree_source=job-log`); + expect(receiptSummaryLine(receipt([{ lane: 'tests', run: RUN_ID, attempt: 1, tree: TREE }]))).not.toContain('tree_source'); + expect(validateReceipt(receipt([{ lane: 'tests', run: RUN_ID, attempt: 1, tree: TREE, tree_source: 'artifact' }])).length).toBeGreaterThan(0); + }); + + // Value: protects=the LANE line /ship prints; fails_when=record-ci's marker is not parsed into the lane result + test('record-ci\'s recorded line carries tree_source into the LANE line', () => { + const got = recordCiResult(`gstack-evidence: recorded label=tests exit=0 tests_ran=2 unit=files log=- source=ci runs=${RUN_ID}/1 tree_source=job-log policy=origin/main@abc`); + expect(got).toMatchObject({ runs: `${RUN_ID}/1`, tests: '2', files: true, treeSource: 'job-log' }); + expect(laneLine({ label: 'tests', exit: 0, testsRan: '2 (files)', log: '-', carried: got.runs, treeSource: got.treeSource }).line).toBe(`LANE: tests carried run=${RUN_ID}/1 tests_ran=2 (files) tree_source=job-log`); + expect(recordCiResult(`gstack-evidence: recorded label=tests exit=0 tests_ran=2 log=- source=ci runs=${RUN_ID}/1 policy=p`).treeSource).toBeUndefined(); + }); +}); diff --git a/test/ship-ci-carry.test.ts b/test/ship-ci-carry.test.ts index cffd7a443..e410e23c6 100644 --- a/test/ship-ci-carry.test.ts +++ b/test/ship-ci-carry.test.ts @@ -786,3 +786,55 @@ describe('GST-3019: --tests-only never starts a lane on its own when its carry r expect(f.run(PREFLIGHT, ['--base', 'main', '--run-uncarried-here', '--lane', `free=${FREE}`]).status).toBe(2); }); }); + +describe('HARN-118: a run with no artifact carries from the one tree.json line its named step logged', () => { + const STEP = 'Print tree.json'; + const LOGGED = { ...BASE, '.gstack/pregate.json': JSON.stringify({ run_profiles: { 'ci/free': { ...PROFILE, tree_log_step: STEP } }, ship_lanes_from: { [FREE]: { profiles: ['ci/free'], inventory: "printf 'test/a.test.ts\\ntest/b.test.ts\\n'" } } }, null, 2) + '\n' }; + /** Run 7 green with no artifact; job 70's named step logs `printed` (the real line unless overridden) 1.5 s after the test step ended. */ + function logWorld(f: ShipFx, printed?: (rec: string) => string[]) { + const head = f.git('rev-parse', 'HEAD').stdout.trim(); + const tree = f.git('rev-parse', 'HEAD^{tree}').stdout.trim(); + const t0 = Math.floor(Date.now() / 1000) * 1000 - 10 * 60_000; + const s = (sec: number) => new Date(t0 + sec * 1000).toISOString().replace('.000Z', 'Z'); + const stamp = (sec: number) => new Date(t0 + sec * 1000).toISOString().replace('Z', '0000Z'); + const rec = Buffer.from(JSON.stringify({ run_id: 7, run_attempt: 1, event: 'workflow_dispatch', sha: head, tree, files: ['test/a.test.ts', 'test/b.test.ts'], tests: 12, os: 'Linux' })).toString('base64'); + const steps = [ + { number: 1, name: 'Run tests', conclusion: 'success', started_at: s(0), completed_at: s(60) }, + { number: 2, name: STEP, conclusion: 'success', started_at: s(60), completed_at: s(63) }, + ]; + const run = { id: 7, run_attempt: 1, status: 'completed', conclusion: 'success', event: 'workflow_dispatch', head_sha: head, path: '.github/workflows/free.yml', created_at: s(-5), run_started_at: s(-5), updated_at: s(64), repository: { full_name: 'fx/fx' }, head_repository: { full_name: 'fx/fx' } }; + const lines = printed ? printed(rec) : [`${stamp(61.5)} GSTACK_TREE_JSON tested-tree ${rec}`]; + const w: FakeWorld = { runs: [run] as FakeWorld['runs'], jobs: { 7: [{ id: 70, name: 'suite', conclusion: 'success', started_at: s(0), completed_at: s(63), steps }] }, trees: {}, + logs: { 70: [`${stamp(0)} ##[group]Run bun test`, ...lines, `${stamp(63.5)} Cleaning up orphan processes`].join('\n') } }; + const file = path.join(f.root, 'world.json'); + fs.writeFileSync(file, JSON.stringify(w)); + f.gh(fakeGhScript(file)); + } + + // Value: protects=HARN-118: a green run whose artifact upload hit the storage quota carries end to end and says so on LANE, CI_CARRIED and the receipt's entry; + // fails_when=the lane runs here beside a green run, or the carried entry hides where tree.json came from; why_new=harness-evals #501 run 44577938128 (0 artifacts); seam=fake gh, gstack-evidence ledger + test('the lane carries with tree_source=job-log, and carried --json records it per run', () => { + const f = shipFixture(LOGGED); + fixtures.push(f); + logWorld(f); + const out = preflight(f, [`free=${FREE}`]); + expect(out.status).toBe(0); + expect(lane(out.stdout, 'free')).toBe('LANE: free carried run=7/1 tests_ran=2 (files) tree_source=job-log'); + expect(runs(f, 'free-runs')).toBe(0); + const carried = f.run(path.join(BIN, 'gstack-evidence'), ['carried', '--base', 'main', '--json']); + expect(JSON.parse(carried.stdout)).toEqual([expect.objectContaining({ lane: 'free', run: 7, attempt: 1, tree_source: 'job-log' })]); + expect(f.run(path.join(BIN, 'gstack-evidence'), ['carried', '--base', 'main']).stdout).toMatch(/^CI_CARRIED: free run=7\/1 tree=[0-9a-f]{12} tree_source=job-log$/m); + }); + + // Value: protects=the forged-line control through the real gh-backed reader; fails_when=a line a test printed during the test step carries + test('a line a test printed during the test step runs the lane here', () => { + const f = shipFixture(LOGGED); + fixtures.push(f); + logWorld(f, rec => [`${new Date(Math.floor(Date.now() / 1000) * 1000 - 10 * 60_000 + 30_000).toISOString().replace('Z', '0000Z')} GSTACK_TREE_JSON tested-tree ${rec}`]); + const out = preflight(f, [`free=${FREE}`], UNCARRIED_HERE); + expect(out.stdout).toContain('tree_log_unbound'); + expect(lane(out.stdout, 'free')).toMatch(/^LANE: free pass exit=0 tests_ran=3 /); + expect(runs(f, 'free-runs')).toBe(1); + }); +}); +