From 299e051010337cec0ce74bc43d829ebedafc106a Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 16 Oct 2026 18:14:35 -0700 Subject: [PATCH 1/2] v1.91.112.0 fix(cso): --paths scopes an audit and reports every skipped path (no more hand-trimmed copies) (#3230) * fix(cso): --paths scopes an audit and reports every skipped path A repo over the 64 MiB cap was audited on hand-trimmed copies, so the verdict read assessed over partial input. --paths records out-of-scope files unread as snapshot-inputs gaps (coverage partial), keeps --diff fail-closed, and carries the scope through submit, recheck and replay. Field list 2 item 12; related #2993. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(cso): close /review findings on --paths scoping - Comparison mode hashes out-of-scope files in bounded 1 MiB chunks and re-hashes them in both stability passes instead of reading whole files. - --diff --paths refuses every changed path outside the scope except host dependency trees, including credential and agent configuration. - --paths prefixes that report persistence would rewrite are refused. - In-scope gaps sort first; out-of-scope files and deletions share the skipped tier and are dropped from transformations. - History is limited to --paths, reported as partial, and history queries on out-of-scope paths are refused. - A --diff recheck refused for out-of-scope changes says to start a new audit. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(cso): troubleshooting names --paths for the source cap Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): /cso --paths entry Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.112.0 fix(cso): --paths scopes an audit and reports every skipped path (#3230) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 20 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- cso/SKILL.md | 13 +- cso/SKILL.md.tmpl | 13 +- docs/troubleshooting.md | 14 +- lib/cso/cli.ts | 149 +++++++++++--- lib/cso/contracts.ts | 1 + lib/cso/snapshot.ts | 84 ++++++-- package.json | 2 +- test/cso-audit-paths.test.ts | 347 +++++++++++++++++++++++++++++++++ 11 files changed, 588 insertions(+), 59 deletions(-) create mode 100644 test/cso-audit-paths.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a7577b1a7..ed442b456 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.91.112.0] - 2026-10-17 + +**`/cso --paths` audits part of a repository and says exactly what it skipped.** A repository over /cso's 64 MiB source cap used to get one piece of advice, "run /cso on a smaller checkout". The trimmed copy then read as the whole repository, and the verdict never said what was left out. `gstack-cso start --paths src/core,src/mcp` now audits only those prefixes and reports every other path as skipped, with snapshot and history coverage marked partial. + +### What this means for you + +- **`--paths` narrows an audit without hiding what it left out.** Files outside the prefixes are never copied, read or counted toward the cap. The run's snapshot manifest lists each one, the report counts them, and snapshot-inputs and history-inputs coverage read partial, so the verdict can't look complete. Git history is limited to the same prefixes. +- **`--diff --paths` fails closed.** A changed, added or deleted file outside the prefixes refuses the run and is named, including `.npmrc`, `bunfig.toml`, `.claude/` and other credential or agent configuration. Only dependency trees such as `node_modules`, which /cso never reads, are exempt. +- **Findings stay in scope.** A finding, recheck evidence or `history` query on a skipped path is refused, and rechecks and replays keep the original `--paths`. +- **The source-cap message points to `--paths`** and warns against hand-trimmed copies. The manifest cap and the 100,000-entry limit still count every file, so their advice is unchanged (#2993). + +### If you… + +- **…audited a hand-trimmed copy to get under the cap:** rerun /cso on the full checkout with `--paths` set to the directories you want audited, so the report says what was skipped. + +### For contributors + +- New `test/cso-audit-paths.test.ts` covers the cap advice, the withheld entries and gaps, the `--diff` refusal in both directions, bounded 1 MiB hashing of out-of-scope files when a comparison base is set, recheck and replay scope, and out-of-scope history and findings. +- Known limits: under `--paths` the original-source hash ignores content changes outside the scope, out-of-scope hashing has no byte cap beyond the time budget, `--paths` with `--comprehensive` runs on a trimmed tree, and a recheck refuses when the audited directory was renamed. + ## [1.91.111.0] - 2026-10-16 **Four background reds in the paid PR gate were caused by a detector, a fixture interface, a gstack helper and a Review Army workflow block, not by the PRs that hit them.** Each is fixed where it starts, and each fix has a free regression built from the red run's captured events. Over the last 99 gate runs, 10 of 28 full-selection PR runs had a red. Replayed under these fixes, that drops to 8, and red verdicts drop from 13 to 10. No threshold, budget, trial count or quarantine changed. diff --git a/VERSION b/VERSION index d43ad3748..4e01f37aa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.111.0 +1.91.112.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 27fa17b85..945d1da1f 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.111.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.112.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/cso/SKILL.md b/cso/SKILL.md index b604f1de8..46e19240b 100644 --- a/cso/SKILL.md +++ b/cso/SKILL.md @@ -45,6 +45,7 @@ Do not send findings, source, secrets, harnesses, or bundles to gbrain, telemetr | `--base ` | Select the comparison base, including for diff mode. | | `--budget ` | Bound wall-clock investigation time, including setup. | | `--offline` | Disable helper, scanner, download, and advisory network access. | +| `--paths ` | Audit only these repo prefixes (never a trimmed copy); others become snapshot-inputs gaps. | ## Mode Resolution @@ -64,10 +65,10 @@ Phases 0, 1, 12, 13, and 14 always run for an investigation. Select the remainin Diff mode may read unchanged callers, middleware, schemas, configuration, and dependencies needed for assessment. Report out-of-scope variants as follow-up scope. Historical-secret coverage uses only the helper's pinned base/snapshot. -Use `gstack-cso schema` for JSON inputs and `--help` for the installed contract. Below, `gstack-cso` means the trusted absolute launcher: +`gstack-cso schema` gives JSON inputs; `--help` the installed contract. Below, `gstack-cso` is the trusted absolute launcher: ```text -gstack-cso start --repo [scope/diff/base/budget/offline flags] [--comprehensive] +gstack-cso start --repo [scope/diff/base/budget/offline/paths] [--comprehensive] gstack-cso doctor --repo gstack-cso resume gstack-cso replay [--source ] @@ -117,7 +118,7 @@ Use the snapshot inventory and redacted source to map stacks, **actors, assets, Use stack detection to prioritize nested services and cross-language paths. Comprehensive setup supports Node, Bun, Python, and Rails only with a matching qualified runtime profile. Missing profiles, runtimes, or tools are execution prerequisites, not vulnerabilities. Reduce coverage only for unfinished assessment work. -In comprehensive mode, review snapshot transformations before reproduction. If sanitization removes or replaces the tested boundary, block reproduction. Never claim sanitized configuration equivalence without evidence. Readiness failures do not block static work. +In comprehensive mode, review snapshot transformations first; block reproduction if sanitization removed or replaced the tested boundary. Never claim sanitized configuration equivalence without evidence. Readiness failures do not block static work. ### Phase 1: Attack surface census @@ -136,7 +137,7 @@ This **CSO evidence rubric** governs CSO instead of shared review confidence ins Daily reports contain **supported findings**: a concrete attacker-controlled entrypoint, a path across an intended security boundary, demonstrated impact, and a challenge of relevant protective controls. Comprehensive reports retain unresolved candidates separately as **labeled hypotheses**, never mixed into supported totals. Disproved candidates are retained as disposition/coverage evidence, not vulnerabilities. -Do not apply blanket exclusions for development dependencies, availability/resource attacks, historical secrets, user-role prompt injection, or gstack-owned skills. Analyze attacker control and impact. UUIDs do not provide authorization; user-controlled URL paths can still cross a sensitive boundary; environment variables may originate from untrusted workflows; and safe defaults can be bypassed by framework escape hatches. Missing hardening alone needs a concrete failure scenario before becoming a finding. +Do not apply blanket exclusions for development dependencies, availability/resource attacks, historical secrets, user-role prompt injection, or gstack-owned skills. Analyze attacker control and impact. UUIDs are not authorization; user-controlled URL paths can cross a sensitive boundary; environment variables may come from untrusted workflows; framework escape hatches can bypass safe defaults. Missing hardening alone needs a concrete failure scenario before becoming a finding. For each candidate, use an already-authorized independent reviewer when available. Give it the relevant locations, invariant, and rubric without the producer's conclusion; have it inspect callers, middleware, configuration, validation, legitimate behavior, and mitigations. Use at most three; await them. Do not request broader tool access solely to obtain an independent reviewer. Otherwise perform a separate skeptical pass labeled **sequential challenge; independent agent unavailable**. Record dissent and assumptions. Agreement and scanner warnings do not prove runtime behavior. @@ -153,13 +154,13 @@ Keep finding evidence, reproduction outcome, patch validation, test-completion a ### Phase 13: Report and repair artifacts -Finish every audit through the helper, including empty, cancelled, blocked, or interrupted audits when possible. Every report begins with **complete**, **partial**, or **not assessed**, followed by scope and material gaps. Completeness is independent of finding count. For an empty supported set, say **“No supported findings in the assessed scope.”**, or **not assessed** if nothing was. Lead incomplete reports with the helper's Status/Ran/Reason/Next lines. Never infer a clean result from setup failure or absent scanner output. +Finish every audit through the helper, including empty, cancelled, blocked, or interrupted audits when possible. Every report begins with **complete**, **partial**, or **not assessed**, followed by scope and material gaps; a `--paths` run states partial snapshot-inputs coverage and summarizes skipped paths. Completeness is independent of finding count. For an empty supported set, say **“No supported findings in the assessed scope.”**, or **not assessed** if nothing was. Lead incomplete reports with the helper's Status/Ran/Reason/Next lines. Never infer a clean result from setup failure or absent scanner output. Present a compact SECURITY FINDINGS table with stable finding ID, severity, confidence/rationale, evidence state, location, and impact. Each finding needs an attacker scenario, supporting references, counterevidence considered, and a concrete repair recommendation. Include coverage, transformations, scanner versions/outcomes/freshness, runtime prerequisites, timing, and proposed repair-candidate paths. Include `runtime_tested` bundle paths only when an authenticated assertion witness actually produced one. Beside every bundle, show assertion, test-completion, and review assurance exactly as recorded; never collapse those labels or imply that self-reported project tests are authenticated. Comprehensive hypotheses belong in a separate labeled appendix. Use `RunReportV3`, `FindingV3`, `CoverageRecord`, `VerificationManifest`, and `RepairBundle` from `lib/cso/contracts.ts` through the installed schema command. Helper validation/persistence is mandatory. If saving fails, report **PERSISTENCE_FAILED** and a safe summary; never claim the report was saved. If redaction fails, withhold the payload entirely; do not show raw excerpts to explain the failure. -The private state namespace is `security/cso//` under the existing state root, outside synchronization allowlists. Execution copies and services are removed immediately. Sanitized snapshots expire after seven days; redacted reports, repair candidates, and bundles after thirty days. Public archive caches are hash-verified, capped at 10 GiB, and evicted by LRU. Explicitly exported artifacts remain under user control. Repair candidates preserve their redacted request, patch, external observations, self-reported project-test outcome, provenance, and required input hashes. Authenticated bundles additionally preserve the inputs required for replay and their separate assurance labels. Currently issued `runtime_tested` bundles record authenticated external assertions and `self_reported` project-test completion. +The private state namespace is `security/cso//` under the existing state root, outside synchronization allowlists. Execution copies and services are removed immediately. Sanitized snapshots expire after seven days; redacted reports, repair candidates, and bundles after thirty days. Public archive caches are hash-verified, capped at 10 GiB, and evicted by LRU. Explicitly exported artifacts remain under user control. Repair candidates preserve their redacted request, patch, external observations, self-reported project-test outcome, provenance, and required input hashes. Authenticated bundles additionally preserve the inputs required for replay and their separate assurance labels. ### Phase 14: Recovery and current-source rechecks diff --git a/cso/SKILL.md.tmpl b/cso/SKILL.md.tmpl index 129f4faaf..9f936311a 100644 --- a/cso/SKILL.md.tmpl +++ b/cso/SKILL.md.tmpl @@ -45,6 +45,7 @@ Do not send findings, source, secrets, harnesses, or bundles to gbrain, telemetr | `--base ` | Select the comparison base, including for diff mode. | | `--budget ` | Bound wall-clock investigation time, including setup. | | `--offline` | Disable helper, scanner, download, and advisory network access. | +| `--paths ` | Audit only these repo prefixes (never a trimmed copy); others become snapshot-inputs gaps. | ## Mode Resolution @@ -64,10 +65,10 @@ Phases 0, 1, 12, 13, and 14 always run for an investigation. Select the remainin Diff mode may read unchanged callers, middleware, schemas, configuration, and dependencies needed for assessment. Report out-of-scope variants as follow-up scope. Historical-secret coverage uses only the helper's pinned base/snapshot. -Use `gstack-cso schema` for JSON inputs and `--help` for the installed contract. Below, `gstack-cso` means the trusted absolute launcher: +`gstack-cso schema` gives JSON inputs; `--help` the installed contract. Below, `gstack-cso` is the trusted absolute launcher: ```text -gstack-cso start --repo [scope/diff/base/budget/offline flags] [--comprehensive] +gstack-cso start --repo [scope/diff/base/budget/offline/paths] [--comprehensive] gstack-cso doctor --repo gstack-cso resume gstack-cso replay [--source ] @@ -110,7 +111,7 @@ Use the snapshot inventory and redacted source to map stacks, **actors, assets, Use stack detection to prioritize nested services and cross-language paths. Comprehensive setup supports Node, Bun, Python, and Rails only with a matching qualified runtime profile. Missing profiles, runtimes, or tools are execution prerequisites, not vulnerabilities. Reduce coverage only for unfinished assessment work. -In comprehensive mode, review snapshot transformations before reproduction. If sanitization removes or replaces the tested boundary, block reproduction. Never claim sanitized configuration equivalence without evidence. Readiness failures do not block static work. +In comprehensive mode, review snapshot transformations first; block reproduction if sanitization removed or replaced the tested boundary. Never claim sanitized configuration equivalence without evidence. Readiness failures do not block static work. ### Phase 1: Attack surface census @@ -128,7 +129,7 @@ This **CSO evidence rubric** governs CSO instead of shared review confidence ins Daily reports contain **supported findings**: a concrete attacker-controlled entrypoint, a path across an intended security boundary, demonstrated impact, and a challenge of relevant protective controls. Comprehensive reports retain unresolved candidates separately as **labeled hypotheses**, never mixed into supported totals. Disproved candidates are retained as disposition/coverage evidence, not vulnerabilities. -Do not apply blanket exclusions for development dependencies, availability/resource attacks, historical secrets, user-role prompt injection, or gstack-owned skills. Analyze attacker control and impact. UUIDs do not provide authorization; user-controlled URL paths can still cross a sensitive boundary; environment variables may originate from untrusted workflows; and safe defaults can be bypassed by framework escape hatches. Missing hardening alone needs a concrete failure scenario before becoming a finding. +Do not apply blanket exclusions for development dependencies, availability/resource attacks, historical secrets, user-role prompt injection, or gstack-owned skills. Analyze attacker control and impact. UUIDs are not authorization; user-controlled URL paths can cross a sensitive boundary; environment variables may come from untrusted workflows; framework escape hatches can bypass safe defaults. Missing hardening alone needs a concrete failure scenario before becoming a finding. For each candidate, use an already-authorized independent reviewer when available. Give it the relevant locations, invariant, and rubric without the producer's conclusion; have it inspect callers, middleware, configuration, validation, legitimate behavior, and mitigations. Use at most three; await them. Do not request broader tool access solely to obtain an independent reviewer. Otherwise perform a separate skeptical pass labeled **sequential challenge; independent agent unavailable**. Record dissent and assumptions. Agreement and scanner warnings do not prove runtime behavior. @@ -145,13 +146,13 @@ Keep finding evidence, reproduction outcome, patch validation, test-completion a ### Phase 13: Report and repair artifacts -Finish every audit through the helper, including empty, cancelled, blocked, or interrupted audits when possible. Every report begins with **complete**, **partial**, or **not assessed**, followed by scope and material gaps. Completeness is independent of finding count. For an empty supported set, say **“No supported findings in the assessed scope.”**, or **not assessed** if nothing was. Lead incomplete reports with the helper's Status/Ran/Reason/Next lines. Never infer a clean result from setup failure or absent scanner output. +Finish every audit through the helper, including empty, cancelled, blocked, or interrupted audits when possible. Every report begins with **complete**, **partial**, or **not assessed**, followed by scope and material gaps; a `--paths` run states partial snapshot-inputs coverage and summarizes skipped paths. Completeness is independent of finding count. For an empty supported set, say **“No supported findings in the assessed scope.”**, or **not assessed** if nothing was. Lead incomplete reports with the helper's Status/Ran/Reason/Next lines. Never infer a clean result from setup failure or absent scanner output. Present a compact SECURITY FINDINGS table with stable finding ID, severity, confidence/rationale, evidence state, location, and impact. Each finding needs an attacker scenario, supporting references, counterevidence considered, and a concrete repair recommendation. Include coverage, transformations, scanner versions/outcomes/freshness, runtime prerequisites, timing, and proposed repair-candidate paths. Include `runtime_tested` bundle paths only when an authenticated assertion witness actually produced one. Beside every bundle, show assertion, test-completion, and review assurance exactly as recorded; never collapse those labels or imply that self-reported project tests are authenticated. Comprehensive hypotheses belong in a separate labeled appendix. Use `RunReportV3`, `FindingV3`, `CoverageRecord`, `VerificationManifest`, and `RepairBundle` from `lib/cso/contracts.ts` through the installed schema command. Helper validation/persistence is mandatory. If saving fails, report **PERSISTENCE_FAILED** and a safe summary; never claim the report was saved. If redaction fails, withhold the payload entirely; do not show raw excerpts to explain the failure. -The private state namespace is `security/cso//` under the existing state root, outside synchronization allowlists. Execution copies and services are removed immediately. Sanitized snapshots expire after seven days; redacted reports, repair candidates, and bundles after thirty days. Public archive caches are hash-verified, capped at 10 GiB, and evicted by LRU. Explicitly exported artifacts remain under user control. Repair candidates preserve their redacted request, patch, external observations, self-reported project-test outcome, provenance, and required input hashes. Authenticated bundles additionally preserve the inputs required for replay and their separate assurance labels. Currently issued `runtime_tested` bundles record authenticated external assertions and `self_reported` project-test completion. +The private state namespace is `security/cso//` under the existing state root, outside synchronization allowlists. Execution copies and services are removed immediately. Sanitized snapshots expire after seven days; redacted reports, repair candidates, and bundles after thirty days. Public archive caches are hash-verified, capped at 10 GiB, and evicted by LRU. Explicitly exported artifacts remain under user control. Repair candidates preserve their redacted request, patch, external observations, self-reported project-test outcome, provenance, and required input hashes. Authenticated bundles additionally preserve the inputs required for replay and their separate assurance labels. ### Phase 14: Recovery and current-source rechecks diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b87e11067..2e7738dd0 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1204,8 +1204,18 @@ or too large. Each message names the cap that applied: withheld from the audit. - **Symlinks.** A tracked symlink anywhere in the repository stops the run. -**Fix.** No setting raises these caps. Run /cso on a smaller checkout that -holds the code you want audited and no symlinks. Add your file count or size to +**Fix.** No setting raises these caps. For the 64 MiB source cap, rerun with +`--paths` set to the comma-separated repo-relative directories you want audited +(for example `--paths src/core,src/mcp`). Paths outside them are never +captured or assessed and do not count toward the cap; snapshot and history +coverage then read **partial**, the report counts every skipped path, and the run's +snapshot manifest lists each one, so the verdict says what was not audited. +With `--diff`, every changed path outside dependency directories must be inside +`--paths`, or the run is refused. Do not audit a hand-trimmed copy: /cso would treat it as the whole +repository and report nothing as skipped. `--paths` does not lower the +manifest cap or the 100,000-entry limit, because every skipped path still has a +manifest entry (about 50,000 files fit). Symlinks must be removed. Add your +file count or size to [#2993](https://github.com/garrytan/gstack/issues/2993). diff --git a/lib/cso/cli.ts b/lib/cso/cli.ts index 91fad065e..6061afbd0 100644 --- a/lib/cso/cli.ts +++ b/lib/cso/cli.ts @@ -35,7 +35,15 @@ import { validateFinding, validateVerificationRequest, } from './contracts'; -import { capture, containedFile, assertSnapshot, readSnapshotManifest } from './snapshot'; +import { + capture, + containedFile, + assertSnapshot, + inAuditPaths, + noReadPath, + OUTSIDE_AUDIT_PATHS, + readSnapshotManifest, +} from './snapshot'; import { assertStateOutside, boundedList, @@ -167,7 +175,7 @@ const GENERATION_LOCK_FD = (() => { void GENERATION_LOCK_FD; const HELP = `gstack-cso ${VERSION} (helper ABI ${ABI}) Usage: - gstack-cso start --repo PATH [--comprehensive] [--diff] [--base REF] [--budget SECONDS] [--offline] [--infra|--code|--skills|--supply-chain|--owasp|--scope DOMAIN] + gstack-cso start --repo PATH [--comprehensive] [--diff] [--base REF] [--budget SECONDS] [--offline] [--paths PREFIX[,PREFIX...]] [--infra|--code|--skills|--supply-chain|--owasp|--scope DOMAIN] gstack-cso doctor --repo PATH gstack-cso provision-images [--setup-summary] [--per-image-seconds 5..300] gstack-cso resume RUN @@ -190,6 +198,7 @@ Usage: gstack-cso inspect-v2 IMPORT_ID gstack-cso schema +--paths audits only source under the listed repo-relative prefixes; every other path is never captured and is reported as a snapshot-inputs gap. Static runs never execute application code. Target and scanner execution is Docker-only and requires a qualified immutable catalog.`; const SCHEMA = { @@ -475,16 +484,28 @@ function planned(scope: string): CoverageRecord[] { // material gaps keep the largest share; findings and events keep the rest. const SNAPSHOT_DISCLOSURE_BUDGET = { gaps: 128 * 1024, transformations: 96 * 1024, exclusions: 32 * 1024 }; const UNLISTED = "to keep report.json within its 1 MiB bound; the run's snapshot manifest keeps every entry"; -function snapshotCoverage(manifest: Awaited>, scope: string): CoverageRecord { +function snapshotCoverage( + manifest: Awaited>, + scope: string, + auditPaths?: string[], +): CoverageRecord { const omitted = manifest.entries.filter((entry) => !entry.executionHash), excluded = omitted.filter((entry) => entry.transformation?.startsWith('excluded:')), excludedSet = new Set(excluded); // Classify every unexplained omission as a material coverage gap. Coverage // must not depend on transformation prose retaining a particular prefix. - const unread = omitted.filter((entry) => !excludedSet.has(entry)); - const deleted = manifest.deletedPaths ?? [], - captured = manifest.entries.length - omitted.length, - missing = unread.length + deleted.length; + // Out-of-scope --paths entries are listed last so they never crowd material + // in-scope gaps out of the bounded list. + const unread = omitted.filter((entry) => !excludedSet.has(entry)), + inScope = unread.filter((entry) => entry.transformation !== OUTSIDE_AUDIT_PATHS), + allDeleted = manifest.deletedPaths ?? [], + deleted = allDeleted.filter((item) => !auditPaths || inAuditPaths(item.path, auditPaths)), + outside = [ + ...unread.filter((entry) => entry.transformation === OUTSIDE_AUDIT_PATHS), + ...allDeleted.filter((item) => !deleted.includes(item)), + ]; + const captured = manifest.entries.length - omitted.length, + missing = unread.length + allDeleted.length; return { domain: 'snapshot-inputs', scope, @@ -493,12 +514,17 @@ function snapshotCoverage(manifest: Awaited>, scope: gaps: boundedList( missing, (index) => - index < unread.length - ? `${publicSnapshotPath(manifest, unread[index].path, unread[index].pathId).path}: in-scope source payload was unread and withheld from static and runtime assessment` - : `${publicSnapshotPath(manifest, deleted[index - unread.length].path, deleted[index - unread.length].pathId).path}: tracked source is deleted from the worktree; only retained history is available for assessment`, + index < inScope.length + ? `${publicSnapshotPath(manifest, inScope[index].path, inScope[index].pathId).path}: in-scope source payload was unread and withheld from static and runtime assessment` + : index < inScope.length + deleted.length + ? `${publicSnapshotPath(manifest, deleted[index - inScope.length].path, deleted[index - inScope.length].pathId).path}: tracked source is deleted from the worktree; only retained history is available for assessment` + : `${publicSnapshotPath(manifest, outside[index - inScope.length - deleted.length].path, outside[index - inScope.length - deleted.length].pathId).path}: outside the --paths audit scope; payload was not captured or assessed`, { bytes: SNAPSHOT_DISCLOSURE_BUDGET.gaps }, 4, - (count) => `${count} more unread or deleted in-scope inputs are not listed ${UNLISTED}`, + (count) => + outside.length + ? `${count} more unread, deleted or out-of-scope inputs are not listed ${UNLISTED}` + : `${count} more unread or deleted in-scope inputs are not listed ${UNLISTED}`, ), exclusions: boundedList( excluded.length, @@ -509,18 +535,22 @@ function snapshotCoverage(manifest: Awaited>, scope: (count) => `${count} more explicit exclusions are not listed ${UNLISTED}`, ), evidence: [ - `${captured} sanitized execution input${captured === 1 ? '' : 's'} captured; ${excluded.length} explicit non-executable exclusion${excluded.length === 1 ? '' : 's'}; ${unread.length} unread in-scope input${unread.length === 1 ? '' : 's'}; ${deleted.length} tracked deletion${deleted.length === 1 ? '' : 's'}`, + `${captured} sanitized execution input${captured === 1 ? '' : 's'} captured; ${excluded.length} explicit non-executable exclusion${excluded.length === 1 ? '' : 's'}; ${inScope.length} unread in-scope input${inScope.length === 1 ? '' : 's'}; ${deleted.length} tracked deletion${deleted.length === 1 ? '' : 's'}${auditPaths ? `; audit limited by --paths to ${auditPaths.join(', ')}, so ${outside.length} source file${outside.length === 1 ? ' was' : 's were'} skipped` : ''}`, ], }; } -function historyCoverage(status: any, scope: string): CoverageRecord { +function historyCoverage(status: any, scope: string, auditPaths?: string[]): CoverageRecord { return status?.status === 'captured' ? { domain: 'history-inputs', scope, - status: 'assessed', + status: auditPaths ? 'partial' : 'assessed', method: 'bounded helper-retained Git history', - gaps: [], + gaps: auditPaths + ? [ + `History outside the --paths audit scope (${auditPaths.join(', ')}) was not retained or assessed`, + ] + : [], exclusions: [], evidence: [ `Captured ${String(status.commits ?? 'bounded commits')} from ${String(status.range ?? 'the pinned source range')}`, @@ -550,11 +580,36 @@ function helperOwnedCoverage(domain: string): boolean { domain.startsWith('execution:') ); } +function auditPaths(raw: string): string[] { + const values = [...new Set(raw.split(',').map((value) => value.replace(/\/+$/, '')))].sort(); + if (raw.length > 1024 || values.length > 32) + throw new CsoError('INVALID_ARGUMENT', '--paths accepts at most 32 prefixes in 1024 characters'); + for (const value of values) + if ( + !value || + value.length > 256 || + value.startsWith('-') || + value.startsWith('/') || + /[\\\u0000-\u001f\u007f]/.test(value) || + value.split('/').some((part) => !part || part === '.' || part === '..') + ) + throw new CsoError( + 'INVALID_ARGUMENT', + `Invalid --paths entry ${JSON.stringify(value.slice(0, 80))}: use repo-relative path prefixes of at most 256 characters, without empty, '.' or '..' segments, a leading '-' or '/', backslashes, or control characters`, + ); + else if (sanitizeHelperForJson(value) !== value) + throw new CsoError( + 'INVALID_ARGUMENT', + '--paths entry would be altered by report redaction; choose a shorter or broader prefix', + ); + return values; +} function parseStart(args: string[]): { repo: string; policy: RunPolicy; comparisonBase?: string } { const repo = callerPath(need(args, '--repo')), comprehensive = take(args, '--comprehensive'), diff = take(args, '--diff'), - offline = take(args, '--offline'); + offline = take(args, '--offline'), + paths = args.includes('--paths') ? auditPaths(need(args, '--paths')) : undefined; const scopeFlags = ['--infra', '--code', '--skills', '--supply-chain', '--owasp'].filter((f) => args.includes(f), ); @@ -586,6 +641,7 @@ function parseStart(args: string[]): { repo: string; policy: RunPolicy; comparis budgetSeconds, maxWorkers: 3, maxRepairs: 3, + ...(paths ? { paths } : {}), }, ...(diff || explicitBase ? { comparisonBase: base } : {}), }; @@ -611,15 +667,40 @@ async function start( const run = newRun(repo); let manifest: Awaited>; try { - manifest = await capture(repo, run.dir, comparisonBase, requiredAncestor, { - deadlineMs: createdAt.getTime() + policy.budgetSeconds * 1000 - 60_000, - }); + manifest = await capture( + repo, + run.dir, + comparisonBase, + requiredAncestor, + { deadlineMs: createdAt.getTime() + policy.budgetSeconds * 1000 - 60_000 }, + policy.paths, + ); + if (policy.diff && policy.paths) { + const scoped = policy.paths, + outside = (manifest.changedPaths ?? []).filter( + (path) => !noReadPath(path) && !inAuditPaths(path, scoped), + ); + if (outside.length) + throw new CsoError( + 'INVALID_ARGUMENT', + `--diff found ${outside.length} changed path${outside.length === 1 ? '' : 's'} outside --paths (${scoped.join(', ')}): ${outside + .slice(0, 20) + .map((path) => publicSnapshotPath(manifest, path).path) + .join( + ', ', + )}${outside.length > 20 ? `, and ${outside.length - 20} more` : ''}. ${parent ? 'A recheck keeps the original --diff and --paths policy, so start a new audit with them added to --paths or without --diff' : 'Add them to --paths or run without --diff'}; a changed path is never left silently out of scope.`, + ); + } } catch (error) { fs.rmSync(run.dir, { recursive: true, force: true }); throw error; } - const transformed = manifest.entries.filter((entry) => entry.transformation), - deletedPaths = manifest.deletedPaths ?? []; + const transformed = manifest.entries.filter( + (entry) => entry.transformation && entry.transformation !== OUTSIDE_AUDIT_PATHS, + ), + deletedPaths = (manifest.deletedPaths ?? []).filter( + (item) => !policy.paths || inAuditPaths(item.path, policy.paths), + ); const report: RunReportV3 = { schemaVersion: 3, runId: run.runId, @@ -663,8 +744,8 @@ async function start( invariants: [], }, coverage: [ - snapshotCoverage(manifest, policy.scope), - historyCoverage(readJson(join(run.dir, 'history-status.json')), policy.scope), + snapshotCoverage(manifest, policy.scope, policy.paths), + historyCoverage(readJson(join(run.dir, 'history-status.json')), policy.scope, policy.paths), ...planned(policy.scope), ], findings: [], @@ -1342,6 +1423,11 @@ function submit(args: string[]) { 'INVALID_SCHEMA', `Diff-scope finding root cause is outside the captured changed paths: ${sourceFinding.location.path}`, ); + if (report.policy.paths && !inAuditPaths(sourceLocation.path, report.policy.paths)) + throw new CsoError( + 'INVALID_SCHEMA', + `Finding root cause is outside the --paths audit scope: ${sourceFinding.location.path}`, + ); const safeRaw = object(sanitizeForJson(raw), 'finding'), safeLocation = object(safeRaw.location, 'location'); safeLocation.path = publicSnapshotPath(manifest, sourceLocation.path).path; @@ -1543,6 +1629,12 @@ async function history(args: string[]) { if (args.length) selected = resolveSnapshotPath(manifest, args.shift(), false, 'History path', true).path; if (args.length) throw new CsoError('INVALID_ARGUMENT', 'history accepts at most one path or opaque handle'); + const scoped = loadReport(dir).policy.paths; + if (selected !== undefined && scoped && !inAuditPaths(selected, scoped)) + throw new CsoError( + 'INVALID_ARGUMENT', + `History path is outside the --paths audit scope (${scoped.join(', ')}); its history was not retained`, + ); const status = readJson(join(dir, 'history-status.json')); if (status.status !== 'captured' || !fs.existsSync(join(dir, 'history.txt'))) throw new CsoError('MISSING_INPUT', status.gap || 'Historical evidence was not retained'); @@ -1860,6 +1952,7 @@ async function recheck(args: string[], dependencies: CsoCliDependencies) { '--budget', String(original.policy.budgetSeconds), ...(original.policy.offline ? ['--offline'] : []), + ...(original.policy.paths ? ['--paths', original.policy.paths.join(',')] : []), ...(original.policy.scope === 'default' ? [] : original.policy.scope.startsWith('domain:') @@ -2522,7 +2615,15 @@ async function replay(args: string[], dependencies: CsoCliDependencies) { const temp = newRun(source); temporary = temp.dir; workDir = temp.dir; - manifest = await capture(source, workDir, undefined, undefined, { deadlineMs: replayDeadline }); + const scoped = loadReport(stored.dir).policy.paths; + manifest = await capture( + source, + workDir, + undefined, + undefined, + { deadlineMs: replayDeadline }, + scoped === undefined ? undefined : auditPaths(Array.isArray(scoped) ? scoped.join(',') : ''), + ); if ( manifest.executionHash !== bundle.requiredInputs.sourceHash || manifest.originalHash !== bundle.requiredInputs.originalHash diff --git a/lib/cso/contracts.ts b/lib/cso/contracts.ts index ad725df10..1f06b4565 100644 --- a/lib/cso/contracts.ts +++ b/lib/cso/contracts.ts @@ -122,6 +122,7 @@ export interface RunPolicy { budgetSeconds: number; maxWorkers: 3; maxRepairs: 3; + paths?: string[]; } export interface RunReportV3 { schemaVersion: 3; diff --git a/lib/cso/snapshot.ts b/lib/cso/snapshot.ts index 72805590c..8a498e67b 100644 --- a/lib/cso/snapshot.ts +++ b/lib/cso/snapshot.ts @@ -40,7 +40,8 @@ function containsDirectory(path: string, components: Set, sequences: str parts.slice(0, -1).some((_, index) => sequence.every((part, offset) => parts[index + offset] === part)), ); } -function noReadPath(path: string): boolean { +/** Host dependency trees: never read, copied, or counted, with or without --paths. */ +export function noReadPath(path: string): boolean { return containsDirectory(path, NO_READ_COMPONENTS, [['vendor', 'bundle']]); } function omittedPath(path: string): boolean { @@ -63,7 +64,12 @@ export function snapshotManifestCapMessage(entries: number, bytes: number): stri return `Snapshot manifest cap exceeded: ${entries} source entries need a ${mib(bytes)} (${bytes} byte) manifest; the snapshot manifest cap is ${mib(SNAPSHOT_MANIFEST_LIMIT)}. One entry is recorded for ${COUNTED_SOURCE}. Next step: run /cso on a smaller checkout of the code you want audited, and add your entry count to ${CAPACITY_ISSUE}. No supported workaround raises this cap.`; } export function sourceCapMessage(bytes: number, files: number): string { - return `64 MiB source cap exceeded: ${files} source files hold about ${mib(bytes)} (${bytes} bytes); the source admission cap is ${mib(SOURCE_LIMIT)}. The full size of ${COUNTED_SOURCE} counts, including files over 1 MiB whose payloads are withheld from the audit. Next step: run /cso on a smaller checkout of the code you want audited, and add your measured size to ${CAPACITY_ISSUE}. No supported workaround raises this cap.`; + return `64 MiB source cap exceeded: ${files} source files hold about ${mib(bytes)} (${bytes} bytes); the source admission cap is ${mib(SOURCE_LIMIT)}. The full size of ${COUNTED_SOURCE} inside the --paths audit scope counts (the whole repository when --paths is not set), including files over 1 MiB whose payloads are withheld from the audit. Next step: rerun with --paths set to the comma-separated repo-relative directories you want audited (narrower ones if --paths is already set); snapshot-inputs coverage is then partial, the report counts every skipped path, and the run's snapshot manifest lists each one. Do not audit a hand-trimmed copy, which hides what was skipped. Add your measured size to ${CAPACITY_ISSUE}. No supported workaround raises this cap.`; +} +/** Transformation recorded for a source path outside the start --paths prefixes; its payload is never copied or assessed. */ +export const OUTSIDE_AUDIT_PATHS = 'withheld: outside the --paths audit scope'; +export function inAuditPaths(path: string, prefixes: readonly string[]): boolean { + return prefixes.some((prefix) => path === prefix || path.startsWith(`${prefix}/`)); } export function readSnapshotManifest(runDir: string): SnapshotManifest { return readJson(join(runDir, 'snapshot.json'), SNAPSHOT_MANIFEST_LIMIT) as SnapshotManifest; @@ -367,7 +373,12 @@ export function assertOpenedFileContained(root: string, full: string, fd: number ) throw new CsoError('SNAPSHOT_RACE', 'Opened source identity changed during containment validation'); } -function readStable(root: string, path: string, maxBytes = MAX_OUTPUT): { data: Buffer; mode: number } { +function readStableWith( + root: string, + path: string, + maxBytes: number, + read: (fd: number, size: number) => { bytes: number; value: T }, +): { value: T; mode: number } { const ancestors = directoryIdentities(root, path), full = containedFile(root, path), named = fs.lstatSync(full); @@ -405,12 +416,7 @@ function readStable(root: string, path: string, maxBytes = MAX_OUTPUT): { data: 'MISSING_INPUT', `Source file exceeds the ${maxBytes}-byte snapshot admission limit: ${path}`, ); - const buffer = Buffer.alloc(Math.min(maxBytes + 1, before.size + 1)); - let bytes = 0, - count = 0; - while (bytes < buffer.length && (count = fs.readSync(fd, buffer, bytes, buffer.length - bytes, null)) > 0) - bytes += count; - const data = buffer.subarray(0, bytes), + const { bytes, value } = read(fd, before.size), after = fs.fstatSync(fd), current = fs.lstatSync(full); if ( @@ -427,11 +433,38 @@ function readStable(root: string, path: string, maxBytes = MAX_OUTPUT): { data: throw new CsoError('SNAPSHOT_RACE', `Source changed while reading: ${path}`); assertOpenedFileContained(root, full, fd, after); assertDirectoryIdentities(ancestors, path); - return { data, mode: before.mode & 0o777 }; + return { value, mode: before.mode & 0o777 }; } finally { fs.closeSync(fd); } } +function readStable(root: string, path: string, maxBytes = MAX_OUTPUT): { data: Buffer; mode: number } { + const { value, mode } = readStableWith(root, path, maxBytes, (fd, size) => { + const buffer = Buffer.alloc(Math.min(maxBytes + 1, size + 1)); + let bytes = 0, + count = 0; + while (bytes < buffer.length && (count = fs.readSync(fd, buffer, bytes, buffer.length - bytes, null)) > 0) + bytes += count; + return { bytes, value: buffer.subarray(0, bytes) }; + }); + return { data: value, mode }; +} +/** Stream a source file's Git blob hash in bounded chunks under readStable's identity checks. */ +function gitBlobStable(root: string, path: string, objectFormat: string, tick: () => void): string { + return readStableWith(root, path, Number.MAX_SAFE_INTEGER, (fd, size) => { + const hash = createHash(objectFormat).update(`blob ${size}\0`), + chunk = Buffer.alloc(Math.min(MAX_OUTPUT, size + 1)); + let bytes = 0, + count = 0; + while ((count = fs.readSync(fd, chunk, 0, chunk.length, null)) > 0) { + tick(); + bytes += count; + if (bytes > size) break; + hash.update(chunk.subarray(0, count)); + } + return { bytes, value: hash.digest('hex') }; + }).value; +} async function resolveHeadCommit(repo: string, home: string): Promise { try { return (await git(repo, ['rev-parse', '--verify', 'HEAD^{commit}'], home)).trim(); @@ -589,6 +622,7 @@ export async function capture( base?: string, requiredAncestor?: string, limits: SnapshotCaptureLimits = {}, + auditPaths?: readonly string[], ): Promise { repo = fs.realpathSync(repo); const state = fs.realpathSync(runDir), @@ -620,6 +654,10 @@ export async function capture( guard(); const list = await paths(repo, home, headCommit, admission); guard(); + const unmatched = auditPaths?.find((prefix) => !list.some((path) => inAuditPaths(path, [prefix]))); + if (unmatched !== undefined) + throw new CsoError('INVALID_ARGUMENT', `--paths entry matches no source file: ${unmatched}`); + const outsideAudit = (path: string) => auditPaths !== undefined && !inAuditPaths(path, auditPaths); await rejectSpecialFiles(repo, home, admission); guard(); const manifest: SnapshotManifest = { @@ -642,7 +680,7 @@ export async function capture( let measured = 0, files = 0; for (const candidate of list) { - if (noReadPath(candidate)) continue; + if (noReadPath(candidate) || outsideAudit(candidate)) continue; try { measured += fs.lstatSync(join(repo, candidate)).size; files++; @@ -662,20 +700,27 @@ export async function capture( } throw error; } // tracked deletions are represented by absence and the diff manifest - // Host dependency trees aren't copied or read. Their omission is still explicit. - if (noReadPath(path)) { + // Host dependency trees and paths outside --paths aren't copied, assessed, + // or counted toward the source cap. Their omission is still explicit. + const dependency = noReadPath(path); + if (dependency || outsideAudit(path)) { const stat = fs.lstatSync(containedFile(repo, path)); if (stat.isSymbolicLink() || !stat.isFile()) throw new CsoError('UNSAFE_PATH', `Special source input: ${path}`); gitModes.set(path, stat.mode & 0o111 ? '100755' : '100644'); - const reason = exclusion(path) ?? 'host dependency input'; + // A comparison base must still see every out-of-scope change, so the + // payload is hashed for changed-path detection but never retained. + if (!dependency && manifest.baseCommit) + gitHashes.set(path, gitBlobStable(repo, path, objectFormat, admission.time)); entries.push({ path, pathId: snapshotPathId(repo, path), originalHash: 'not-read', bytes: stat.size, mode: stat.mode & 0o777, - transformation: `excluded: ${reason}`, + transformation: dependency + ? `excluded: ${exclusion(path) ?? 'host dependency input'}` + : OUTSIDE_AUDIT_PATHS, }); guard(); continue; @@ -762,13 +807,15 @@ export async function capture( for (const e of entries) { guard(); if (e.originalHash === 'not-read') { - const current = fs.lstatSync(containedFile(repo, e.path)); + const current = fs.lstatSync(containedFile(repo, e.path)), + compared = gitHashes.get(e.path); if ( current.isSymbolicLink() || !current.isFile() || current.nlink !== 1 || current.size !== e.bytes || - (current.mode & 0o777) !== e.mode + (current.mode & 0o777) !== e.mode || + (compared !== undefined && gitBlobStable(repo, e.path, objectFormat, admission.time) !== compared) ) throw new CsoError('SNAPSHOT_RACE', `${message}: ${e.path}`); } else { @@ -843,6 +890,7 @@ export async function capture( '-p', range, '--', + ...(auditPaths ?? []).map((prefix) => `:(literal)${prefix}`), ], home, ); @@ -851,7 +899,7 @@ export async function capture( atomicWriteSync(join(runDir, 'history.txt'), safe, { mode: 0o600 }); writeJson(join(runDir, 'history-status.json'), { status: 'captured', - range, + range: auditPaths ? `${range}, limited to --paths ${auditPaths.join(', ')}` : range, commits: 'at most 100', bytes: Buffer.byteLength(safe), }); diff --git a/package.json b/package.json index 8a4ff0ce2..a4c43d2f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.111", + "version": "1.91.112", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/cso-audit-paths.test.ts b/test/cso-audit-paths.test.ts new file mode 100644 index 000000000..c4636487c --- /dev/null +++ b/test/cso-audit-paths.test.ts @@ -0,0 +1,347 @@ +// /cso --paths audit scope (field list 2, item 12): a repository over the source +// cap can be audited by naming the subtree to read, and every skipped path stays +// in the manifest unread and is reported as a snapshot-inputs gap. +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { OUTSIDE_AUDIT_PATHS, capture, snapshotManifestCapMessage, sourceCapMessage } from '../lib/cso/snapshot'; +import { newRun, readJson, saveReport } from '../lib/cso/state'; +import { MAX_OUTPUT, canonical, sha256, type VerificationObservation } from '../lib/cso/contracts'; +import { ISOLATION_POLICY_HASH } from '../lib/cso/docker'; +import { canonicalStartPlan, canonicalTestPlan, patchHash, verifyRepair } from '../lib/cso/verification'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const launcher = path.join(ROOT, 'bin', process.platform === 'win32' ? 'gstack-cso-launcher.exe' : 'gstack-cso-launcher'); +const MIB = 1024 * 1024; +let root = ''; +let state = ''; +const originalHome = process.env.GSTACK_HOME; + +function git(repo: string, ...args: string[]): string { + const result = spawnSync('/usr/bin/git', ['-C', repo, ...args], { encoding: 'utf8', env: { HOME: root, PATH: '/usr/bin:/bin' }, timeout: 60_000 }); + if (result.status) throw new Error(result.stderr); + return result.stdout; +} +function cso(repo: string, args: string[], home = state) { + return spawnSync(launcher, args, { cwd: repo, encoding: 'utf8', env: { HOME: root, GSTACK_HOME: home, PATH: '/usr/bin:/bin' }, timeout: 120_000, maxBuffer: 64 * MIB }); +} +function makeRepo(name: string, files: Record): string { + const repo = path.join(root, name); + fs.mkdirSync(repo); + git(repo, 'init', '-q'); + git(repo, 'config', 'user.email', 'fixture@example.test'); + git(repo, 'config', 'user.name', 'Fixture'); + for (const [file, body] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(repo, file)), { recursive: true }); + fs.writeFileSync(path.join(repo, file), body); + } + git(repo, 'add', '-A'); + git(repo, 'commit', '-qm', 'fixture'); + return repo; +} +function runDir(run: { repoId: string; runId: string }): string { + return path.join(state, 'security', 'cso', run.repoId, run.runId); +} +function listFiles(dir: string): string[] { + if (!fs.existsSync(dir)) return []; + return fs.readdirSync(dir, { recursive: true, withFileTypes: true }).filter((item) => item.isFile()).map((item) => path.relative(dir, path.join(item.parentPath, item.name)).split(path.sep).join('/')).sort(); +} +function runDirs(home: string): string[] { + const base = path.join(home, 'security', 'cso'); + if (!fs.existsSync(base)) return []; + return fs.readdirSync(base, { withFileTypes: true }).filter((repo) => repo.isDirectory()).flatMap((repo) => fs.readdirSync(path.join(base, repo.name), { withFileTypes: true }).filter((run) => run.isDirectory()).map((run) => `${repo.name}/${run.name}`)).sort(); +} +function error(result: ReturnType) { + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(''); + return JSON.parse(result.stderr).error; +} +const USERS = 'export const tenantQuery = (id:string) => db.user.findUnique({where:{id}})\n'; +const application = { actors: ['tenant user'], assets: ['tenant records'], entrypoints: ['GET /users/:id'], tenantBoundaries: ['tenant id'], sensitiveOperations: ['record read'], invariants: ['tenant isolation'] }; +const finding = { title: 'Cross-tenant user read', rootCause: 'Tenant query omits caller tenant predicate', location: { path: 'src/core/users.ts', line: 1, symbol: 'tenantQuery' }, advisoryIds: [], severity: 'high', confidence: 'high', confidenceRationale: 'The caller-to-query trace and missing tenant predicate directly support the finding', evidence: 'supported', attackerControl: 'Authenticated caller chooses the record ID', impact: 'Another tenant record is returned', scenario: 'A tenant supplies a known record ID owned by another tenant and receives that record', trace: ['GET /users/:id', 'tenantQuery', 'findUnique by id'], references: ['src/core/users.ts:1', 'OWASP API1:2023'], recommendation: 'Bind the lookup predicate to the authenticated tenant identifier', challenge: { reviewer: 'independent-2', independent: true, mode: 'independent_agent', callers: 'Authenticated route forwards the ID', controls: 'Authentication does not bind tenant', counterevidence: 'Opaque IDs reduce guessing but do not authorize', conclusion: 'The authorization invariant is absent' } }; + +beforeAll(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'cso-paths-')); + state = path.join(root, 'state'); + process.env.GSTACK_HOME = state; +}); +afterAll(() => { + if (originalHome === undefined) delete process.env.GSTACK_HOME; + else process.env.GSTACK_HOME = originalHome; + fs.rmSync(root, { recursive: true, force: true }); +}); + +describe('CSO --paths audit scope', () => { + test('only the source cap advises --paths; the manifest cap and entry limit, which still count skipped paths, do not', async () => { + const source = sourceCapMessage(346_199_207, 7_561); + expect(source).toContain('rerun with --paths'); + expect(source).toContain('counts every skipped path'); + expect(source).not.toContain('smaller checkout'); + const manifest = snapshotManifestCapMessage(61_234, 17_000_000); + expect(manifest).not.toContain('--paths'); + expect(manifest).toContain('No supported workaround raises this cap'); + + const repo = makeRepo('entry-limit-scoped', { 'src/core/users.ts': USERS, 'a.txt': 'a\n', 'b.txt': 'b\n' }); + const refused = await capture(repo, newRun(repo).dir, undefined, undefined, { maxEntries: 2 }, ['src/core']).then( + () => undefined, + (error) => error, + ); + expect(refused?.code).toBe('MISSING_INPUT'); + expect(refused?.message).toBe('Source tree exceeds the 2-entry snapshot admission limit'); + }); + + test('a repository over the source cap is admitted with --paths, and every skipped path is disclosed unread', () => { + const repo = makeRepo('over-cap-scoped', { + 'src/core/users.ts': USERS, + 'src/mcp/server.ts': 'export const serve = () => 1\n', + 'assets/a.bin': Buffer.alloc(40 * MIB, 1), + 'assets/b.bin': Buffer.alloc(25 * MIB, 1), + 'README.md': '# fixture\n', + 'src/core-utils/x.ts': 'export const lookalike = 1\n', + }); + const capped = error(cso(repo, ['start', '--repo', repo, '--offline'])).message; + expect(capped).toStartWith('64 MiB source cap exceeded:'); + expect(capped).toContain('rerun with --paths'); + + const started = cso(repo, ['start', '--repo', repo, '--offline', '--paths', 'src/mcp,src/core/,src/core']); + expect(started.stderr).toBe(''); + expect(started.status).toBe(0); + const run = JSON.parse(started.stdout), + dir = runDir(run), + report = readJson(path.join(dir, 'report.json')), + manifest = readJson(path.join(dir, 'snapshot.json'), 16 * MIB); + expect(report.policy.paths).toEqual(['src/core', 'src/mcp']); + + const skipped = ['README.md', 'assets/a.bin', 'assets/b.bin', 'src/core-utils/x.ts']; + for (const file of skipped) { + const entry = manifest.entries.find((item: any) => item.path === file); + expect(entry).toMatchObject({ originalHash: 'not-read', transformation: OUTSIDE_AUDIT_PATHS, bytes: fs.statSync(path.join(repo, file)).size }); + expect(entry.executionHash).toBeUndefined(); + } + expect(OUTSIDE_AUDIT_PATHS).toStartWith('withheld:'); + expect(listFiles(path.join(dir, 'snapshot'))).toEqual(['src/core/users.ts', 'src/mcp/server.ts']); + expect(listFiles(path.join(dir, 'readable'))).toEqual(['src/core/users.ts', 'src/mcp/server.ts']); + + const coverage = report.coverage.find((item: any) => item.domain === 'snapshot-inputs'); + expect(coverage.status).toBe('partial'); + expect(coverage.gaps).toEqual(skipped.map((file) => `${file}: outside the --paths audit scope; payload was not captured or assessed`)); + expect(coverage.evidence).toEqual(['2 sanitized execution inputs captured; 0 explicit non-executable exclusions; 0 unread in-scope inputs; 0 tracked deletions; audit limited by --paths to src/core, src/mcp, so 4 source files were skipped']); + expect(report.source.transformations).toEqual([]); + const history = fs.readFileSync(path.join(dir, 'history.txt'), 'utf8'); + expect(history).toContain('src/core/users.ts'); + expect(history).not.toContain('assets/a.bin'); + expect(history).not.toContain('src/core-utils/x.ts'); + const historyCoverage = report.coverage.find((item: any) => item.domain === 'history-inputs'); + expect(historyCoverage.evidence[0]).toContain('limited to --paths src/core, src/mcp'); + expect(historyCoverage).toMatchObject({ status: 'partial', gaps: ['History outside the --paths audit scope (src/core, src/mcp) was not retained or assessed'] }); + expect(cso(repo, ['history', run.runId, 'src/core/users.ts']).stdout).toContain('src/core/users.ts'); + expect(error(cso(repo, ['history', run.runId, 'assets/a.bin']))).toEqual({ code: 'INVALID_ARGUMENT', message: 'History path is outside the --paths audit scope (src/core, src/mcp); its history was not retained' }); + + const based = cso(repo, ['start', '--repo', repo, '--offline', '--base', 'HEAD', '--paths', 'src/core,src/mcp']); + expect(based.stderr).toBe(''); + expect(readJson(path.join(runDir(JSON.parse(based.stdout)), 'snapshot.json'), 16 * MIB).changedPaths).toEqual([]); + + fs.rmSync(path.join(repo, 'README.md')); + const deleted = cso(repo, ['start', '--repo', repo, '--offline', '--paths', 'src/core,src/mcp']); + expect(deleted.stderr).toBe(''); + const deletedReport = readJson(path.join(runDir(JSON.parse(deleted.stdout)), 'report.json')), + deletedCoverage = deletedReport.coverage.find((item: any) => item.domain === 'snapshot-inputs'); + expect(deletedReport.source.transformations).toEqual([]); + expect(deletedCoverage.gaps).toEqual([...skipped.slice(1), 'README.md'].map((file) => `${file}: outside the --paths audit scope; payload was not captured or assessed`)); + expect(deletedCoverage.evidence[0]).toContain('0 tracked deletions; audit limited by --paths to src/core, src/mcp, so 4 source files were skipped'); + }, 180_000); + + test('in-scope gaps are listed before skipped paths, so the bounded list never hides them', () => { + const files: Record = { 'z/app.ts': USERS, 'z/blob.dat': Buffer.from('a\0b') }; + for (let i = 0; i < 3000; i++) files[`a/long-out-of-scope-directory-name/file_number_${i}.ts`] = `export const v${i} = ${i};\n`; + const repo = makeRepo('gap-order', files), + started = cso(repo, ['start', '--repo', repo, '--offline', '--paths', 'z']); + expect(started.status).toBe(0); + const coverage = readJson(path.join(runDir(JSON.parse(started.stdout)), 'report.json')).coverage.find((item: any) => item.domain === 'snapshot-inputs'); + expect(coverage.gaps[0]).toBe('z/blob.dat: in-scope source payload was unread and withheld from static and runtime assessment'); + expect(coverage.gaps.at(-1)).toMatch(/^\d+ more unread, deleted or out-of-scope inputs are not listed/); + expect(coverage.evidence[0]).toContain('1 unread in-scope input;'); + expect(coverage.evidence[0]).toContain('so 3000 source files were skipped'); + }, 180_000); + + test('a comparison base hashes out-of-scope files in bounded chunks', async () => { + const repo = makeRepo('bounded-hash', { 'src/core/users.ts': USERS, 'assets/big.bin': Buffer.alloc(8 * MIB, 0x61) }); + const realAlloc = Buffer.alloc; + let largest = 0; + (Buffer as any).alloc = (size: number, ...rest: any[]) => { + largest = Math.max(largest, size); + return (realAlloc as any).call(Buffer, size, ...rest); + }; + try { + expect((await capture(repo, newRun(repo).dir, 'HEAD', undefined, {}, ['src/core'])).changedPaths).toEqual([]); + expect(largest).toBeLessThanOrEqual(MAX_OUTPUT); + fs.appendFileSync(path.join(repo, 'assets', 'big.bin'), 'x'); + expect((await capture(repo, newRun(repo).dir, 'HEAD', undefined, {}, ['src/core'])).changedPaths).toEqual(['assets/big.bin']); + } finally { + (Buffer as any).alloc = realAlloc; + } + }, 120_000); + + test.skipIf(process.platform === 'win32' || process.getuid?.() === 0)('capture never opens an out-of-scope payload when there is no comparison base', async () => { + const repo = makeRepo('unreadable-outside', { 'src/core/users.ts': USERS, 'secret/blob.txt': 'outside the audit\n' }); + fs.chmodSync(path.join(repo, 'secret', 'blob.txt'), 0o000); + try { + await expect(capture(repo, newRun(repo).dir)).rejects.toThrow('Source could not be opened as a regular file: secret/blob.txt'); + const run = newRun(repo), + manifest = await capture(repo, run.dir, undefined, undefined, {}, ['src/core']); + expect(manifest.entries.find((entry) => entry.path === 'secret/blob.txt')).toMatchObject({ originalHash: 'not-read', transformation: OUTSIDE_AUDIT_PATHS, bytes: 18 }); + expect(listFiles(path.join(run.dir, 'snapshot'))).toEqual(['src/core/users.ts']); + await expect(capture(repo, newRun(repo).dir, 'HEAD', undefined, {}, ['src/core'])).rejects.toThrow('Source could not be opened as a regular file: secret/blob.txt'); + } finally { + fs.chmodSync(path.join(repo, 'secret', 'blob.txt'), 0o644); + } + }); + + test('invalid --paths values are refused and leave no run', () => { + const repo = makeRepo('invalid-paths', { 'src/core/users.ts': USERS }); + const home = path.join(root, 'invalid-state'), + many = Array.from({ length: 33 }, (_, i) => `src/p${i}`).join(','); + for (const value of ['', ',', '../src', 'src/../x', '/src', '-src', 'src//core', 'src/./core', 'src\\core', 'src\ncore', many, 'a'.repeat(257), 'src,'.repeat(260)]) { + const refused = error(cso(repo, ['start', '--repo', repo, '--offline', '--paths', value], home)); + expect(refused.code).toBe('INVALID_ARGUMENT'); + } + const unmatched = error(cso(repo, ['start', '--repo', repo, '--offline', '--paths', 'src/core,src/typo'], home)); + expect(unmatched).toEqual({ code: 'INVALID_ARGUMENT', message: '--paths entry matches no source file: src/typo' }); + expect(error(cso(repo, ['start', '--repo', repo, '--paths'], home)).message).toBe('--paths requires a value'); + expect(error(cso(repo, ['start', '--repo', repo, '--offline', '--paths', 'rootfs/home/user/app'], home))).toEqual({ code: 'INVALID_ARGUMENT', message: '--paths entry would be altered by report redaction; choose a shorter or broader prefix' }); + expect(error(cso(repo, ['start', '--repo', repo, '--offline', '--paths', 'mirror/actions/runs/42'], home)).message).toBe('--paths entry would be altered by report redaction; choose a shorter or broader prefix'); + expect(runDirs(home)).toEqual([]); + }, 120_000); + + test('--diff refuses changed paths outside --paths and compares unchanged ones without reporting them', () => { + const repo = makeRepo('diff-scope', { 'src/core/users.ts': USERS, 'src/other/a.ts': 'export const a = 1\n', 'docs/b.md': 'b\n', 'AGENTS.md': '# agents\n', 'node_modules/dep/index.js': 'module.exports = 1\n' }), + base = git(repo, 'rev-parse', 'HEAD').trim(), + home = path.join(root, 'diff-state'), + start = () => cso(repo, ['start', '--repo', repo, '--offline', '--diff', '--base', base, '--paths', 'src/core'], home); + + fs.writeFileSync(path.join(repo, 'src/core/users.ts'), USERS + '// changed\n'); + fs.writeFileSync(path.join(repo, 'node_modules/dep/index.js'), 'module.exports = 2\n'); + const accepted = start(); + expect(accepted.stderr).toBe(''); + expect(accepted.status).toBe(0); + const run = JSON.parse(accepted.stdout), + manifest = readJson(path.join(home, 'security', 'cso', run.repoId, run.runId, 'snapshot.json'), 16 * MIB); + expect(manifest.changedPaths).toEqual(['node_modules/dep/index.js', 'src/core/users.ts']); + + fs.writeFileSync(path.join(repo, 'AGENTS.md'), '# agents, edited\n'); + fs.writeFileSync(path.join(repo, '.npmrc'), 'registry=https://registry.example.test/\n'); + expect(error(start()).message).toContain('--diff found 2 changed paths outside --paths (src/core): .npmrc, AGENTS.md.'); + git(repo, 'checkout', '--', 'AGENTS.md'); + fs.rmSync(path.join(repo, '.npmrc')); + + fs.writeFileSync(path.join(repo, 'src/other/a.ts'), 'export const a = 2\n'); + const modified = error(start()); + expect(modified.code).toBe('INVALID_ARGUMENT'); + expect(modified.message).toContain('--diff found 1 changed path outside --paths (src/core): src/other/a.ts.'); + git(repo, 'checkout', '--', 'src/other/a.ts'); + + fs.rmSync(path.join(repo, 'docs/b.md')); + fs.writeFileSync(path.join(repo, 'new.ts'), 'export const n = 1\n'); + const added = error(start()); + expect(added.message).toContain('--diff found 2 changed paths outside --paths (src/core): docs/b.md, new.ts.'); + fs.mkdirSync(path.join(repo, 'many')); + for (let i = 0; i < 21; i++) fs.writeFileSync(path.join(repo, 'many', `f${String(i).padStart(2, '0')}.ts`), `export const f = ${i}\n`); + expect(error(start()).message).toContain('--diff found 23 changed paths outside --paths (src/core): docs/b.md, many/f00.ts'); + expect(error(start()).message).toContain(', and 3 more. Add them to --paths or run without --diff'); + expect(runDirs(home)).toEqual([`${run.repoId}/${run.runId}`]); + }, 180_000); + + test('recheck rebuilds the original --paths scope', () => { + const repo = makeRepo('recheck-scope', { 'src/core/users.ts': USERS, 'tools/build.ts': 'export const build = 1\n' }), + started = cso(repo, ['start', '--repo', repo, '--scope', 'auth', '--offline', '--paths', 'src/core']); + expect(started.status).toBe(0); + const run = JSON.parse(started.stdout), + dir = runDir(run), + initial = readJson(path.join(dir, 'report.json')), + evidence = path.join(root, 'recheck-evidence.json'); + fs.writeFileSync(evidence, JSON.stringify({ application, findings: [finding], coverage: initial.coverage.filter((item: any) => !['snapshot-inputs', 'history-inputs'].includes(item.domain)).map((item: any) => ({ ...item, status: 'assessed', method: 'fresh caller and boundary trace', gaps: [], evidence: ['current captured source'] })), gaps: [] })); + const outside = path.join(root, 'outside-finding.json'); + fs.writeFileSync(outside, JSON.stringify({ findings: [{ ...finding, location: { ...finding.location, path: 'tools/build.ts' }, references: ['tools/build.ts:1'] }] })); + const refusedFinding = error(cso(repo, ['submit', run.runId, outside])); + expect(refusedFinding).toEqual({ code: 'INVALID_SCHEMA', message: 'Finding root cause is outside the --paths audit scope: tools/build.ts' }); + expect(cso(repo, ['submit', run.runId, evidence]).status).toBe(0); + expect(cso(repo, ['finish', run.runId]).status).toBe(0); + const final = readJson(path.join(dir, 'report.json')); + expect(final.findings).toHaveLength(1); + const rechecked = cso(repo, ['recheck', final.findings[0].id, '--repo', repo]); + expect(rechecked.stderr).toBe(''); + expect(rechecked.status).toBe(0); + const childRun = JSON.parse(rechecked.stdout), + child = readJson(path.join(runDir({ ...childRun, repoId: run.repoId }), 'report.json')); + expect(child.policy.paths).toEqual(['src/core']); + expect(child.coverage.find((item: any) => item.domain === 'snapshot-inputs').gaps).toEqual(['tools/build.ts: outside the --paths audit scope; payload was not captured or assessed']); + const claim = path.join(root, 'withheld-claim.json'); + fs.writeFileSync(claim, JSON.stringify({ recheck: { findingId: final.findings[0].id, outcome: 'resolved', rootCause: finding.rootCause, evidence: [{ kind: 'caller', path: 'tools/build.ts', line: 1, observation: 'A caller outside the audit scope' }, { kind: 'security_boundary', path: 'src/core/users.ts', line: 1, observation: 'Fresh boundary trace' }] } })); + expect(error(cso(repo, ['submit', childRun.runId, claim]))).toEqual({ code: 'INVALID_SCHEMA', message: 'Recheck evidence must reference freshly captured readable source' }); + const base = git(repo, 'rev-parse', 'HEAD').trim(); + fs.appendFileSync(path.join(repo, 'src/core/users.ts'), '// reviewed change\n'); + const diffStarted = cso(repo, ['start', '--repo', repo, '--scope', 'auth', '--offline', '--diff', '--base', base, '--paths', 'src/core']); + expect(diffStarted.stderr).toBe(''); + const diffRun = JSON.parse(diffStarted.stdout), + diffCoverage = readJson(path.join(runDir(diffRun), 'report.json')).coverage.filter((item: any) => !['snapshot-inputs', 'history-inputs'].includes(item.domain)); + fs.writeFileSync(evidence, JSON.stringify({ application, findings: [finding], coverage: diffCoverage.map((item: any) => ({ ...item, status: 'assessed', method: 'fresh caller and boundary trace', gaps: [], evidence: ['current captured source'] })), gaps: [] })); + expect(cso(repo, ['submit', diffRun.runId, evidence]).status).toBe(0); + expect(cso(repo, ['finish', diffRun.runId]).status).toBe(0); + fs.writeFileSync(path.join(repo, 'tools/build.ts'), 'export const build = 2\n'); + expect(error(cso(repo, ['recheck', final.findings[0].id, '--run', diffRun.runId, '--repo', repo])).message).toContain('--diff found 1 changed path outside --paths (src/core): tools/build.ts. A recheck keeps the original --diff and --paths policy, so start a new audit with them added to --paths or without --diff;'); + git(repo, 'checkout', '--', '.'); + git(repo, 'mv', 'src/core', 'src/kernel'); + git(repo, 'commit', '-qm', 'rename the audited directory'); + expect(error(cso(repo, ['recheck', final.findings[0].id, '--run', run.runId, '--repo', repo]))).toEqual({ code: 'INVALID_ARGUMENT', message: '--paths entry matches no source file: src/core' }); + }, 180_000); + + test.skipIf(process.platform === 'win32')('replay from supplied source recaptures with the original --paths scope', async () => { + const repo = makeRepo('replay-scope', { + 'package.json': JSON.stringify({ name: 'cso-paths-replay', version: '1.0.0', private: true, scripts: { start: 'node app.js', test: 'node --test' } }) + '\n', + 'package-lock.json': JSON.stringify({ name: 'cso-paths-replay', version: '1.0.0', lockfileVersion: 3, requires: true, packages: { '': { name: 'cso-paths-replay', version: '1.0.0' } } }) + '\n', + 'app.js': 'module.exports = "vulnerable"\n', + 'app.test.js': "const test=require('node:test');test('legitimate control remains available',()=>{});\n", + 'docs/notes.md': 'outside the audit\n', + }); + const started = cso(repo, ['start', '--repo', repo, '--comprehensive', '--offline', '--paths', 'app.js,app.test.js,package.json,package-lock.json']); + expect(started.stderr).toBe(''); + const run = JSON.parse(started.stdout), + dir = runDir(run), + snapshot = path.join(dir, 'snapshot'), + manifest = readJson(path.join(dir, 'snapshot.json'), 16 * MIB); + expect(manifest.entries.find((entry: any) => entry.path === 'docs/notes.md').transformation).toBe(OUTSIDE_AUDIT_PATHS); + const recordFinding = { ...finding, location: { path: 'app.js', line: 1, symbol: 'recordLookup' }, references: ['app.js:1', 'OWASP API1:2023'] }, + submission = path.join(root, 'replay-finding.json'); + fs.writeFileSync(submission, JSON.stringify({ findings: [recordFinding] })); + expect(cso(repo, ['submit', run.runId, submission]).status).toBe(0); + const report = readJson(path.join(dir, 'report.json')), + start = canonicalStartPlan(snapshot, 'node', 3456), + tests = canonicalTestPlan(snapshot, 'node'), + arch = process.arch === 'arm64' ? 'arm64' : 'amd64'; + const request: any = { findingId: report.findings[0].id, runtimeProfile: `node-24.4.0-${arch}`, port: 3456, start: start.command, legitimate: [{ name: 'legitimate control', path: '/control', method: 'GET', expected: { status: 200, includes: 'CONTROL_OK' } }], security: { name: 'unauthorized record is denied', path: '/records/other', method: 'GET', expected: { status: 403, includes: 'DENIED' }, vulnerable: { status: 200, includes: 'RECORD' } }, existingTests: tests.commands, fixtures: {}, boundaryFiles: ['app.js'], testFiles: tests.files, changes: [{ path: 'app.js', beforeSha256: sha256(fs.readFileSync(path.join(snapshot, 'app.js'))), after: 'module.exports = "fixed"\n', effect: 'source' }], review: { reviewer: 'independent-reviewer', independent: true, rootCauseRepaired: true, featurePreserved: true, boundaryMocks: false, rationale: 'The caller predicate is added while the legitimate control and existing test remain unchanged.', reviewedPatchHash: '' } }; + request.review.reviewedPatchHash = patchHash(request); + const tap = 'TAP version 13\n# Subtest: legitimate control remains available\nok 1 - legitimate control remains available\n ---\n duration_ms: 1\n ...\n1..1\n# tests 1\n# suites 0\n# pass 1\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n', + runtime: any = { id: request.runtimeProfile, stack: 'node', platform: `linux/${arch}`, image: `ghcr.io/garrytan/gstack/cso-staging/node-${arch}@sha256:${'b'.repeat(64)}` }, + executor = { + observe: async (_source: string, phase: 'before' | 'after', received: any, _runtime: any, _verifier: any, _work: string, _control: string, _execution: any, evidence: any, witness: any) => { + const observation: VerificationObservation = { booted: true, legitimate: true, security: phase === 'before' ? 'intended_failure' : 'pass', existingTests: false, output: `external ${phase} assertions passed`, inputHash: '' }, + receipt = await witness.attest(observation, [{ command: received.existingTests[0], code: 0, output: tap, minimumPassingTests: evidence.minimumPassingTests[0] }]); + return { observation: { ...observation, existingTests: receipt.diagnosticTestsPassed, inputHash: witness.binding.harnessHash }, witness: receipt }; + }, + }; + const verified = await verifyRepair({ runId: run.runId, runDir: dir, manifest, rawRequest: request, runtime, verifier: runtime, policyHash: ISOLATION_POLICY_HASH, auditPolicyHash: sha256(canonical(report.policy)), archives: [], executor } as any); + report.findings[0].reproduction = 'reproduced'; + report.findings[0].repair = 'runtime_tested'; + report.findings[0].verificationId = verified.bundle.id; + report.findings[0].verificationAssurance = { assertions: 'authenticated_out_of_process', testCompletion: 'self_reported', review: 'self_attested' }; + saveReport(dir, report); + fs.rmSync(snapshot, { recursive: true, force: true }); + fs.rmSync(path.join(dir, 'readable'), { recursive: true, force: true }); + const supplied = cso(repo, ['replay', verified.bundle.id, '--source', repo]); + expect(supplied.status).not.toBe(0); + expect(supplied.stderr).not.toContain('does not match the bundle input hashes'); + expect(supplied.stderr).toContain('MISSING_QUALIFIED_RUNTIME'); + }, 60_000); +}); From 73f1394b0a02af88b3c5d392100f2ede1e53f7e5 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 16 Oct 2026 18:52:40 -0700 Subject: [PATCH 2/2] v1.91.113.0 fix(capy): trailers say behind= with a fix, a skipped upgrade is never ok, and a reused pid no longer holds the upgrade lease (#3232) * fix(capy): the install trailer says behind= and its fix, so a stale install shows in every relayed trailer (field list 2 item 13, #2378) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(capy): a lease records its holder's start identity, so a pid reused after a wake is reclaimed; a skipped session-start upgrade ends fail behind, never ok (GBRA-105, GBRA-106) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(lease): stale needs proof (unreadable or malformed identities keep the lease), ps identity is timezone-independent, and reclaim never deletes a concurrent renewal Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(lease): one directory lock for publish, prune and reclaim; reclaim rereads the record judged stale, so a renewal is never deleted or hidden Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(lease): the directory lock names its holder; broken only when the holder is dead or its pid reused, unlock removes only its own lock, touch exits 3 under a live holder Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(lease): breakers serialize on .lock.break and reread the lock before removing it; gstack-skill-start reports a lease the held lock kept out Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(lease): name the double-crash limit of breaking a dead lock Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(third-party-actions): the Aside drive-offer detector matches the product name, not the English idiom (paid run 38988927989 tpa-absent-linux false positive) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): field list 2 F1 entry; capy/support.env GSTACK_MIN_REVISION to this release Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.113.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 16 +++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- bin/gstack-capy-install | 26 +++- bin/gstack-capy-session | 35 ++++- bin/gstack-lease | 128 ++++++++++++++++-- bin/gstack-skill-start | 8 +- capy/skill/gstack/SKILL.md | 2 +- capy/support.env | 2 +- docs/capy-install.md | 47 +++++-- docs/troubleshooting.md | 13 +- package.json | 2 +- test/capy-skill-wrapper.test.ts | 83 +++++++++++- test/gstack-capy-install.test.ts | 21 +++ test/gstack-lease.test.ts | 196 +++++++++++++++++++++++++++- test/gstack-upgrade-bin.test.ts | 2 +- test/helpers/third-party-actions.ts | 2 +- test/third-party-actions.test.ts | 3 + 18 files changed, 543 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed442b456..02c4d5156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## [1.91.113.0] - 2026-10-17 + +**A Capy install that has fallen behind now says so in the one line lanes relay, a skipped upgrade is never reported ok, and a reused pid no longer blocks every upgrade.** Lanes pass along only the `gstack:` trailer, so a session that was 12 commits behind origin looked current. Two field reports (GBRA-105, GBRA-106) showed session-start upgrades wedged behind a lease whose pid now belonged to dbus-daemon or systemd-logind, while the trailer still said ok. + +### What this means for you + +- **The ok trailer carries the gap and the fix:** `gstack: ok for= (behind= (latest: ); fix: )`. The fix is the command that actually closes the gap: a rerun, `gstack-upgrade` for a dirty, detached or diverged checkout, or `unset ` when upgrades are switched off. Run it. +- **A skipped session-start upgrade fails.** When the upgrade is busy, refused or missing, install mode ends `gstack: fail behind — … (UPGRADE_BUSY|UPGRADE_REFUSED)` with exit 1, and workflows refuse until the fix runs. An explicit opt-out stays ok with the `behind=` note. +- **A lease records who holds it.** `gstack-lease` stores the holder's start identity (boot id plus start time, or `ps` lstart). A lease whose pid now belongs to another process is reclaimed, with a `gstack-lease: reclaimed stale lease …` line. `live`, UPGRADE_BUSY and `fail behind` name each holder as `pid

(m )`. +- **The supported minimum rises to this release**, so older Capy checkouts are fast-forwarded at install-mode session start. + +### For contributors + +- The lease directory has a lock: a symlink naming its holder, broken only when that holder is dead or its pid was reused. Breakers serialize on `.lock.break` and reread before removing. While a live process holds the lock, `touch` exits 3, and `gstack-skill-start` reports that the session holds no workflow lease. Known limit: two gstack-lease processes killed in a row can still let a reclaim delete a renewal. +- The paid-gate Aside drive-offer detector (`test/helpers/third-party-actions.ts`) now matches the product name only. The English "set this aside" was a false positive in paid run 38988927989. + ## [1.91.112.0] - 2026-10-17 **`/cso --paths` audits part of a repository and says exactly what it skipped.** A repository over /cso's 64 MiB source cap used to get one piece of advice, "run /cso on a smaller checkout". The trimmed copy then read as the whole repository, and the verdict never said what was left out. `gstack-cso start --paths src/core,src/mcp` now audits only those prefixes and reports every other path as skipped, with snapshot and history coverage marked partial. diff --git a/VERSION b/VERSION index 4e01f37aa..e1375eb2a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.91.112.0 +1.91.113.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 945d1da1f..e68aa64c6 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.91.112.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.91.113.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/bin/gstack-capy-install b/bin/gstack-capy-install index 0b71d6577..a7a2205bc 100755 --- a/bin/gstack-capy-install +++ b/bin/gstack-capy-install @@ -521,11 +521,18 @@ EOF_SHIM # 4. Certify. The trailer is the contract, in one grammar: # gstack: ok for=[ (requested: …; pins: skipped (planning-only|no project); latest: available)] +# gstack: ok for= ([requested: …; ][pins: …; ]behind= (latest: ); fix: ) # Notes, in that fixed order, ride only on an ok trailer: `requested` when the # existing checkout is not at --revision (it is never moved here), `pins` when -# the doctor skipped the project pins, `latest` when the bootstrapper read a -# newer VERSION from origin in this run, exported as GSTACK_CAPY_REMOTE_VERSION. The -# failure head (`gstack: fail `) is unchanged. +# the doctor skipped the project pins, then either `behind= (latest: )` +# and `fix: ` when the session ends behind origin's default branch +# (bin/gstack-capy-session exports GSTACK_CAPY_BEHIND, _BEHIND_TO, _BEHIND_FIX), or +# `latest` when the bootstrapper read a newer VERSION from origin in this run, +# exported as GSTACK_CAPY_REMOTE_VERSION. The failure head (`gstack: fail `) +# is unchanged. An install-mode session whose self-upgrade was skipped +# (the session exports GSTACK_CAPY_BEHIND_BLOCKED) is never ok: its ok trailer becomes +# gstack: fail behind — installed is commits behind origin (latest ) +# and the session-start upgrade was skipped: ; fix: () INSTALLED=$(head -1 "$ROOT/VERSION") TRAILER="$(printf '%s\n' "$OUT" | grep '^gstack: ' | tail -1)" NOTES="" @@ -538,11 +545,20 @@ if printf '%s\n' "$OUT" | grep -q '^SKIP pins (planning-only'; then note "pins: elif printf '%s\n' "$OUT" | grep -Eq '^SKIP pins \((no project|project directory )'; then note "pins: skipped (no project)" fi REMOTE_VERSION="${GSTACK_CAPY_REMOTE_VERSION:-}" -if [ -n "$REMOTE_VERSION" ] && [ "$REMOTE_VERSION" != "$INSTALLED" ] \ +if [ -n "${GSTACK_CAPY_BEHIND:-}" ]; then + note "behind=$GSTACK_CAPY_BEHIND (latest: ${GSTACK_CAPY_BEHIND_TO:-${REMOTE_VERSION:-unknown}})" + note "fix: ${GSTACK_CAPY_BEHIND_FIX:-bash /scripts/install.sh}" +elif [ -n "$REMOTE_VERSION" ] && [ "$REMOTE_VERSION" != "$INSTALLED" ] \ && [ "$(printf '%s\n%s\n' "$INSTALLED" "$REMOTE_VERSION" | sort -V | head -1)" = "$INSTALLED" ]; then note "latest: $REMOTE_VERSION available" fi -case "$TRAILER" in "gstack: ok "*) [ -n "$NOTES" ] && TRAILER="$TRAILER ($NOTES)" ;; esac +BLOCKED="${GSTACK_CAPY_BEHIND_BLOCKED:-}" +case "$TRAILER" in "gstack: ok "*) if [ -n "$BLOCKED" ]; then + case "$BLOCKED" in *"; fix: "*) ;; *) BLOCKED="$BLOCKED; fix: ${GSTACK_CAPY_BEHIND_FIX:-$ROOT/bin/gstack-upgrade}" ;; esac + case "$BLOCKED" in *"("[A-Z]*_[A-Z]*")") ;; *) BLOCKED="$BLOCKED (UPGRADE_REFUSED)" ;; esac + TRAILER="gstack: fail behind — installed $INSTALLED is ${GSTACK_CAPY_BEHIND:-?} commits behind origin (latest ${GSTACK_CAPY_BEHIND_TO:-unknown}) and the session-start upgrade was skipped: $BLOCKED" + RC=1 +elif [ -n "$NOTES" ]; then TRAILER="$TRAILER ($NOTES)"; fi ;; esac record_verdict "$TRAILER" "$([ "$RC" -eq 0 ] && echo 1 || echo 0)" [ "$CHECK_ONLY" -eq 1 ] || write_shim printf '%s\n' "$OUT" | grep -v '^gstack: ' diff --git a/bin/gstack-capy-session b/bin/gstack-capy-session index ac04de7ac..017af4685 100755 --- a/bin/gstack-capy-session +++ b/bin/gstack-capy-session @@ -28,6 +28,15 @@ # GSTACK_CAPY_AUTO_UPGRADE=0>)`. # --check never moves the checkout; behind origin it prints # `gstack-capy-session: behind origin/: -> ( commits); fix: ...`. +# Whenever the session ends behind origin (--check, or a self-upgrade that was +# skipped or disabled) it exports GSTACK_CAPY_BEHIND=, GSTACK_CAPY_BEHIND_TO= and GSTACK_CAPY_BEHIND_FIX= for the installer's trailer notes +# `behind= (latest: ); fix: `. The command never contains +# "; " (the note separator): after a refused self-upgrade it is bin/gstack-upgrade, +# which prints its own refusal and fix. A skipped self-upgrade (refused, busy, +# or gstack-upgrade missing) also exports GSTACK_CAPY_BEHIND_BLOCKED=, and the installer turns an ok trailer into `gstack: fail behind`: a +# session that meant to upgrade and could not is never certified ok. # # Usage: gstack-capy-session [--check] --root [gstack-capy-install arguments] # --check certify only (bin/gstack-capy-install --check: the doctor with the @@ -65,18 +74,36 @@ behind() { git -C "$ROOT" rev-parse -q --verify FETCH_HEAD >/dev/null 2>&1 || return 1 ! git -C "$ROOT" merge-base --is-ancestor FETCH_HEAD HEAD 2>/dev/null } -disabled() { echo "SELF_UPGRADE: disabled ($1)"; } # VAR — the switch that turned the fast-forward off +disabled() { echo "SELF_UPGRADE: disabled ($1)"; behind && report_behind "unset $1, then $RERUN"; } # VAR — the switch that turned the fast-forward off +# report_behind FIX — hand the installer the gap and the command that closes +# it, for the trailer's `behind=` note (lanes relay only the trailer). +report_behind() { + export GSTACK_CAPY_BEHIND="$(git -C "$ROOT" rev-list --count HEAD..FETCH_HEAD 2>/dev/null || echo '?')" \ + GSTACK_CAPY_BEHIND_TO="$(git -C "$ROOT" show FETCH_HEAD:VERSION 2>/dev/null | head -1)" GSTACK_CAPY_BEHIND_FIX="$1" +} +unset GSTACK_CAPY_BEHIND GSTACK_CAPY_BEHIND_TO GSTACK_CAPY_BEHIND_FIX GSTACK_CAPY_BEHIND_BLOCKED +RERUN="bash ${GSTACK_CAPY_WRAPPER_PATH:-/scripts/install.sh}${SKILL:+ --for $SKILL}" if [ "$CHECK" -eq 1 ]; then if behind; then + # The rerun fast-forwards only what gstack-upgrade would move; otherwise + # gstack-upgrade itself names the refusal and its fix. + if [ "${GSTACK_CAPY_SELF_UPGRADE:-}" = off ]; then report_behind "unset GSTACK_CAPY_SELF_UPGRADE, then $RERUN" + elif [ "${GSTACK_CAPY_AUTO_UPGRADE:-1}" = 0 ]; then report_behind "unset GSTACK_CAPY_AUTO_UPGRADE, then $RERUN" + elif [ "$(git -C "$ROOT" symbolic-ref -q --short HEAD 2>/dev/null)" != "$(sed -n "1s/.*branch '\([^']*\)' of .*/\1/p" "$(git -C "$ROOT" rev-parse --absolute-git-dir 2>/dev/null)/FETCH_HEAD" 2>/dev/null)" ] \ + || [ -n "$(git -C "$ROOT" status --porcelain --untracked-files=no 2>/dev/null)" ] \ + || ! git -C "$ROOT" merge-base --is-ancestor HEAD FETCH_HEAD 2>/dev/null; then report_behind "$ROOT/bin/gstack-upgrade" + else report_behind "$RERUN" + fi BRANCH=$(git -C "$ROOT" symbolic-ref -q --short HEAD 2>/dev/null || echo HEAD) - echo "gstack-capy-session: behind origin/$BRANCH: $(head -1 "$ROOT/VERSION" 2>/dev/null) -> $(git -C "$ROOT" show FETCH_HEAD:VERSION 2>/dev/null | head -1) ($(git -C "$ROOT" rev-list --count HEAD..FETCH_HEAD 2>/dev/null || echo '?') commits); fix: bash ${GSTACK_CAPY_WRAPPER_PATH:-/scripts/install.sh}${SKILL:+ --for $SKILL} (fast-forwards)" + echo "gstack-capy-session: behind origin/$BRANCH: $(head -1 "$ROOT/VERSION" 2>/dev/null) -> $GSTACK_CAPY_BEHIND_TO ($GSTACK_CAPY_BEHIND commits); fix: $GSTACK_CAPY_BEHIND_FIX$([ "$GSTACK_CAPY_BEHIND_FIX" = "$RERUN" ] && echo ' (fast-forwards)')" fi elif [ "${GSTACK_CAPY_SELF_UPGRADED:-}" != 1 ]; then if [ "${GSTACK_CAPY_SELF_UPGRADE:-}" = off ]; then disabled GSTACK_CAPY_SELF_UPGRADE elif [ "${GSTACK_CAPY_AUTO_UPGRADE:-1}" = 0 ]; then disabled GSTACK_CAPY_AUTO_UPGRADE elif [ -z "${GSTACK_CAPY_REMOTE_MIN:-}" ] || [ "$GSTACK_CAPY_REMOTE_MIN" = unknown ]; then echo "SELF_UPGRADE: none (remote minimum unknown)" elif ! behind; then echo "SELF_UPGRADE: none (current)" - elif [ ! -x "$ROOT/bin/gstack-upgrade" ]; then echo "SELF_UPGRADE: skipped ($ROOT/bin/gstack-upgrade is missing)" + elif [ ! -x "$ROOT/bin/gstack-upgrade" ]; then echo "SELF_UPGRADE: skipped ($ROOT/bin/gstack-upgrade is missing)"; report_behind "$RERUN" + export GSTACK_CAPY_BEHIND_BLOCKED="$ROOT/bin/gstack-upgrade is missing; fix: $RERUN (UPGRADE_REFUSED)" else OLD=$(head -1 "$ROOT/VERSION" 2>/dev/null) UP_OUT=$("$ROOT/bin/gstack-upgrade" --ff-only-no-reinstall 2>&1); UP_RC=$? @@ -89,6 +116,8 @@ elif [ "${GSTACK_CAPY_SELF_UPGRADED:-}" != 1 ]; then REASON=$(printf '%s\n' "$UP_OUT" | sed -n 's/^gstack-upgrade: \(.*\); nothing was changed; fix: /\1; fix: /p' | tail -1) [ -n "$REASON" ] || REASON=$(printf '%s\n' "$UP_OUT" | sed -n 's/^gstack-upgrade: //p' | tail -1) echo "SELF_UPGRADE: skipped (${REASON:-gstack-upgrade exit $UP_RC})" + report_behind "$ROOT/bin/gstack-upgrade" + export GSTACK_CAPY_BEHIND_BLOCKED="${REASON:-gstack-upgrade exit $UP_RC; fix: $ROOT/bin/gstack-upgrade (UPGRADE_REFUSED)}" fi fi exec "$INSTALLER" "$@" diff --git a/bin/gstack-lease b/bin/gstack-lease index 431b3fba8..ded79313e 100755 --- a/bin/gstack-lease +++ b/bin/gstack-lease @@ -9,21 +9,29 @@ # # Usage: # gstack-lease touch [--root ] [--pid

| --from ] -# write /gstack-leases/; pid defaults to the harness pid -# behind this call (bin/gstack-harness-pid.sh, the same derivation the -# session marker uses), or behind --from (a caller whose own parent is -# short-lived passes its parent pid); leases older than 120 min are pruned +# write /gstack-leases/ holding the pid's start identity; +# pid defaults to the harness pid behind this call +# (bin/gstack-harness-pid.sh, the same derivation the session marker +# uses), or behind --from (a caller whose own parent is short-lived +# passes its parent pid); leases older than 120 min are pruned. Exits 3 +# and writes nothing when a live process holds the directory lock # gstack-lease live [--root ] [--exclude-ancestry-of ] -# print ` m` per lease whose pid is alive and is neither the -# given pid nor one of its ancestors (the caller's own session); exit 0 -# when any live lease exists, 1 when none +# print ` m ` per lease held by a live process that +# is neither the given pid nor one of its ancestors (the caller's own +# session); exit 0 when any live lease exists, 1 when none. A stale lease +# is reclaimed (deleted, one `gstack-lease: reclaimed …` line on stderr) # gstack-lease list [--root ] -# print ` m alive|dead` for every lease +# print ` m alive|dead|stale` for every lease +# A pid alone is not a holder: pids are reused after a reboot or a machine +# wake. A lease records its process's start identity (Linux: boot id and +# /proc start time; elsewhere `ps -o lstart`), and a lease whose pid now +# belongs to another process is stale. A lease without an identity (written by +# an older gstack) is stale once it is older than the 120-minute prune window. # defaults to the source checkout behind this bin (an installed # runtime root's .source-path, else ../). A root that is not a git checkout # exits 2 and writes nothing. GSTACK_LEASE_DIR overrides the lease directory # (hermetic tests; a dev checkout's own tests must not lease it). Bash 3.2 -# plus git, ps, find. +# plus git, ps, find, ln, readlink. set -u BIN="$(cd "$(dirname "$0")" && pwd -P)" # shellcheck source=bin/gstack-install-verdict.sh @@ -54,6 +62,88 @@ GIT_DIR_ABS=$(git -C "$ROOT" rev-parse --absolute-git-dir 2>/dev/null) \ DIR="${GSTACK_LEASE_DIR:-$GIT_DIR_ABS/gstack-leases}" alive() { kill -0 "$1" 2>/dev/null || ps -p "$1" >/dev/null 2>&1; } +identity() { # PID — the process's start identity, empty when unknown + if [ -r "/proc/$1/stat" ] && [ -r /proc/sys/kernel/random/boot_id ]; then + _st=$(cat "/proc/$1/stat" 2>/dev/null) || return 0 + _st=${_st##*) } + _id="linux $(cat /proc/sys/kernel/random/boot_id 2>/dev/null) $(printf '%s\n' "$_st" | awk '{print $20}')" + else + _id="ps $(LC_ALL=C TZ=UTC0 ps -o lstart= -p "$1" 2>/dev/null | sed 's/^ *//; s/ *$//')" + fi + valid_identity "$_id" && printf '%s\n' "$_id" + return 0 +} +# valid_identity ID — a well-formed identity from either backend +valid_identity() { + case "$1" in + "linux "*) printf '%s\n' "$1" | grep -Eq '^linux [0-9a-f-]{36} [0-9]+$' ;; + "ps "?*) case "$1" in "ps "*[!\ ]*) return 0 ;; esac; return 1 ;; + *) return 1 ;; + esac +} +# state PID RECORD AGE — alive | dead | stale for a lease whose record (first +# line) and age were read once by the caller. Stale needs proof: two +# well-formed identities from the same backend that differ. An identity that +# cannot be read now, a malformed record or a backend change keeps the lease +# (never reclaim a live session on missing evidence). +state() { + alive "$1" || { echo dead; return; } + if [ -z "$2" ]; then + [ "$3" -ge 120 ] && echo stale || echo alive + return + fi + _now=$(identity "$1") + if valid_identity "$2" && [ -n "$_now" ] && [ "${2%% *}" = "${_now%% *}" ] && [ "$2" != "$_now" ]; then echo stale + else echo alive + fi +} +# The lease directory's lock: touch's publish and prune, and reclaim's +# reread-and-delete, run under it, so a reclaim never deletes a renewal and a +# lease never disappears while it is live. The lock is a symlink whose target +# names its holder (` `): ln -s creates it and its owner +# in one atomic step. It is broken only when that holder is dead or its pid +# now belongs to another process, never by age, and unlock removes only the +# caller's own lock. A live holder makes lock() give up after +# GSTACK_LEASE_LOCK_TRIES tries 0.1 s apart (default 300). +LOCK="$DIR/.lock" TOKEN="" +lock() { + [ -n "$TOKEN" ] || TOKEN="$$ $(identity $$)" + _tries=0 + until ln -s "$TOKEN" "$LOCK" 2>/dev/null; do + _tries=$((_tries + 1)) + LOCK_OWNER=$(readlink "$LOCK" 2>/dev/null) + [ -n "$LOCK_OWNER" ] && holder_gone "$LOCK_OWNER" && break_lock "$LOCK_OWNER" && continue + [ "$_tries" -ge "${GSTACK_LEASE_LOCK_TRIES:-300}" ] && return 1 + sleep 0.1 + done +} +holder_gone() { case "$(state "${1%% *}" "${1#* }" 0)" in dead|stale) return 0 ;; esac; return 1; } +# break_lock OWNER — remove the lock only if it still names OWNER. Breakers +# take a second symlink, .lock.break, and reread the lock under it, so a lock +# taken since the verdict is never removed; a .lock.break whose holder is gone +# is cleared. Returns 1 when another breaker holds .lock.break. Known limit: +# that clear is not atomic, so two gstack-lease processes killed in a row (one +# holding .lock, the next inside this function) can let a reclaim delete a +# renewal; one crash is always safe. +break_lock() { + if ! ln -s "$TOKEN" "$LOCK.break" 2>/dev/null; then + _b=$(readlink "$LOCK.break" 2>/dev/null) + [ -n "$_b" ] && holder_gone "$_b" && [ "$(readlink "$LOCK.break" 2>/dev/null)" = "$_b" ] && rm -f "$LOCK.break" + return 1 + fi + [ "$(readlink "$LOCK" 2>/dev/null)" = "$1" ] && rm -f "$LOCK" + rm -f "$LOCK.break" + return 0 +} +unlock() { [ "$(readlink "$LOCK" 2>/dev/null)" = "$TOKEN" ] && rm -f "$LOCK"; return 0; } +# reclaim FILE RECORD — delete a stale lease only if, under the lock, it still +# holds the record judged stale; a renewal since then keeps it. +reclaim() { + lock || return 1 + if [ "$(head -1 "$1" 2>/dev/null)" = "$2" ]; then rm -f "$1"; unlock; return 0; fi + unlock + return 1 +} age_min() { # FILE — minutes since its mtime _m=$(stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || echo 0) echo $(( ( $(date +%s) - _m ) / 60 )) @@ -63,14 +153,18 @@ case "$VERB" in touch) [ -n "$PID" ] || PID=$(gstack_harness_pid "${FROM:-$PPID}") mkdir -p "$DIR" || exit 2 - : > "$DIR/$PID" || exit 2 + lock || { echo "gstack-lease: the lease directory is locked by live process ${LOCK_OWNER%% *}; lease $PID not renewed" >&2; exit 3; } + _tmp="$DIR/.$PID.$$" + identity "$PID" > "$_tmp" && mv -f "$_tmp" "$DIR/$PID" || { rm -f "$_tmp"; unlock; exit 2; } find "$DIR" -mmin +120 -type f -exec rm -f {} + 2>/dev/null + unlock exit 0 ;; list) for f in "$DIR"/*; do [ -f "$f" ] || continue p=${f##*/} - if alive "$p"; then echo "$p $(age_min "$f")m alive"; else echo "$p $(age_min "$f")m dead"; fi + _age=$(age_min "$f") + echo "$p ${_age}m $(state "$p" "$(head -1 "$f" 2>/dev/null)" "$_age")" done exit 0 ;; esac @@ -89,8 +183,16 @@ for f in "$DIR"/*; do [ -f "$f" ] || continue p=${f##*/} case "$ANCESTRY" in *" $p "*) continue ;; esac - alive "$p" || continue - echo "$p $(age_min "$f")m" + _rec=$(head -1 "$f" 2>/dev/null) _age=$(age_min "$f") + case "$(state "$p" "$_rec" "$_age")" in + dead) continue ;; + stale) + if reclaim "$f" "$_rec"; then + echo "gstack-lease: reclaimed stale lease $p (${_age}m; recorded '$_rec', pid $p is now $(ps -o comm= -p "$p" 2>/dev/null | head -1) '$(identity "$p")')" >&2 + continue + fi ;; + esac + echo "$p $(age_min "$f")m $(ps -o comm= -p "$p" 2>/dev/null | head -1 | sed 's|.*/||')" FOUND=0 done exit $FOUND diff --git a/bin/gstack-skill-start b/bin/gstack-skill-start index 6d2a3f352..da9a7b26f 100755 --- a/bin/gstack-skill-start +++ b/bin/gstack-skill-start @@ -124,8 +124,12 @@ mkdir -p "$_GH/sessions" 2>/dev/null || true touch "$_GH/sessions/$PARENT_PID" 2>/dev/null || true find "$_GH/sessions" -mmin +120 -type f -exec rm {} + 2>/dev/null || true # The install-scoped twin (bin/gstack-lease): what bin/gstack-upgrade reads -# before moving a checkout other sessions share. Fail-open. -"$_BIN/gstack-lease" touch --pid "$PARENT_PID" >/dev/null 2>&1 || true +# before moving a checkout other sessions share. Fail-open, but a lease the +# held lock kept from being written is reported. +"$_BIN/gstack-lease" touch --pid "$PARENT_PID" >/dev/null 2>&1 || { + [ $? -eq 3 ] && echo "gstack-lease: the lease directory stayed locked by a live process; this session holds no workflow lease, so bin/gstack-upgrade cannot see it" >&2 + true +} _SESSIONS=$(find "$_GH/sessions" -mmin -120 -type f 2>/dev/null | wc -l | tr -d ' ') _PROACTIVE=$("$_BIN/gstack-config" get proactive 2>/dev/null || echo "true") diff --git a/capy/skill/gstack/SKILL.md b/capy/skill/gstack/SKILL.md index 746664a77..457748c21 100644 --- a/capy/skill/gstack/SKILL.md +++ b/capy/skill/gstack/SKILL.md @@ -10,7 +10,7 @@ description: Install and use gstack in Capy and Claude Code. Load at session sta Resolve `scripts/install.sh` relative to this SKILL.md, not the working repository. It is a frozen bootstrapper: it prints `gstack-capy-skill: wrapper ()` first, learns the supported minimum from gstack's remote, and hands over to the checkout's own wrapper (`bin/gstack-capy-session`). 1. At session start run `bash scripts/install.sh --check`, adding `--for ` when you know the workflow (`--for autoplan`, `--for review`, `--for qa`). It prints one `PASS|FAIL|SKIP|WARN ` row per component (`WARN` is an owner action), a `GSTACK_LAUNCH:` line, and exactly one trailer as the last line: `gstack: ok for=[ ()]` (exit 0) or `gstack: fail …` (exit 1). Read the exit status and grep `^gstack: ` directly; never pipe the check through `tail`, `head` or `grep` alone. `--check` never moves the checkout. -2. If the trailer names `revision`, `runtime` or `pins` (`gstack: fail revision …`, `gstack: fail runtime,pins`, `gstack: fail pins`) and a checkout exists, or the output carries a `gstack-capy-session: behind origin/: …` line, run the same command without `--check` (900-second timeout). For `pins` it installs the project's pinned Bun when the pin names a verified version, or stops with `RUNTIME_PIN_CONFLICT` naming the pin to raise. Any clean, unpinned checkout behind origin is fast-forwarded in install mode (below the `GSTACK_MIN_REVISION` floor by the bootstrapper, `gstack-capy-skill: upgraded -> `; above it by the checkout, `SELF_UPGRADE: done -> `), reinstalled and rechecked in one locked step. A pinned (`--revision`, detached), dirty or diverged checkout, or one another live session holds a lease on (`UPGRADE_BUSY`), is never moved: below the floor the trailer names the cause and the exact commands; above it the run prints `SELF_UPGRADE: skipped ()` and continues on the installed revision. `GSTACK_CAPY_AUTO_UPGRADE=0` disables every fast-forward, `GSTACK_CAPY_SELF_UPGRADE=off` only the above-floor one. +2. If the trailer names `revision`, `runtime` or `pins` (`gstack: fail revision …`, `gstack: fail runtime,pins`, `gstack: fail pins`) and a checkout exists, or an ok trailer carries `behind= (latest: ); fix: ` (the checkout is commits behind origin; the output also has a `gstack-capy-session: behind origin/: …` line), run the same command without `--check` (900-second timeout), or the `fix:` command when it names another. Report the trailer verbatim, `behind=` included: a lane that is behind says so. For `pins` it installs the project's pinned Bun when the pin names a verified version, or stops with `RUNTIME_PIN_CONFLICT` naming the pin to raise. Any clean, unpinned checkout behind origin is fast-forwarded in install mode (below the `GSTACK_MIN_REVISION` floor by the bootstrapper, `gstack-capy-skill: upgraded -> `; above it by the checkout, `SELF_UPGRADE: done -> `), reinstalled and rechecked in one locked step. A pinned (`--revision`, detached), dirty or diverged checkout, or one another live session holds a lease on (`UPGRADE_BUSY`), is never moved: the trailer names the cause and the exact commands (above the floor it is a `fail behind` trailer after `SELF_UPGRADE: skipped ()`, with `UPGRADE_BUSY` or `UPGRADE_REFUSED`). A lease whose pid was reused after a wake is reclaimed on its own. `GSTACK_CAPY_AUTO_UPGRADE=0` disables every fast-forward, `GSTACK_CAPY_SELF_UPGRADE=off` only the above-floor one. 3. If the trailer is `gstack: fail not-installed …` and this session runs no gstack workflow, record that line and continue; nothing is installed. Install right before the first workflow with `bash scripts/install.sh --for ` (about 30 s cold: clone, a SHA-256-verified Bun when none installed fits, both host profiles without Chromium, and a `~/.local/bin/bun` shim so fresh shells get the recorded Bun). 4. Any other `FAIL` row carries a `fix:` command and a reason code with a troubleshooting anchor; run the fix, then `--check` again. `SKIP codex-cli (optional; …)`, `SKIP browser (lazy; …)` and `SKIP pins (planning-only; …)` are not failures. diff --git a/capy/support.env b/capy/support.env index 01fd89653..29b7f8cdf 100644 --- a/capy/support.env +++ b/capy/support.env @@ -12,7 +12,7 @@ # lines; refusals skip and continue). Raise it only for install-honesty or # security fixes, at stamp time, to the VERSION being released # (docs/capy-install.md "Raising the supported minimum"); it never exceeds VERSION. -GSTACK_MIN_REVISION=1.91.96.0 +GSTACK_MIN_REVISION=1.91.113.0 # GSTACK_CAPY_MIN_WRAPPER: the oldest bootstrapper revision (GSTACK_CAPY_WRAPPER in # capy/skill/gstack/scripts/install.sh) the owner should run. An older copy still # works and prints an owner action (CAPY_WRAPPER_STALE). diff --git a/docs/capy-install.md b/docs/capy-install.md index 524d183fb..1e6f915e5 100644 --- a/docs/capy-install.md +++ b/docs/capy-install.md @@ -42,14 +42,20 @@ git clone --single-branch --depth 1 https://github.com/garrytan/gstack.git ~/.lo refusal of `gstack-upgrade` applies), prints `SELF_UPGRADE: done -> ` and re-execs the new checkout's wrapper, which installs from the new revision (`GSTACK_CAPY_UPGRADED_FROM` set, so setup is never skipped). - A refusal prints `SELF_UPGRADE: skipped ()` and the session - continues on the installed revision; the trailer stays the doctor's. + A refusal prints `SELF_UPGRADE: skipped ()`, and the session is + never certified ok: the trailer is `gstack: fail behind — installed is + commits behind origin (latest ) and the session-start upgrade was + skipped: ; fix: (UPGRADE_BUSY|UPGRADE_REFUSED)`, exit 1. `SELF_UPGRADE: none (current)` when nothing is ahead; `GSTACK_CAPY_SELF_UPGRADE=off` (or `GSTACK_CAPY_AUTO_UPGRADE=0`) prints `SELF_UPGRADE: disabled ()`. `--check` never moves the checkout; behind origin it prints `gstack-capy-session: behind origin/: -> ( commits); fix: bash /scripts/install.sh - [--for ] (fast-forwards)` and the agent runs that fix. + [--for ] (fast-forwards)` and the agent runs that fix. Whenever the + session ends behind origin (`--check`, or a self-upgrade that was skipped or + disabled), the ok trailer also says so, because lanes relay only the + trailer: `behind= (latest: ); fix: ` (see the trailer + grammar below). 4. When the fetch fails (offline), the bootstrapper says `remote minimum unknown (fetch failed)` and checks the local install only; it never claims the minimum, and nothing is fast-forwarded. @@ -72,10 +78,28 @@ resets. Several agents sharing one checkout (eight Capy subagents on a machine) hold workflow leases on it: every `bin/gstack-skill-start` touches `/gstack-leases/` (`bin/gstack-lease touch`; pruned after -120 minutes). `bin/gstack-upgrade`, and so the session-start self-upgrade, -refuses with [UPGRADE_BUSY](troubleshooting.md#upgrade-busy) while a lease -whose pid is alive and outside its own process chain exists; -`bin/gstack-lease list` shows them and `GSTACK_UPGRADE_FORCE=1` overrides. +120 minutes). Each lease records its process's start identity (Linux: boot id +and `/proc` start time; elsewhere `ps -o lstart`), because a pid is reused after +a reboot or a machine wake. `bin/gstack-upgrade`, and so the session-start +self-upgrade, refuses with [UPGRADE_BUSY](troubleshooting.md#upgrade-busy) +while a lease held by a live process outside its own chain exists, naming each +holder as `pid

(m )`. A lease whose pid now belongs to another +process, or an identity-less lease from an older gstack past 120 minutes, is +stale: `bin/gstack-lease live` reclaims it (`gstack-lease: reclaimed stale +lease …`). Stale needs proof: two well-formed identities from the same backend +that differ (`ps` is read under `LC_ALL=C TZ=UTC0`). An identity that cannot be +read, a malformed record or a backend change keeps the lease. `touch` (publish +by rename, prune) and reclaim (reread, then delete only the record judged +stale) run under one lock in the lease directory, a symlink naming its holder's +pid and start identity, so a renewal is never deleted and a live lease never +disappears from a concurrent scan. The lock is broken only when its holder is +dead or its pid was reused, never by age, and breakers serialize on a second +symlink and reread the lock before removing it. One killed gstack-lease process +is always safe; two killed in a row, one holding the lock and the next while +breaking it, can still let a reclaim delete a renewal. While a live process holds it, +`touch` exits 3 without writing (`gstack-skill-start` reports that the +session holds no lease) and a reclaim keeps the lease. `bin/gstack-lease list` shows `alive|dead|stale`, and +`GSTACK_UPGRADE_FORCE=1` overrides. ## What the installer does @@ -213,13 +237,18 @@ The last line is the trailer: gstack: ok 1.91.96.0 for=autoplan gstack: ok 1.91.96.0 for=review (requested: v1.91.90.0 (run /bin/gstack-upgrade to move); latest: 1.91.101.0 available) gstack: ok 1.91.96.0 for=autoplan (pins: skipped (planning-only)) +gstack: ok 1.91.101.0 for=autoplan (behind=12 (latest: 1.91.107.0); fix: bash /scripts/install.sh --for autoplan) gstack: fail runtime,pins ``` An ok trailer carries at most one parenthesised note group, `; `-separated, in this order: `requested: …` (an existing checkout at another revision than -`--revision`), `pins: skipped (planning-only|no project)`, and `latest: -available` (origin's default branch has a newer VERSION; the bootstrapper +`--revision`), `pins: skipped (planning-only|no project)`, then either +`behind= (latest: )` followed by `fix: ` (the session ended + commits behind origin's default branch after a fetch that succeeded in this +run; `bin/gstack-capy-session` exports `GSTACK_CAPY_BEHIND`, `_BEHIND_TO` and +`_BEHIND_FIX`, so this works with every bootstrapper revision), or `latest: + available` (origin's default branch has a newer VERSION; the bootstrapper exports it as `GSTACK_CAPY_REMOTE_VERSION` only after a fetch that succeeded in this run). A fail trailer never carries notes. Match `^gstack: ok ` or `^gstack: fail `; never parse past the first space after the version. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 2e7738dd0..aa98a7886 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1413,15 +1413,20 @@ machine, say). Each session start writes a workflow lease, anything because a lease whose pid is alive and not in its own process ancestry exists: fast-forwarding would change gstack under a session mid-workflow. Leases older than 120 minutes are pruned; a lease whose -process has exited never blocks. The upgrader's own session (the pid chain it +process has exited never blocks, and neither does a stale one: each lease +records its process's start identity, so a pid reused after a reboot or a +machine wake (GBRA-105's lease named pid 379, which had become dbus-daemon) +is reclaimed instead of blocking. The upgrader's own session (the pid chain it runs under) never counts. **Fix.** Wait for those sessions to finish (`bin/gstack-lease list` shows -every lease with its age and `alive|dead`), then re-run `bin/gstack-upgrade`. +every lease with its age and `alive|dead|stale`; a holder is named as +`pid

(m )`), then re-run `bin/gstack-upgrade`. When you know they are idle, `GSTACK_UPGRADE_FORCE=1 bin/gstack-upgrade` proceeds. At a Capy session start the same refusal appears as -`SELF_UPGRADE: skipped (live gstack sessions hold this installation: ...)` -and the session continues on the installed revision. +`SELF_UPGRADE: skipped (live gstack sessions hold this installation: ...)`, +and the trailer is `gstack: fail behind — … (UPGRADE_BUSY)`: a session that +could not upgrade is never certified ok. ### `gstack-upgrade: could not reach origin (...) (UPGRADE_FETCH_FAILED)` diff --git a/package.json b/package.json index a4c43d2f4..678926225 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.91.112", + "version": "1.91.113", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/capy-skill-wrapper.test.ts b/test/capy-skill-wrapper.test.ts index b7f8b6c10..711973ad0 100644 --- a/test/capy-skill-wrapper.test.ts +++ b/test/capy-skill-wrapper.test.ts @@ -14,7 +14,7 @@ * depth-1 file:// clone at A, on the simulated Capy host. */ import { afterEach, describe, expect, test } from 'bun:test'; -import { spawnSync } from 'child_process'; +import { spawn, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import { RESULT_CODES } from '../lib/result-codes'; @@ -308,6 +308,87 @@ describe('capy skill bootstrapper: GSTACK_CAPY_REMOTE_VERSION', () => { }); }); +// Field list 2 item 13: lanes relay only the trailer, so a session that ends +// behind origin hands the installer the gap and the fix for its trailer note. +describe('capy skill: the session hands the installer GSTACK_CAPY_BEHIND', () => { + const ENV_INSTALLER = '#!/usr/bin/env bash\necho "gstack: ok $(cat "$(dirname "$0")/../VERSION") for=autoplan behind=${GSTACK_CAPY_BEHIND:-none} to=${GSTACK_CAPY_BEHIND_TO:-none} fix=${GSTACK_CAPY_BEHIND_FIX:-none}${GSTACK_CAPY_BEHIND_BLOCKED:+ blocked=$GSTACK_CAPY_BEHIND_BLOCKED}"\n'; + const rerun = `bash ${BOOT} --for autoplan`; + + test('--check behind origin: the commit count, origin\'s VERSION and the fast-forwarding command; a checkout the rerun would not move points at gstack-upgrade', () => { + const f = fixture({ cloneAt: 'B' }); + write(path.join(f.root, 'bin/gstack-capy-install'), ENV_INSTALLER, 0o755); + f.git(['update-index', '--assume-unchanged', 'bin/gstack-capy-install']); + const r = f.run(['--check', '--for', 'autoplan']); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.last).toBe(`gstack: ok 1.0.1 for=autoplan behind=1 to=1.0.2 fix=${rerun}`); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); + fs.writeFileSync(path.join(f.root, 'README.md'), 'local edit\n'); + expect(f.run(['--check', '--for', 'autoplan']).last).toBe(`gstack: ok 1.0.1 for=autoplan behind=1 to=1.0.2 fix=${f.root}/bin/gstack-upgrade`); + f.git(['checkout', '-q', '--', 'README.md']); + f.git(['checkout', '-q', '--detach']); + expect(f.run(['--check', '--for', 'autoplan']).last).toBe(`gstack: ok 1.0.1 for=autoplan behind=1 to=1.0.2 fix=${f.root}/bin/gstack-upgrade`); + f.git(['checkout', '-q', '-b', 'topic']); + expect(f.run(['--check', '--for', 'autoplan']).last).toBe(`gstack: ok 1.0.1 for=autoplan behind=1 to=1.0.2 fix=${f.root}/bin/gstack-upgrade`); + f.git(['checkout', '-q', 'main']); + expect(f.run(['--check', '--for', 'autoplan']).stdout).toMatch(/^gstack-capy-session: behind origin\/main: 1\.0\.1 -> 1\.0\.2 \(1 commits\); fix: bash .+ --for autoplan \(fast-forwards\)$/m); + expect(f.run(['--check', '--for', 'autoplan'], { GSTACK_CAPY_AUTO_UPGRADE: '0' }).last).toContain('fix=unset GSTACK_CAPY_AUTO_UPGRADE, then '); + }); + + test('install mode: a skipped self-upgrade points at gstack-upgrade (which names its refusal); a disabled one names the variable to unset; no fix carries the "; " note separator', () => { + const f = fixture({ cloneAt: 'B' }); + write(path.join(f.root, 'bin/gstack-capy-install'), ENV_INSTALLER, 0o755); + const skipped = f.run(['--for', 'autoplan']); + expect(skipped.stdout).toMatch(/^SELF_UPGRADE: skipped \(tracked files changed: bin\/gstack-capy-install; fix: .+\)$/m); + expect(skipped.last).toMatch(new RegExp(`^gstack: ok 1\\.0\\.1 for=autoplan behind=1 to=1\\.0\\.2 fix=${f.root.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/bin/gstack-upgrade blocked=tracked files changed: bin/gstack-capy-install; fix: .+ \\(UPGRADE_REFUSED\\)$`)); + const disabled = f.run(['--for', 'autoplan'], { GSTACK_CAPY_SELF_UPGRADE: 'off' }); + expect(disabled.last).toBe(`gstack: ok 1.0.1 for=autoplan behind=1 to=1.0.2 fix=unset GSTACK_CAPY_SELF_UPGRADE, then ${rerun}`); + for (const r of [skipped, disabled]) expect(r.last.split(' fix=')[1].split(' blocked=')[0]).not.toContain('; '); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); + }); + + // GBRA-105 / GBRA-106: a lease naming a pid that a machine wake handed to + // another process wedged every upgrade, and the skipped upgrade still ended ok. + test('install mode: a lease whose pid was reused is reclaimed and the checkout upgrades; a live holder blocks it and the session hands the installer the busy reason with the holder', () => { + const f = fixture({ cloneAt: 'B' }); + write(path.join(f.root, 'bin/gstack-capy-install'), ENV_INSTALLER, 0o755); + f.git(['update-index', '--assume-unchanged', 'bin/gstack-capy-install']); + const leaseDir = path.join(f.host.base, 'leases'); + const holder = spawn('sleep', ['300'], { stdio: 'ignore' }); + try { + const lease = (args: string[]) => spawnSync('bash', [path.join(f.root, 'bin/gstack-lease'), ...args, '--root', f.root], { encoding: 'utf8', env: { ...f.host.env, GSTACK_LEASE_DIR: leaseDir }, timeout: 30_000 }); + expect(lease(['touch', '--pid', String(holder.pid)]).status).toBe(0); + const busy = f.run(['--for', 'autoplan'], { GSTACK_LEASE_DIR: leaseDir }); + expect(busy.stdout).toMatch(new RegExp(`^SELF_UPGRADE: skipped \\(live gstack sessions hold this installation: pid ${holder.pid} \\(\\d+m sleep\\); fix: .+ \\(UPGRADE_BUSY\\)\\)$`, 'm')); + expect(busy.last).toMatch(new RegExp(`behind=1 to=1\\.0\\.2 fix=.+ blocked=live gstack sessions hold this installation: pid ${holder.pid} \\(\\d+m sleep\\); fix: .+GSTACK_UPGRADE_FORCE=1.+ \\(UPGRADE_BUSY\\)$`)); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.b); + const file = path.join(leaseDir, String(holder.pid)); + const recorded = fs.readFileSync(file, 'utf8').trim(); + fs.writeFileSync(file, (recorded.startsWith('linux ') ? recorded.replace(/\d+$/, (n) => String(Number(n) + 1)) : 'ps Thu Jan 1 00:00:00 1970') + '\n'); + const reclaimed = f.run(['--for', 'autoplan'], { GSTACK_LEASE_DIR: leaseDir }); + expect(reclaimed.stdout).toMatch(/^SELF_UPGRADE: done 1\.0\.1 -> 1\.0\.2$/m); + expect(f.git(['rev-parse', 'HEAD'])).toBe(f.c); + expect(fs.existsSync(file)).toBe(false); + } finally { + holder.kill('SIGKILL'); + } + }); + + test('current, or the fetch failed: nothing is exported, and values inherited from a parent session are dropped', () => { + const f = fixture({ cloneAt: 'B' }); + f.git(['fetch', '-q', 'origin', 'main']); + f.git(['merge', '--ff-only', '-q', 'FETCH_HEAD']); + write(path.join(f.root, 'bin/gstack-capy-install'), ENV_INSTALLER, 0o755); + const inherited = { GSTACK_CAPY_BEHIND: '9', GSTACK_CAPY_BEHIND_TO: '9.9.9', GSTACK_CAPY_BEHIND_FIX: 'stale' }; + expect(f.run(['--check', '--for', 'autoplan'], inherited).last).toBe('gstack: ok 1.0.2 for=autoplan behind=none to=none fix=none'); + f.git(['reset', '-q', '--hard', f.b]); + write(path.join(f.root, 'bin/gstack-capy-install'), ENV_INSTALLER, 0o755); + f.git(['remote', 'set-url', 'origin', `file://${f.host.base}/missing.git`]); + const offline = f.run(['--check', '--for', 'autoplan'], inherited); + expect(offline.stdout).toContain('remote minimum unknown (fetch failed)'); + expect(offline.last).toBe('gstack: ok 1.0.1 for=autoplan behind=none to=none fix=none'); + }); +}); + describe('capy skill: vendored text and support file', () => { const support = fs.existsSync(path.join(REPO, 'capy/support.env')) ? fs.readFileSync(path.join(REPO, 'capy/support.env'), 'utf8') : ''; test('the wrapper revision meets the checkout minimum; GSTACK_MIN_REVISION <= VERSION', () => { diff --git a/test/gstack-capy-install.test.ts b/test/gstack-capy-install.test.ts index 8f9c74c29..05ff1c71f 100644 --- a/test/gstack-capy-install.test.ts +++ b/test/gstack-capy-install.test.ts @@ -269,6 +269,27 @@ describe('gstack-capy-install', () => { expect(lastLine(failed.stdout)).toBe('gstack: fail codex'); }); + test('trailer grammar: behind= and its fix replace latest, after requested and pins; never on a fail trailer', () => { + const f = fixture(); + const behind = { GSTACK_CAPY_BEHIND: '5', GSTACK_CAPY_BEHIND_TO: '1.2.8', GSTACK_CAPY_BEHIND_FIX: 'bash /s/install.sh --for autoplan', GSTACK_CAPY_REMOTE_VERSION: '1.2.8' }; + expect(lastLine(run(f, [], behind).stdout)).toBe('gstack: ok 1.2.3 for=autoplan (behind=5 (latest: 1.2.8); fix: bash /s/install.sh --for autoplan)'); + const all = run(f, ['--revision', 'v9.9.9.9'], { ...behind, STUB_PINS_SKIP: 'planning-only; repo pins differ: x' }); + expect(all.status, all.stdout + all.stderr).toBe(0); + expect(lastLine(all.stdout)).toBe(`gstack: ok 1.2.3 for=autoplan (requested: v9.9.9.9 (run ${f.root}/bin/gstack-upgrade to move); pins: skipped (planning-only); behind=5 (latest: 1.2.8); fix: bash /s/install.sh --for autoplan)`); + expect(lastLine(run(f, [], { ...behind, STUB_DOCTOR_FAIL: '1' }).stdout)).toBe('gstack: fail codex'); + }); + + test('a skipped session-start upgrade is never ok: the trailer is `fail behind` with the reason, the fix and the code, exit 1, and the verdict records the failure', () => { + const f = fixture(); + const busy = { GSTACK_CAPY_BEHIND: '3', GSTACK_CAPY_BEHIND_TO: '1.2.6', GSTACK_CAPY_BEHIND_FIX: `${f.root}/bin/gstack-upgrade`, GSTACK_CAPY_BEHIND_BLOCKED: 'live gstack sessions hold this installation: pid 379 (124m dbus-daemon); fix: wait for those sessions to finish, or re-run with GSTACK_UPGRADE_FORCE=1 when you know they are idle (UPGRADE_BUSY)' }; + const r = run(f, [], busy); + expect(r.status, r.stdout + r.stderr).toBe(1); + expect(lastLine(r.stdout)).toBe('gstack: fail behind — installed 1.2.3 is 3 commits behind origin (latest 1.2.6) and the session-start upgrade was skipped: live gstack sessions hold this installation: pid 379 (124m dbus-daemon); fix: wait for those sessions to finish, or re-run with GSTACK_UPGRADE_FORCE=1 when you know they are idle (UPGRADE_BUSY)'); + const bare = run(f, [], { ...busy, GSTACK_CAPY_BEHIND_BLOCKED: 'gstack-upgrade exit 9' }); + expect(lastLine(bare.stdout)).toBe(`gstack: fail behind — installed 1.2.3 is 3 commits behind origin (latest 1.2.6) and the session-start upgrade was skipped: gstack-upgrade exit 9; fix: ${f.root}/bin/gstack-upgrade (UPGRADE_REFUSED)`); + expect(lastLine(run(f, [], { ...busy, STUB_DOCTOR_FAIL: '1' }).stdout)).toBe('gstack: fail codex'); + }); + test('--root clones when the directory is absent and refuses a non-gstack directory', () => { const f = fixture(); const clone = path.join(f.base, 'clone'); diff --git a/test/gstack-lease.test.ts b/test/gstack-lease.test.ts index 0518136a9..f2b6585d2 100644 --- a/test/gstack-lease.test.ts +++ b/test/gstack-lease.test.ts @@ -54,6 +54,18 @@ function deadPid(): number { return r.pid!; } +/** The same backend's identity for a different process start (what a reused pid shows). */ +function otherStart(recorded: string): string { + return recorded.startsWith('linux ') ? recorded.replace(/\d+$/, (n) => String(Number(n) + 1)) : 'ps Thu Jan 1 00:00:00 1970'; +} + +/** A BASH_ENV file whose functions shadow commands inside the lease script (fault injection). */ +function bashEnv(dir: string, body: string): string { + const file = path.join(dir, 'bash-env.sh'); + fs.writeFileSync(file, body); + return file; +} + describe('gstack-lease', () => { test('bash -n passes and the bin is executable', () => { expect(fs.existsSync(LEASE), LEASE).toBe(true); @@ -86,7 +98,7 @@ describe('gstack-lease', () => { expect(run(['touch', '--root', dir, '--pid', String(live)]).status).toBe(0); const some = run(['live', '--root', dir]); expect(some.status, some.stdout + some.stderr).toBe(0); - expect(some.stdout.trim().split('\n')).toEqual([expect.stringMatching(new RegExp(`^${live} \\d+m$`))]); + expect(some.stdout.trim().split('\n')).toEqual([expect.stringMatching(new RegExp(`^${live} \\d+m sleep$`))]); const list = run(['list', '--root', dir]); expect(list.status).toBe(0); @@ -94,12 +106,174 @@ describe('gstack-lease', () => { expect(list.stdout).toMatch(new RegExp(`^${dead} \\d+m dead$`, 'm')); }); + // GBRA-105 / GBRA-106: after a machine wake the recorded pid was dbus-daemon + // and systemd-logind, and every upgrade refused with UPGRADE_BUSY. + test('a reused pid is not a holder: a lease whose recorded start identity differs is reclaimed by live and listed stale', () => { + const { dir, leases } = checkout(); + const reused = sleeper(); + expect(run(['touch', '--root', dir, '--pid', String(reused)]).status).toBe(0); + const recorded = fs.readFileSync(path.join(leases, String(reused)), 'utf8').trim(); + expect(recorded).toMatch(/^(linux [0-9a-f-]+ \d+|ps .+)$/); + fs.writeFileSync(path.join(leases, String(reused)), otherStart(recorded) + '\n'); + expect(run(['list', '--root', dir]).stdout).toMatch(new RegExp(`^${reused} \\d+m stale$`, 'm')); + const live = run(['live', '--root', dir]); + expect(live.status, live.stdout + live.stderr).toBe(1); + expect(live.stdout.trim()).toBe(''); + expect(live.stderr).toMatch(new RegExp(`^gstack-lease: reclaimed stale lease ${reused} \\(\\d+m; recorded '.+', pid ${reused} is now sleep '.+'\\)$`, 'm')); + expect(fs.existsSync(path.join(leases, String(reused)))).toBe(false); + }); + + test('missing evidence never reclaims a live lease: an unreadable current identity, a malformed record or another backend keeps it', () => { + const { dir, leases } = checkout(); + const holder = sleeper(); + const file = path.join(leases, String(holder)); + expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0); + const recorded = fs.readFileSync(file, 'utf8').trim(); + const blind = bashEnv(dir, 'cat() { case "$1" in /proc/*/stat) return 1 ;; esac; command cat "$@"; }\nps() { case "$*" in *lstart*) return 1 ;; esac; command ps "$@"; }\n'); + const unreadable = run(['live', '--root', dir], { BASH_ENV: blind }); + expect(unreadable.status, unreadable.stdout + unreadable.stderr).toBe(0); + expect(unreadable.stderr).not.toContain('reclaimed'); + for (const record of ['linux not-a-boot-id 12', 'linux ' + '0'.repeat(36) + ' x1', recorded.startsWith('linux ') ? 'ps Thu Jan 1 00:00:00 1970' : 'linux ' + '0'.repeat(8) + '-0000-0000-0000-' + '0'.repeat(12) + ' 1']) { + fs.writeFileSync(file, record + '\n'); + const r = run(['live', '--root', dir]); + expect(r.status, record + ': ' + r.stderr).toBe(0); + expect(fs.existsSync(file), record).toBe(true); + } + }); + + test('the recorded identity does not depend on the caller\'s timezone or locale', () => { + const { dir } = checkout(); + const holder = sleeper(); + expect(run(['touch', '--root', dir, '--pid', String(holder)], { TZ: 'UTC', LC_ALL: 'C' }).status).toBe(0); + const r = run(['live', '--root', dir], { TZ: 'Pacific/Kiritimati', LC_ALL: 'C.UTF-8' }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.stderr).not.toContain('reclaimed'); + }); + + test('a renewal that lands between the stale verdict and the reclaim survives: reclaim rereads under the lock', () => { + const { dir, leases } = checkout(); + const holder = sleeper(); + const file = path.join(leases, String(holder)); + expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0); + const fresh = fs.readFileSync(file, 'utf8'); + fs.writeFileSync(file, otherStart(fresh.trim()) + '\n'); + const renewed = path.join(dir, 'renewed'); + fs.writeFileSync(renewed, fresh); + // The holder renews just before reclaim takes the lock, as if touch ran after the stale verdict. + const racing = bashEnv(dir, `ln() { case "$*" in *"/.lock") command cp "${renewed}" "${file}" ;; esac; command ln "$@"; }\n`); + const r = run(['live', '--root', dir], { BASH_ENV: racing }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.stderr).not.toContain('reclaimed'); + expect(fs.readFileSync(file, 'utf8')).toBe(fresh); + expect(fs.readdirSync(leases).filter((n) => n.startsWith('.'))).toEqual([]); + }); + + test('a lock whose holder died, or whose pid now belongs to another process, is broken at once', () => { + const { dir, leases } = checkout(); + const holder = sleeper(); + fs.mkdirSync(leases, { recursive: true }); + fs.symlinkSync(`${deadPid()} `, path.join(leases, '.lock')); + const r = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '3' }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(fs.existsSync(path.join(leases, '.lock'))).toBe(false); + const identity = fs.readFileSync(path.join(leases, String(holder)), 'utf8').trim(); + expect(identity).toMatch(/^(linux|ps) /); + fs.symlinkSync(`${holder} ${otherStart(identity)}`, path.join(leases, '.lock')); + const again = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '3' }); + expect(again.status, again.stdout + again.stderr).toBe(0); + expect(fs.existsSync(path.join(leases, '.lock'))).toBe(false); + }); + + test('a live holder\'s lock is never broken, however old: touch exits 3 without writing and a reclaim keeps the lease', () => { + const { dir, leases } = checkout(); + const owner = sleeper(); + const holder = sleeper(); + expect(run(['touch', '--root', dir, '--pid', String(owner)]).status).toBe(0); + expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0); + const ownerToken = `${owner} ${fs.readFileSync(path.join(leases, String(owner)), 'utf8').trim()}`; + const lockPath = path.join(leases, '.lock'); + fs.symlinkSync(ownerToken, lockPath); + const old = new Date(Date.now() - 5 * 60_000); + fs.lutimesSync(lockPath, old, old); + const holderFile = path.join(leases, String(holder)); + const stale = otherStart(fs.readFileSync(holderFile, 'utf8').trim()) + '\n'; + fs.writeFileSync(holderFile, stale); + const t = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '3' }); + expect(t.status, t.stdout + t.stderr).toBe(3); + expect(t.stderr).toContain(`locked by live process ${owner}`); + expect(fs.readFileSync(holderFile, 'utf8')).toBe(stale); + const l = run(['live', '--root', dir], { GSTACK_LEASE_LOCK_TRIES: '3' }); + expect(l.status, l.stdout + l.stderr).toBe(0); + expect(l.stderr).not.toContain('reclaimed'); + expect(l.stdout).toContain(`${holder} `); + expect(fs.existsSync(holderFile)).toBe(true); + expect(fs.readlinkSync(lockPath)).toBe(ownerToken); + }); + + test('two breakers of one dead lock: the second rereads under .lock.break and leaves the lock the first one took', () => { + const { dir, leases } = checkout(); + const owner = sleeper(); + const holder = sleeper(); + expect(run(['touch', '--root', dir, '--pid', String(owner)]).status).toBe(0); + const ownerToken = `${owner} ${fs.readFileSync(path.join(leases, String(owner)), 'utf8').trim()}`; + const lockPath = path.join(leases, '.lock'); + fs.symlinkSync(`${deadPid()} `, lockPath); + // Between this breaker's dead verdict and its .lock.break, another breaker removes the dead lock and takes it. + const first = bashEnv(dir, `ln() { case "$*" in *"/.lock.break") command rm -f "${lockPath}"; command ln -s "${ownerToken}" "${lockPath}" ;; esac; command ln "$@"; }\n`); + const t = run(['touch', '--root', dir, '--pid', String(holder)], { BASH_ENV: first, GSTACK_LEASE_LOCK_TRIES: '3' }); + expect(t.status, t.stdout + t.stderr).toBe(3); + expect(fs.readlinkSync(lockPath)).toBe(ownerToken); + expect(fs.existsSync(path.join(leases, '.lock.break'))).toBe(false); + }); + + test('a .lock.break left by a dead breaker is cleared and the dead lock is still broken', () => { + const { dir, leases } = checkout(); + const holder = sleeper(); + fs.mkdirSync(leases, { recursive: true }); + fs.symlinkSync(`${deadPid()} `, path.join(leases, '.lock')); + fs.symlinkSync(`${deadPid()} `, path.join(leases, '.lock.break')); + const r = run(['touch', '--root', dir, '--pid', String(holder)], { GSTACK_LEASE_LOCK_TRIES: '5' }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(fs.existsSync(path.join(leases, String(holder)))).toBe(true); + expect(fs.readdirSync(leases).filter((n) => n.startsWith('.'))).toEqual([]); + }); + + test('unlock removes only its own lock: a lock that changed hands during a reclaim stays', () => { + const { dir, leases } = checkout(); + const owner = sleeper(); + const holder = sleeper(); + expect(run(['touch', '--root', dir, '--pid', String(owner)]).status).toBe(0); + expect(run(['touch', '--root', dir, '--pid', String(holder)]).status).toBe(0); + const ownerToken = `${owner} ${fs.readFileSync(path.join(leases, String(owner)), 'utf8').trim()}`; + const holderFile = path.join(leases, String(holder)); + fs.writeFileSync(holderFile, otherStart(fs.readFileSync(holderFile, 'utf8').trim()) + '\n'); + const lockPath = path.join(leases, '.lock'); + // While reclaim deletes the stale lease under its lock, the lock changes hands. + const swap = bashEnv(dir, `rm() { if [ "$*" = "-f ${holderFile}" ]; then command rm -f "${lockPath}"; command ln -s "${ownerToken}" "${lockPath}"; fi; command rm "$@"; }\n`); + const r = run(['live', '--root', dir, '--exclude-ancestry-of', String(owner)], { BASH_ENV: swap }); + expect(r.stderr).toContain(`reclaimed stale lease ${holder}`); + expect(fs.readlinkSync(lockPath)).toBe(ownerToken); + }); + + test('a lease without a recorded identity (older gstack) counts while fresh and is stale past the 120-minute window', () => { + const { dir, leases } = checkout(); + const holder = sleeper(); + fs.mkdirSync(leases, { recursive: true }); + fs.writeFileSync(path.join(leases, String(holder)), ''); + expect(run(['live', '--root', dir]).status).toBe(0); + const old = new Date(Date.now() - 130 * 60_000); + fs.utimesSync(path.join(leases, String(holder)), old, old); + const r = run(['live', '--root', dir]); + expect(r.status, r.stdout + r.stderr).toBe(1); + expect(r.stderr).toContain(`reclaimed stale lease ${holder}`); + }); + test('live --exclude-ancestry-of drops a lease held by that pid or one of its ancestors (this test process)', () => { const { dir } = checkout(); expect(run(['touch', '--root', dir, '--pid', String(process.pid)]).status).toBe(0); const unfiltered = run(['live', '--root', dir]); expect(unfiltered.status, unfiltered.stdout + unfiltered.stderr).toBe(0); - expect(unfiltered.stdout).toMatch(new RegExp(`^${process.pid} \\d+m$`, 'm')); + expect(unfiltered.stdout).toMatch(new RegExp(`^${process.pid} \\d+m \\S+$`, 'm')); // $$ inside the bin is a child of this bun process, so our own lease is ancestry. const own = spawnSync('bash', ['-c', `"$LEASE" live --root "$ROOT" --exclude-ancestry-of $$`], { encoding: 'utf8', env: { ...baseEnv, LEASE, ROOT: dir }, timeout: 30_000, @@ -112,7 +286,7 @@ describe('gstack-lease', () => { encoding: 'utf8', env: { ...baseEnv, LEASE, ROOT: dir }, timeout: 30_000, }); expect(mixed.status).toBe(0); - expect(mixed.stdout.trim().split('\n')).toEqual([expect.stringMatching(new RegExp(`^${foreign} \\d+m$`))]); + expect(mixed.stdout.trim().split('\n')).toEqual([expect.stringMatching(new RegExp(`^${foreign} \\d+m sleep$`))]); }); test('touch without --pid uses the same harness pid gstack-skill-start derives (this process past the tool-call shell)', () => { @@ -151,4 +325,20 @@ describe('gstack-lease', () => { fs.rmSync(lease, { force: true }); } }); + + test('gstack-skill-start reports a lease the held lock kept from being written, and still starts', () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-lease-home-')); + const state = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-lease-state-')); + dirs.push(home, state); + fs.writeFileSync(path.join(state, 'config.yaml'), 'update_check: false\n'); + const leaseDir = path.join(state, 'leases'); + fs.mkdirSync(leaseDir); + fs.symlinkSync(`${sleeper()} `, path.join(leaseDir, '.lock')); + const r = spawnSync(START, ['--skill', 'testskill', '--parent-pid', '424244'], { + encoding: 'utf8', cwd: home, env: { PATH: process.env.PATH!, HOME: home, GSTACK_HOME: state, GSTACK_LEASE_DIR: leaseDir, GSTACK_LEASE_LOCK_TRIES: '2' }, timeout: 30_000, + }); + expect(r.status, r.stdout + r.stderr).toBe(0); + expect(r.stderr).toContain('this session holds no workflow lease'); + expect(fs.existsSync(path.join(leaseDir, '424244'))).toBe(false); + }); }); diff --git a/test/gstack-upgrade-bin.test.ts b/test/gstack-upgrade-bin.test.ts index 98f4e3420..a9676603e 100644 --- a/test/gstack-upgrade-bin.test.ts +++ b/test/gstack-upgrade-bin.test.ts @@ -188,7 +188,7 @@ describe('gstack-upgrade', () => { expect(r.status, r.stdout + r.stderr).toBe(3); expect(r.stderr).toContain('(UPGRADE_BUSY)'); expect(r.stderr).toContain('nothing was changed'); - expect(r.stderr).toMatch(new RegExp(`pid ${live} \\(\\d+m\\)`)); + expect(r.stderr).toMatch(new RegExp(`pid ${live} \\(\\d+m sleep\\)`)); expect(r.stderr).not.toContain(`pid ${process.pid} `); expect(r.stderr).toContain('wait for those sessions to finish, or re-run with GSTACK_UPGRADE_FORCE=1 when you know they are idle'); expect(f.git(['rev-parse', 'HEAD'])).toBe(f.a); diff --git a/test/helpers/third-party-actions.ts b/test/helpers/third-party-actions.ts index e2c7774b9..725b60610 100644 --- a/test/helpers/third-party-actions.ts +++ b/test/helpers/third-party-actions.ts @@ -20,6 +20,6 @@ export function asideDriveOptions(text: string): string[] { /\b(?:I|we)(?:['’]ll| will) (?:re[- ]?ask|ask again)(?: (?:you|this question))? with (?:the )?(?:"[^"]+"|“[^”]+”|[\w -]+?) option(?: included)?\b/gi, '', ); - return /\bAside\b/i.test(option) && /\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(currentOffer); + return /\bAside\b/.test(option) && /\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(currentOffer); }); } diff --git a/test/third-party-actions.test.ts b/test/third-party-actions.test.ts index 49a589ad3..23055df59 100644 --- a/test/third-party-actions.test.ts +++ b/test/third-party-actions.test.ts @@ -63,6 +63,9 @@ D) Defer test('allows recovery questions but rejects conditional drive consent', () => { expect(asideDriveOptions('A) Open the Aside app so I can re-run the probe.')).toEqual([]); + // Captured tpa-absent-linux answer (paid run 38988927989): the English idiom is not the Aside browser. + expect(asideDriveOptions('D) Defer. I set this aside and mark it blocked on you, with no browsing now.')).toEqual([]); + expect(asideDriveOptions('D) Defer, or I drive it in Aside later.')).toHaveLength(1); expect(asideDriveOptions('A) Open the Aside app; if READY, I drive the dashboard.')).toHaveLength(1); });