CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-11 07:29:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

test: consent-gate E2E suite + functional fs-capability probes

Browse Source 
Five hermetic gate-tier E2E cases (tpa-present / absent-linux / broken /
absent-darwin / apple-ban) drive the real contract section through
claude -p with PATH shims for aside and uname; the absent cases filter
any REAL aside binary out of the child PATH and assert absence with
Bun.which before spawning, so dev machines cannot leak into detection.
Registered per-case in E2E_TOUCHFILES/E2E_TIERS with template-level
deps (ship/SKILL.md.tmpl, gen-skill-docs.ts) and added to the evals.yml
matrix with tier: gate. eval:bg:periodic's detach timeout rises to
36000s for the grown periodic shard census (floor-enforced by
test/eval-detach-timeout-floor.test.ts); CLAUDE.md doc updated to match.

test/helpers/fs-caps.ts adds canRevokeWrites/canRevokeReads functional
probes; 13 chmod-based tests swap their uid-0-only guards for the
probes so suites skip honestly on CAP_DAC_OVERRIDE containers (this
sandbox: uid 1000 with full caps) instead of asserting revocations the
kernel ignores. path-validation's symlink test targets /etc/passwd
(exists everywhere; /etc/crontab is absent on Amazon Linux).
This commit is contained in:
Garry Tan
2026-08-28 04:47:50 +00:00
parent ade441afd0
commit 0873fce89a
21 changed files with 375 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/fs-caps.ts
+58
View File
@@ -0,0 +1,58 @@
/**
* Functional filesystem-capability probe for tests that simulate failure by
* revoking permissions (chmod 0500 a dir, then assert writes fail).
*
* The existing `process.getuid?.() === 0` guards catch only literal root —
* but containers with CAP_DAC_OVERRIDE, user-namespace sandboxes, and some
* overlay/fuse mounts ignore mode bits for non-root users too (observed:
* Vercel sandbox, uid 1000, writes succeed in a 0500 dir). There the
* "unwritable" simulation silently holds nothing and the test asserts a
* failure that never happens. This probe tests the actual behavior once per
* process instead of guessing from the uid.
*/
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
let cachedWrite: boolean | null = null;
let cachedRead: boolean | null = null;
/** True when chmod 0000 on a file actually blocks this process's reads. */
export function canRevokeReads(): boolean {
if (cachedRead !== null) return cachedRead;
if (process.platform === 'win32' || process.getuid?.() === 0) return (cachedRead = false);
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fs-caps-'));
const file = path.join(dir, 'probe');
fs.writeFileSync(file, 'x');
fs.chmodSync(file, 0o000);
try {
fs.readFileSync(file);
cachedRead = false; // read succeeded → mode bits are not enforced here
} catch {
cachedRead = true;
} finally {
fs.chmodSync(file, 0o600);
fs.rmSync(dir, { recursive: true, force: true });
}
return cachedRead;
}
/** True when chmod 0500 on a directory actually blocks this process's writes. */
export function canRevokeWrites(): boolean {
if (cachedWrite !== null) return cachedWrite;
if (process.platform === 'win32' || process.getuid?.() === 0) return (cachedWrite = false);
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'fs-caps-'));
const locked = path.join(dir, 'locked');
fs.mkdirSync(locked);
fs.chmodSync(locked, 0o500);
try {
fs.writeFileSync(path.join(locked, 'probe'), 'x');
cachedWrite = false; // write succeeded → mode bits are not enforced here
} catch {
cachedWrite = true;
} finally {
fs.chmodSync(locked, 0o700);
fs.rmSync(dir, { recursive: true, force: true });
}
return cachedWrite;
}
test/helpers/touchfiles-data.ts
+11
View File
@@ -133,6 +133,11 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
'budget-regression-pty': ['test/helpers/eval-store.ts', 'test/skill-budget-regression.test.ts'],
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'tpa-broken': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'tpa-absent-darwin': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'tpa-apple-ban': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts'],
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
// Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design,
@@ -547,6 +552,12 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
'plan-design-with-ui-scope': 'gate', // ~$0.80/run
'budget-regression-pty': 'gate', // free, library-only assertion
'ship-idempotency-pty': 'periodic', // ~$3/run, real /ship in plan mode
'tpa-present': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts
'tpa-absent-linux': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts
'tpa-broken': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts
'tpa-absent-darwin': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts
'tpa-apple-ban': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts
'ship-section-loading': 'periodic', // ~$3/run, real /ship; asserts section reads
'plan-ceo-section-loading': 'periodic', // ~$3-5/run, real /plan-ceo-review; asserts section read
'carve-section-loading': 'periodic', // ~$1-2/skill, data-driven; GSTACK_CARVE_SKILL scopes to one