Files
gstack/test/docsync-lifecycle-interface.test.ts
T
ac20ef1e13 v1.91.16.0 fix: safe installs and upgrades, skills start everywhere, truthful completion (parallel wave) (#3014)
* fix(make-pdf,design): real TOC page numbers and anchors; design models move to gpt-5.5 + gpt-image-2 (#2903, #2807)

* ci: pin every action to a SHA, Scorecard, checksummed Bun, scoped Codex CLI in the CI image, macOS named-regressions job (#1948, #1706, #1997)

* fix(cso,browse): Windows/macOS lease identity, retention skips bad runs, truthful 'not assessed' reports, env.kv false positive, Windows browse spawn fixes (#2894, #2912, #2048, #2637, #2805)

* fix(outside-voice): one Codex model precedence for probe and dispatch, bounded timeouts with KILL escalation, probe cache identity, guarded mktemp, skills hidden in nested Codex (#2776, #2914, #2787, #2881, #2847)

* feat(setup): install registry, ./setup --status, staged activation, per-scope migrations, host tiers and conformance kit, OpenCode/Hermes fixes, Copilot CLI (experimental), disabled_skills, slimmer CLAUDE.md (#1925, #2879, #2347, #2338, #2626, #2629, #2825, #2015, #393, #1206, #2096)

Co-authored-by: Andrey Esipov <61357035+andrey-esipov@users.noreply.github.com>
Co-authored-by: Matthew Williams <5863886+ridermw@users.noreply.github.com>
Co-authored-by: Rocky Wu <92764750+lolisaigao1234@users.noreply.github.com>

* fix(preamble): skills start in worktree-isolated Claude Code sessions — one literal skill-start line, context-recovery helper, no eval in skill blocks, $N token tripwire, honest plan-mode precedence (#2763, #2896, #2851)

* fix(setup): per-install renders for renamed, vendored and custom-config installs, space-safe alias roots, per-host /cso launcher path (#1882, #2906)

* test: free tests never write the real HOME (home-write tripwire), no orphaned browse daemons, symlinked-TMPDIR (macOS) fixes, gbrain-on-PATH isolation (#2895, #2816, #2829, #2841, #2982, #2911, #2909, #2908, #2910)

* chore: integrate wave follow-ups — regenerate all hosts, README model docs, TODOS, touchfiles, duration seeds, Bun 1.3 telemetry test fix, replay fixtures pinned to their recorded bytes

* v1.91.14.0: release metadata

* test(browse): group-kill cleanup falls back to the leader when macOS returns EPERM for a live group

* ci: Codex login steps run under bash (the CI container's default sh rejects pipefail)

* fix: full-suite repairs — private state roots for 7 real-HOME writers, truthful home-guard attribution with --attribute-home, gstack-paths shim in ship-hook fixtures, Copilot in team-init probes, test-of-test detector, cookie review fixture pinned to the approved excerpt

* fix(windows): LF for Dockerfiles, normalized home-guard exclusions, path.relative fixture containment, ship push test spawn timeouts

* test(design-detect): widen the whole-scan budget test's timing margins so a loaded macOS runner can't time out the single-batch call

* fix(evals): qa protocol names defined for the judge, async PTY judge so overlapping sessions keep their cleanup deadline, bws setup cases use the standard JUDGE_MS tier

* test(pty): the non-blocking judge case is POSIX-only (its fake judge is a shebang script, like the PTY harness it covers)

* test(evals): ship-coverage-audit names absolute skill paths, uses Read, and skips the Step 7 artifact nothing consumes (removes the mechanism behind 3 of 4 historical timeouts)

* test: deflake terminal-agent concurrent startup (write the placeholder record before spawning, like the product) and qa-evidence resumed pipe (barrier on a full high-water mark instead of a sleep)

* fix(design): daemon log stream handles open errors instead of crashing the daemon; shutdown test awaits the stubbed exit

* test(evals): skillify checks the product's resolved state root (no GSTACK_HOME bypass), negative cases check every root; tpa shims live outside the agent's cwd and answer consistently

* test(evals): coverage-audit detector splits parallel columns at │ rails and [+] groups; coverage prompts name absolute skill paths; TODOs from the gate eval census

* fix(plan-eng-review): scope gate always sends the menu (no headless 'Scope pending' exit); PTY harness ends idle turn-ends and API-error panels promptly with a precise diagnostic

* test(evals): one position-bound coverage-read rule credits read-only && chains (labels, context ranges, git displays in either order) and still rejects forged output

* fix(ship): documentation section has one saved source inserted by cat (never retyped); spawned-marker evidence is the child output; refreshed codex/factory ship goldens

* test(docsync): the byte-exact insert positive is POSIX-only (the bash-command observer refuses backslash paths by design)

* test(evals): plan-design-with-ui focus-title rule accepts the comma-less 'dimensions or focus' wording (replayed on the captured CI run, negative controls kept)

* fix(qa-callers): exploratory preflight says the scope/method Reads come after exploratory.md; design-review scan check credits an executed scan proven by one engine run over the changed files

---------

Co-authored-by: Andrey Esipov <61357035+andrey-esipov@users.noreply.github.com>
Co-authored-by: Matthew Williams <5863886+ridermw@users.noreply.github.com>
Co-authored-by: Rocky Wu <92764750+lolisaigao1234@users.noreply.github.com>
2026-10-03 07:54:09 -07:00

130 lines
7.7 KiB
TypeScript

import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { fixtureDocs } from './helpers/docsync-fixture';
import { docsCommandAllowed, docsNativeInterface, docsPreambleCommands, docsToolFailures } from './helpers/docsync-observer';
import type { SkillTestResult } from './helpers/session-runner';
let fixture: ReturnType<typeof fixtureDocs>;
beforeAll(() => { fixture = fixtureDocs('updated'); });
afterAll(() => fixture?.clean());
function lifecycleCommands() {
return [...docsNativeInterface(fixture).matchAll(/```bash\n([^`]+)\n```/g)].map(match => match[1]);
}
test('fixture lifecycle guidance exposes owned skill paths and literal commands', () => {
const guidance = docsNativeInterface(fixture);
const skills = fixture.skills.split(path.sep).join('/');
expect(guidance).toContain(`${skills}/document-release/SKILL.md`);
const [start, end] = lifecycleCommands();
expect(lifecycleCommands()).toHaveLength(2);
expect(start).toBe(`GSTACK_SESSION_KIND=spawned ${skills}/bin/gstack-skill-start --skill document-release --model claude`);
expect(end).toBe(`${skills}/bin/gstack-skill-end --skill document-release --outcome OUTCOME --session-id SESSION_ID_VALUE --tel-start TEL_START_VALUE --used-browse no`);
expect(docsCommandAllowed(start, fixture)).toBe(true);
for (const command of [start, end]) expect(command).not.toMatch(/[\n\r~$\\|<>;]/);
expect(docsCommandAllowed(start.replaceAll('/', '\\'), fixture)).toBe(false);
const source = fs.readFileSync(path.join(import.meta.dir, '../bin/gstack-skill-start'), 'utf8');
expect(source).toContain('if [ -z "$PARENT_PID" ]; then');
expect(start).not.toContain('--parent-pid');
});
test.each(['success', 'error', 'abort', 'unknown'])('same-session literal end accepts outcome %s without wrappers', outcome => {
const end = lifecycleCommands()[1].replace('OUTCOME', outcome).replace('SESSION_ID_VALUE', '123-1790299423-control').replace('TEL_START_VALUE', '1790299423');
expect(docsCommandAllowed(end, fixture)).toBe(true);
const result = { toolCalls: [{ tool: 'Bash', input: { command: end }, output: '' }] } as SkillTestResult;
expect(docsToolFailures(result, fixture)).toEqual([]);
});
test('historical misplaced spawned prefix remains rejected while corrected preamble stays supported', () => {
const [original, spawned] = docsPreambleCommands(fixture);
// The pre-#2763 wrapper put a misplaced prefix on its _SS assignment.
const historical = '_SS="$HOME/.claude/skills/gstack/bin/gstack-skill-start"\n[ -x "$_SS" ] || _SS=".claude/skills/gstack/bin/gstack-skill-start"\n"$_SS" --skill "document-release" --model "claude" --parent-pid "$PPID"';
expect(docsCommandAllowed(`GSTACK_SESSION_KIND=spawned ${historical}`, fixture)).toBe(false);
expect(original).toMatch(/^\S+\/gstack-skill-start --skill "document-release" --model "claude"$/);
expect(spawned).toBe(`GSTACK_SESSION_KIND=spawned ${original}`);
expect(docsCommandAllowed(spawned, fixture)).toBe(true);
expect(docsCommandAllowed(original, fixture)).toBe(true);
});
test.each([
['missing preamble heading', '## Other section\n\n```bash\n"$_SS" --skill "document-release" --model "claude" --parent-pid "$PPID"\n```\n'],
['missing preamble code fence', '## Preamble (run first)\n\nNo executable preamble is present.\n'],
['unrelated fenced code', '## Preamble (run first)\n\n```text\nnot shell\n```\n\n```bash\necho not the preamble\n```\n'],
['malformed Bash block', '## Preamble (run first)\n\n```bash\necho unrelated command\n```\n'],
])('malformed source with %s grants no generated-shell exception', (label, content) => {
const file = path.join(fixture.skills, 'document-release/SKILL.md');
const original = fs.readFileSync(file, 'utf8');
try {
fs.writeFileSync(file, content);
expect(docsPreambleCommands(fixture), label).toEqual([]);
expect(docsCommandAllowed('git status --porcelain', fixture)).toBe(true);
expect(docsCommandAllowed('bun arbitrary.ts', fixture)).toBe(false);
expect(docsCommandAllowed('git status && node attack.js', fixture)).toBe(false);
} finally {
fs.writeFileSync(file, original);
}
});
test('appending shell to the preamble block invalidates the exact generated command', () => {
const file = path.join(fixture.skills, 'document-release/SKILL.md');
const original = fs.readFileSync(file, 'utf8');
const [command] = docsPreambleCommands(fixture);
try {
fs.writeFileSync(file, `## Preamble (run first)\n\n\`\`\`bash\n${command}\necho unauthorized\n\`\`\`\n`);
expect(docsPreambleCommands(fixture)).toEqual([]);
expect(docsCommandAllowed(command, fixture)).toBe(false);
expect(docsCommandAllowed('git status --porcelain', fixture)).toBe(true);
} finally {
fs.writeFileSync(file, original);
}
});
test.each([
'~/.claude/skills/gstack/bin/gstack-skill-end --skill "document-release" --outcome success \\\n --session-id "176-1790299423-4e0464a5" --tel-start "1790299423" --used-browse no \\\n --error-message "" --failed-step "" 2>/dev/null || true',
'~/.claude/skills/gstack/bin/gstack-skill-end --skill "document-release" --outcome success --session-id "933-1790299569-1f0c2bb1" --tel-start "1790299569" --used-browse no --error-message "" --failed-step "" 2>/dev/null || true',
])('historical end wrapper remains rejected: %s', command => {
expect(docsCommandAllowed(command, fixture)).toBe(false);
const result = { toolCalls: [{ tool: 'Bash', input: { command }, output: 'SKILL_END: recorded outcome=success' }] } as SkillTestResult;
expect(docsToolFailures(result, fixture)).toContain('command outside declared docs observation interface');
});
test('lifecycle instructions neither widen command authority nor grant mutation approvals', () => {
expect(docsNativeInterface(fixture)).toContain('The only additional scripts are none');
expect(docsNativeInterface(fixture, ['/owned/publish.ts'])).toContain('The only additional scripts are /owned/publish.ts');
for (const suffix of [' 2>/dev/null', ' || true', ' && git status', '\ntrue', ' $EXTRA']) {
expect(docsCommandAllowed(lifecycleCommands()[0] + suffix, fixture)).toBe(false);
}
expect(docsCommandAllowed('bun arbitrary.ts', fixture)).toBe(false);
});
test('Git guidance uses the existing working directory; --git-dir and --work-tree stay unauthorized', () => {
const guidance = docsNativeInterface(fixture);
expect(guidance).toContain(fixture.repo);
for (const command of ['git status', 'git diff --cached', 'git merge-base main HEAD', 'git rev-parse HEAD']) {
expect(guidance).toContain(command);
expect(docsCommandAllowed(command, fixture)).toBe(true);
}
for (const command of [`git --git-dir ${fixture.repo}/.git status`, `git --work-tree ${fixture.repo} status`]) {
expect(docsCommandAllowed(command, fixture)).toBe(false);
}
});
describe('stored docsync observer commands', () => {
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/docsync-observer-commands.json'), 'utf8')) as
{ known_good: string[]; known_bad: string[] };
const fill = (command: string) => command.replaceAll('{repo}', fixture.repo).replaceAll('{other}', path.dirname(fixture.repo));
test.each(stored.known_good)('harmless read form is accepted: %s', command => {
expect(docsCommandAllowed(fill(command), fixture)).toBe(true);
});
test.each(stored.known_bad)('known-bad form stays rejected: %s', command => {
expect(docsCommandAllowed(fill(command), fixture)).toBe(false);
});
test('harmless wrappers never extend to lifecycle commands', () => {
for (const command of lifecycleCommands()) {
for (const suffix of [' 2>/dev/null', ' || true']) expect(docsCommandAllowed(command + suffix, fixture)).toBe(false);
expect(docsCommandAllowed(`cd ${fixture.repo} && ${command}`, fixture)).toBe(false);
}
});
});