test(evals): docsync grammar corpus forms run on POSIX only; Windows fails closed (GSTA-49)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-19 01:26:48 +00:00
1 parent 892ddd4300
commit 13459589df
2 files changed
+8 -3

No files matched your search

+4 -1
View File
@@ -121,12 +121,15 @@ test('Git guidance uses the existing working directory; --git-dir and --work-tre
describe('stored docsync observer commands', () => {
const stored = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/docsync-observer-commands.json'), 'utf8')) as
{ known_good: string[]; known_bad: string[] };
{ known_good: string[]; known_good_posix: string[]; known_bad: string[] };
const fill = (command: string) => command.replaceAll('{repo}', fixture.repo).replaceAll('{other}', path.dirname(fixture.repo))
.replaceAll('{home}', fixture.home);
test.each(stored.known_good)('harmless read form is accepted: %s', command => {
expect(docsCommandAllowed(fill(command), fixture)).toBe(true);
});
test.skipIf(process.platform === 'win32').each(stored.known_good_posix)('shared read-only grammar form is accepted: %s', command => {
expect(docsCommandAllowed(fill(command), fixture)).toBe(true);
});
test.each(stored.known_bad)('known-bad form stays rejected: %s', command => {
expect(docsCommandAllowed(fill(command), fixture)).toBe(false);
});
+4 -2
View File
@@ -1,5 +1,5 @@
{
"_comment": "Bash commands from docsync E2E runs and their mutations. {repo} is the fixture repo, {other} a directory outside it, {home} the fixture home. known_good: read-only forms the observer must accept. known_bad: forms it must keep rejecting. known_bad also keeps genuine model misses from PR gate runs as negative controls: the composed cat; cat of ship-docsync-failure 38984977679 (bash#12). The Bash grep of ship-docsync-missing-asset 38983306684 (bash#13) is a pure read inside the fixture home, so the shared read-only grammar (GSTA-49) accepts it, as it does the Opus slips wc -l (39173752116) and ls with a glob (effort study t10); its tee, outside-path, recursive-credential and denied-glob mutations stay in known_bad.",
"_comment": "Bash commands from docsync E2E runs and their mutations. {repo} is the fixture repo, {other} a directory outside it, {home} the fixture home. known_good: read-only forms the observer must accept. known_bad: forms it must keep rejecting. known_bad also keeps genuine model misses from PR gate runs as negative controls: the composed cat; cat of ship-docsync-failure 38984977679 (bash#12). The Bash grep of ship-docsync-missing-asset 38983306684 (bash#13) is a pure read inside the fixture home, so the shared read-only grammar (GSTA-49) accepts it in known_good_posix (the grammar resolves POSIX paths only and accepts nothing on Windows), as it does the Opus slips wc -l (39173752116) and ls with a glob (effort study t10); its tee, outside-path, recursive-credential and denied-glob mutations stay in known_bad.",
"known_good": [
"git status",
"git -C {repo} status",
@@ -11,7 +11,9 @@
"git status --porcelain 2>/dev/null",
"cat docs/guide.md 2>/dev/null || true",
"git rev-parse HEAD || true",
"ls docs 2>/dev/null",
"ls docs 2>/dev/null"
],
"known_good_posix": [
"grep -n -i \"14.5\\|documentation\\|Step 15\\|Step 16\" {home}/.claude/skills/gstack/ship/SKILL.md",
"wc -l {home}/.claude/skills/gstack/ship/SKILL.md",
"ls -la {home}/.claude/skills/gstack/ship/sections {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/document-release/*"