fix(design-e2e): the shim-dom scan check resolves the command's literal _REPORT/_RUN (W3-10)

PR gate run 38980005793 went red because the agent ran the generated DOM
block, scanning "$_REPORT/dom/$_RUN" with _RUN="run1" assigned in the same
command, and the check looked for the literal dom/run1 anywhere in a command.

domScanOfRun substitutes that command's own literal _REPORT=/_RUN=
assignments and credits a scan only when one of its own arguments is the run
directory. scan --changed, another directory, quoted heredoc report text and
non-literal or conflicting assignments stay negative controls. All 15
captured shim-dom runs in the window pass the new check (14 passed the old).

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-17 19:36:14 +00:00
1 parent fcfba0ec91
commit 20d77403e4
2 files changed
+82 -1

No files matched your search

@@ -252,3 +252,64 @@ test.each(['directory', 'write-error'] as const)('source detector preserves its
if (retention === 'directory') expect(result.retained).toHaveLength(1);
else expect(result.notices.some(note => note.includes('Detector artifact write failed:'))).toBe(true);
});
// PR gate run 38980005793 (design-review-detector-shim-dom): the agent scanned "$_REPORT/dom/$_RUN" with _RUN="run1"
// assigned in the same command, and the check looked for the literal dom/run1. These rows run the real
// `domScanOfRun` (and the `executedShellText` it uses) from test/skill-e2e-design.test.ts.
const declaration = (name: string) => {
const start = source.indexOf(`function ${name}(`);
const end = source.indexOf('\n}\n', start);
if (start < 0 || end < 0) throw new Error(`${name} not found in test/skill-e2e-design.test.ts`);
return source.slice(start, end + 2);
};
const domScanModule = new Bun.Transpiler({ loader: 'ts' }).transformSync(
`globalThis.__domScanOfRun = (() => { ${declaration('executedShellText')}\n${declaration('domScanOfRun')}\nreturn domScanOfRun; })();`);
new Function(domScanModule)();
const domScanOfRun = (globalThis as any).__domScanOfRun as (command: string, runDir: string) => boolean;
const REPORT = '/fixture/gstack-home/projects/shim/designs/design-audit-20260908';
const RUN_DIR = `${REPORT}/dom/run1`;
// Captured from 38980005793 (design-review-detector-shim-dom, head 3f4a98ecac3fc434f8aeef61bc9def17080718d9);
// /fixture/gstack-home stands for the fixture's GSTACK_HOME and /checkout for the checkout.
const CAPTURED = `_D="/tmp/gstack-dom.fDVsi2/home.dom.html"; _REPORT="${REPORT}"; _RUN="run1"
if [ ! -s "$_D" ]; then echo "DOM_DUMP_MISSING: home (the dump script wrote nothing)"
elif [ "$(wc -c < "$_D")" -gt 10485760 ]; then echo "DOM_DUMP_TOO_LARGE: home $(wc -c < "$_D")"; rm -f "$_D"
elif /checkout/bin/gstack-redact --from-file "$_D" --max-bytes 10485760 >/dev/null 2>&1; _RC=$?; [ "$_RC" -ne 0 ] && [ "$_RC" -ne 2 ]; then echo "DOM_DUMP_REDACTION_BLOCKED: home redact-exit=$_RC"; rm -f "$_D"
else mkdir -p "$_REPORT/dom/$_RUN" && cp "$_D" "$_REPORT/dom/$_RUN/" && chmod 600 "$_REPORT/dom/$_RUN/home.dom.html" && rm -f "$_D" && echo "DOM_DUMP_OK: home"; fi
echo "REDACT_RC=$_RC"; ls -la "$_REPORT/dom/$_RUN/" 2>/dev/null
if [ -s "$_REPORT/dom/$_RUN/home.dom.html" ]; then
_DJ=$(mktemp "/tmp/gstack-detect.XXXXXX"); bun --no-env-file run /checkout/bin/gstack-design-detect.ts scan --format gstack --host claude "$_REPORT/dom/$_RUN" > "$_DJ" 2>/tmp/gstack-detect.stderr; echo "DETECT_EXIT_CODE=$?"; echo "DETECT_JSON=$_DJ"
echo "----STDERR----"; cat /tmp/gstack-detect.stderr
echo "----JSON----"; cat "$_DJ"
fi`;
const SCAN = 'bun --no-env-file run /checkout/bin/gstack-design-detect.ts scan --format gstack --host claude';
test('shim-dom: the captured generated DOM block counts as a scan of the run directory', () => {
expect(domScanOfRun(CAPTURED, RUN_DIR)).toBe(true);
});
test('shim-dom: literal, braced, bare and trailing-slash forms of the same scan count', () => {
for (const command of [
`${SCAN} "${RUN_DIR}" > out.json`,
`${SCAN} ${RUN_DIR}/`,
`_REPORT=${REPORT} _RUN=run1; ${SCAN} $_REPORT/dom/$_RUN`,
`_REPORT='${REPORT}'; _RUN='run1'; ${SCAN} "${'$'}{_REPORT}/dom/${'$'}{_RUN}"`,
]) expect(domScanOfRun(command, RUN_DIR), command).toBe(true);
});
test('shim-dom: a source-mode scan, another directory, report text and unresolvable assignments never count', () => {
for (const command of [
CAPTURED.replace('scan --format gstack --host claude', 'scan --changed main --format gstack --host claude'),
CAPTURED.replace('"$_REPORT/dom/$_RUN" >', '--changed "$_REPORT/dom/$_RUN" >'),
CAPTURED.replace('_RUN="run1"', '_RUN="run2"'),
CAPTURED.replace('"$_REPORT/dom/$_RUN" >', '"$_REPORT/dom/other" >'),
CAPTURED.replace('_RUN="run1"', '_RUN="run1"; _RUN="run2"'),
CAPTURED.replace('_RUN="run1"', '_RUN="$(date +%s)"'),
CAPTURED.replace('_RUN="run1"', '_RUN=$RUN_ID'),
CAPTURED.replace(`_REPORT="${REPORT}"`, '_REPORT="$HOME/report"'),
`mkdir -p "${RUN_DIR}" && ${SCAN} /elsewhere`,
`cat > detector-output.md <<'EOF'\nScan: \`${SCAN} ${RUN_DIR}\`\nEOF`,
`echo "${SCAN} ${RUN_DIR}"`.replace('scan --format', 'scan-later --format'),
`${SCAN} ${RUN_DIR}1`,
]) expect(domScanOfRun(command, RUN_DIR), command).toBe(false);
});
+21 -1
View File
@@ -814,6 +814,26 @@ function sourceScanAgainstMain(commands: string[], invocations: string[], repoDi
return targets.join() === 'index.html,styles.css';
}
/** CI 38980005793: the agent ran the generated DOM block, which scans `"$_REPORT/dom/$_RUN"` with `_RUN="run1"`
* assigned in the same command. A DOM scan counts when one of its own arguments, after that command's literal
* `_REPORT=`/`_RUN=` assignments are substituted, is the run directory; `scan --changed` (source mode), a scan of
* another directory, quoted heredoc text and a non-literal or conflicting assignment never count. */
function domScanOfRun(command: string, runDir: string): boolean {
const shell = executedShellText(command);
const assigned = new Map<string, Set<string>>();
for (const m of shell.matchAll(/(?<![\w$])(_REPORT|_RUN)=(?:"([^"$`\\]*)"|'([^']*)'|([^\s;&|<>()"'$`\\]*))(?=[\s;&|)]|$)/g)) {
assigned.set(m[1]!, (assigned.get(m[1]!) ?? new Set()).add(m[2] ?? m[3] ?? m[4] ?? ''));
}
const resolved = shell.replace(/\$(?:\{(_REPORT|_RUN)\}|(_REPORT|_RUN)(?!\w))/g, (all, braced?: string, bare?: string) => {
const values = assigned.get((braced ?? bare)!);
return values?.size === 1 ? [...values][0]! : all;
});
return [...resolved.matchAll(/gstack-design-detect\.ts scan ([^\n;&|<>]*)/g)].some(scan => {
const args = (scan[1]!.match(/"[^"]*"|'[^']*'|\S+/g) ?? []).map(arg => arg.replace(/^(["'])(.*)\1$/, '$2').replace(/\/+$/, ''));
return !args.includes('--changed') && args.includes(runDir);
});
}
if (!evalsEnabled) test('plugin handoff credits an executed source scan whose base resolved to main', () => {
const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plugin-scan-replay-'));
try {
@@ -1245,7 +1265,7 @@ Do not run npx. Do not fix anything.`,
expect(result.exitReason).toBe('success');
const bash = result.toolCalls.filter(c => c.tool === 'Bash').map(c => String(c.input?.command ?? ''));
expect(bash.some(c => c.includes('dom-dump.js') && c.includes('--out') && c.includes('--raw'))).toBe(true); // $B js '('"$_DUMP"')()' with the file spliced in
expect(bash.some(c => /gstack-design-detect\.ts scan /.test(c) && c.includes('dom/run1'))).toBe(true);
expect(bash.some(c => domScanOfRun(c, path.join(allowed, 'dom', 'run1')))).toBe(true);
expect(bash.some(c => /gstack-design-detect\.ts scan --changed/.test(c))).toBe(false);
const dumps = fs.readdirSync(path.join(allowed, 'dom', 'run1')).filter(f => f.endsWith('.dom.html'));
expect(dumps.length).toBeGreaterThan(0);