mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 16:34:00 +01:00
v1.94.42.0 test: red-team retry, docsync replay and plan-seed fixture hold up on loaded runners (GSTA-262) (#3954)
* test: red-team retry, docsync replay and plan-seed fixture hold up on loaded runners (GSTA-262) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: mark the two intended timing bounds with gstack-timing-ok (GSTA-262) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
2a368135de
commit
e9b40bbd82
7 files changed
+27
-13
No files matched your search
@@ -1,5 +1,13 @@
|
||||
# Changelog
|
||||
|
||||
## [1.94.42.0] - 2026-11-03
|
||||
|
||||
### Tests
|
||||
- Three free tests that failed only on a busy remote test machine (remote pregate run 20261103T015240Z-11756-6ff6, PR #3892) now hold up under load:
|
||||
- `test/review-army-budget.test.ts`: the red-team retry fixture spent a real 80 ms sleep inside its 200 ms deadline, so a pause of more than 120 ms failed attempt 2. Attempt 1 now waits for the cancellation and then drains, and attempt 2 doesn't sleep. With random 150 ms pauses injected, 6 of 10 runs passed before and 20 of 20 pass now.
|
||||
- `test/helpers/docsync-fault-callbacks.ts`: the docsync replay assumed fixture setup took less than 5 seconds, and setup takes 5 to 12 seconds on a busy machine. The replay now checks that the session timeout equals the budget minus 15 seconds minus the time already spent, which is how the code computes it. Under 16 busy loops on 4 cores, 0 of 4 runs passed before and 4 of 4 pass now.
|
||||
- `test/fixtures/plan-seed-cli.ts`: the fake CLI drew its screen and its startup question in two separate writes, so a screen read between them saw an empty input and pasted the plan. It now draws both in one write. Under load, 31 of 36 runs passed before and 36 of 36 pass now.
|
||||
|
||||
## [1.94.41.0] - 2026-11-03
|
||||
|
||||
**/ship's receipt counts its review findings by id, keeps every run, and a small change after review gets one reader instead of a second /ship (GSTA-258 L4; GST-1136, GST-1168, GST-1179).** The receipt's `REVIEW_LOOP` counts were wrong in a repeatable way: gbrain-evals#310's receipt said 3 blockers found and 2 fixed when 2 were found, 1 fixed and 1 deferred; gstack-evals#174 counted 16 blocking with 8 unrecorded against 9 found and 9 fixed; #3906 and #3911 showed `fixed=0` after real fixes. The summary paired findings by path:line only, so a native finding with no path:line, a row logged after its fix moved the line, or a late defer re-logged under another fingerprint all broke the count, and rows it couldn't place were dropped without a word.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# gstack digest v1.94.41.0 — regenerate/re-copy after upgrading gstack
|
||||
# gstack digest v1.94.42.0 — regenerate/re-copy after upgrading gstack
|
||||
|
||||
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
|
||||
for agent hosts without a full skill install. The full skills add workflows,
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gstack",
|
||||
"version": "1.94.41",
|
||||
"version": "1.94.42",
|
||||
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
|
||||
Vendored
+5
-5
@@ -22,10 +22,10 @@ const row=(type,content,stop)=>JSON.stringify({type,sessionId:sid,cwd,message:{r
|
||||
const text=s=>[{type:'text',text:s}];
|
||||
const append=(type,content,stop)=>fs.appendFileSync(file,row(type,content,stop));
|
||||
const rule='─'.repeat(120);
|
||||
const frame=(s,history='',top=rule,bottom=rule)=>{
|
||||
const frame=(s,history='',top=rule,bottom=rule,tail='')=>{
|
||||
const inputRows=1+(s.match(/\r\n/g)||[]).length;
|
||||
const topRow=(process.stdout.rows||40)-inputRows-2;
|
||||
process.stdout.write('\x1b[2J\x1b[H'+history+'\x1b['+topRow+';1H'+top+'\r\n❯ '+s+'\r\n'+bottom+'\r\npaste again to expand');
|
||||
process.stdout.write('\x1b[2J\x1b[H'+history+'\x1b['+topRow+';1H'+top+'\r\n❯ '+s+'\r\n'+bottom+'\r\npaste again to expand'+tail);
|
||||
};
|
||||
let input='',seed='',submitted=false;
|
||||
process.stdin.setRawMode(true);process.stdin.resume();
|
||||
@@ -40,12 +40,12 @@ else if(scenario==='startup-ci-placeholder')frame(process.env.CI==='true'&&proce
|
||||
else if(scenario==='startup-ci-typed-hint')frame(hint);
|
||||
else if(scenario==='startup-placeholder-cursor')frame('\x1b[7mT\x1b[27m\x1b[2m'+hint.slice(1)+'\x1b[22m');
|
||||
else if(scenario==='startup-placeholder-unicode')frame('\x1b[2mTry "refactor src/設定.ts"\x1b[22m');
|
||||
else if(['startup-placeholder','startup-prior-conversation','startup-missing-styles','startup-waiting','startup-prose-question','startup-permission','startup-fresh-waiting'].includes(scenario))frame('\x1b[2m'+hint+'\x1b[22m');
|
||||
else if(['startup-placeholder','startup-prior-conversation','startup-missing-styles','startup-waiting','startup-prose-question','startup-permission','startup-fresh-waiting'].includes(scenario))frame('\x1b[2m'+hint+'\x1b[22m','',rule,rule,
|
||||
scenario==='startup-prose-question'?'Which option do you prefer?\r\nA) Full review (recommended)\r\nB) Skip review\r\n'
|
||||
:scenario==='startup-permission'?'Bash command run checks requires permission\r\n':'');
|
||||
else if(scenario==='startup-typed-hint')frame(hint);
|
||||
else if(scenario==='startup-partial-dim')frame('\x1b[2mTry \x1b[22m"refactor <filepath>"');
|
||||
else frame('');
|
||||
if(scenario==='startup-prose-question')process.stdout.write('Which option do you prefer?\r\nA) Full review (recommended)\r\nB) Skip review\r\n');
|
||||
if(scenario==='startup-permission')process.stdout.write('Bash command run checks requires permission\r\n');
|
||||
process.stdin.on('data',chunk=>{
|
||||
input+=chunk.toString();
|
||||
if(input.startsWith('\x1b[200~')&&input.endsWith('\x1b[201~')){
|
||||
|
||||
@@ -25,7 +25,7 @@ const { parseDocsCompletion } = await import(path.join(root, 'test/helpers/docsy
|
||||
const { docsActorSeeded, DOCS_SEEDED_AUDIT_ID } = await import(path.join(root, 'test/helpers/docsync-fault-actor.ts'));
|
||||
const CALLBACK_TEST_MS = ${CALLBACK_TEST_MS};
|
||||
const callbacks = new Map();
|
||||
let fixture, control = '', launches = 0, recorded, legacyReaudit = false;
|
||||
let fixture, control = '', launches = 0, recorded, legacyReaudit = false, bodyStartedAt = 0;
|
||||
let returnedResult: SkillTestResult | undefined;
|
||||
let consumers: string[] = [];
|
||||
function expectResult(result: SkillTestResult) {
|
||||
@@ -47,7 +47,7 @@ mock.module(observerModule, () => ({ ...observers,
|
||||
mock.module(path.join(root, 'test/helpers/e2e-gate.ts'), () => ({ describeE2ETier: () => (_name, body) => body() }));
|
||||
mock.module(path.join(root, 'test/helpers/e2e-helpers.ts'), () => ({
|
||||
runId: 'free-docsync-faults', describeIfSelected: (_name, _names, body) => body(),
|
||||
testConcurrentIfSelected: (name, body, budget) => callbacks.set(name, { body, budget }),
|
||||
testConcurrentIfSelected: (name, body, budget) => callbacks.set(name, { body: () => { bodyStartedAt = Date.now(); return body(); }, budget }),
|
||||
createEvalCollector: () => ({}), finalizeEvalCollector() {},
|
||||
logCost(_name, result) {
|
||||
expectResult(result);
|
||||
@@ -65,8 +65,10 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({ async ru
|
||||
const legacy = options.testName === 'ship-docsync-failure';
|
||||
const scenario = legacy ? 'legacy-completion' : options.testName.slice('ship-docsync-'.length);
|
||||
expect(options.maxTurns).toBe(legacy ? 30 : 24);
|
||||
expect(options.timeout).toBeGreaterThan((legacy ? CAPTURE_LONG_MS : CAPTURE_MS) - 20_000);
|
||||
expect(options.timeout).toBeLessThanOrEqual((legacy ? CAPTURE_LONG_MS : CAPTURE_MS) - 15_000);
|
||||
const remaining = (legacy ? CAPTURE_LONG_MS : CAPTURE_MS) - 15_000;
|
||||
// gstack-timing-ok: runShipDocsFault derives the timeout from elapsed time, so the bound measures the same elapsed time
|
||||
expect(options.timeout).toBeGreaterThanOrEqual(remaining - (Date.now() - bodyStartedAt));
|
||||
expect(options.timeout).toBeLessThanOrEqual(remaining);
|
||||
expect(options.tools).toEqual(['Bash', 'Read', 'Write', 'Edit', 'Glob', 'Grep']);
|
||||
expect(options.allowedTools).toEqual(options.tools);
|
||||
expect(options.prompt).toContain('BOUNDED_CALLBACK_USED');
|
||||
|
||||
@@ -108,7 +108,11 @@ const run = async opts => {
|
||||
fixturePaths.push(opts.workingDirectory);
|
||||
captureOptions.push({ timeout: opts.timeout, maxTurns: opts.maxTurns, model: opts.model ?? null, signal: !!opts.signal });
|
||||
opts.signal?.addEventListener('abort', () => events.push('abort-' + attempt), { once: true });
|
||||
await Bun.sleep(attempt === 1 ? captureMs + drainMs : 80);
|
||||
if (attempt === 1) {
|
||||
await new Promise(resolve => opts.signal.addEventListener('abort', resolve, { once: true }));
|
||||
// gstack-timing-ok: the drain after cancellation is the behavior under test; no deadline remains during it
|
||||
await Bun.sleep(drainMs);
|
||||
}
|
||||
expect(fs.existsSync(opts.workingDirectory)).toBe(true);
|
||||
events.push('capture-done-' + attempt);
|
||||
if (attempt === 2) fs.writeFileSync(path.join(opts.workingDirectory, 'review-output.md'), ${JSON.stringify(validReport ? 'RED TEAM REVIEW' : 'ordinary unrelated prose')});
|
||||
|
||||
Reference in new issue
Block a user