mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 05:56:37 +01:00
v1.91.219.0 fix(eval-bg): local lanes export EVALS_RUN_ID so native-capture cases start (GSTA-70) (#3391)
* fix(eval-bg): local lanes export EVALS_RUN_ID so native-capture cases start (GSTA-70) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.219.0 stamp Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
d73b547066
commit
f73f75ce55
6 files changed
+62
-6
No files matched your search
@@ -1,5 +1,15 @@
|
||||
# Changelog
|
||||
|
||||
## [1.91.219.0] - 2026-10-25
|
||||
|
||||
**Local paid runs from `eval:bg` now set `EVALS_RUN_ID`, so the native-capture cases start instead of failing in 0 seconds.** CI sets `EVALS_RUN_ID` per slice, but `bun run eval:bg:pr --local` launched without it, and the qa-callers, ship-docsync and qa-functional cases refuse to start without it. In GSTA-70's local /ship run, 17 of the 26 selected behaviors failed that way for $0. A failure like that is a launch failure, not a verdict, so the local run told you nothing about those cases.
|
||||
|
||||
### Fixed
|
||||
- **`scripts/eval-bg.ts`**: the local backend launches through a new `launchLocal`. It exports `EVALS_RUN_ID` as the run-scoped log's name, reduced to `[A-Za-z0-9_.-]`, and keeps a value you exported yourself. The id is written on the log's second line, so you can find the run's native captures. `--help` says so. Dispatched CI runs are unchanged.
|
||||
|
||||
### Tests
|
||||
- `test/eval-detach-timeout-floor.test.ts`: a detached child launched by `launchLocal` sees a safe `EVALS_RUN_ID` named after its log, and an exported id is kept. A source check confirms that the local backend launches through `launchLocal`.
|
||||
|
||||
## [1.91.218.0] - 2026-10-25
|
||||
|
||||
**`gstack-team-init required` now blocks skills when its gate can't run, instead of letting them through.** Issue #3319 (exGeni): the required-mode Skill hook is a `node -e` launcher. On a teammate's Mac or Linux machine without Node.js, the shell exits 127 with nothing on stdout. If `.claude/hooks/check-gstack.cjs` is missing, node exits 1. Claude Code treats any exit other than 0 or 2 as a non-blocking error, so in both cases the Skill call went ahead on exactly the machine the gate is for. A live Claude Code 2.1.310 session with node off PATH and no gstack install ran the skill.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# gstack digest v1.91.218.0 — regenerate/re-copy after upgrading gstack
|
||||
# gstack digest v1.91.219.0 — regenerate/re-copy after upgrading gstack
|
||||
|
||||
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
|
||||
for agent hosts without a full skill install. The full skills add workflows,
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gstack",
|
||||
"version": "1.91.218",
|
||||
"version": "1.91.219",
|
||||
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
|
||||
+19
-2
@@ -295,6 +295,22 @@ export function launchDetached(opts: { log: string; label: string; timeoutSecond
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
/** The run id a local lane exports: the run-scoped log's name, reduced to [A-Za-z0-9_.-] as the native-capture cases require. */
|
||||
export function localRunId(log: string): string {
|
||||
return path.basename(log, '.log').replace(/[^\w.-]/g, '-');
|
||||
}
|
||||
|
||||
/**
|
||||
* The local backend: CI sets EVALS_RUN_ID per slice, and the native-capture
|
||||
* cases refuse to start without it, so a local lane exports one too (an
|
||||
* exported value is kept). The id is logged under the header.
|
||||
*/
|
||||
export function launchLocal(opts: { log: string; label: string; timeoutSeconds: number; header: string; command: string[]; env?: NodeJS.ProcessEnv }): string {
|
||||
const env = opts.env ?? process.env;
|
||||
const runId = env.EVALS_RUN_ID || localRunId(opts.log);
|
||||
return launchDetached({ ...opts, header: `${opts.header}\n[eval-bg] EVALS_RUN_ID=${runId}`, env: { ...env, EVALS_RUN_ID: runId } });
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const pollMs = () => Number(process.env.GSTACK_EVAL_BG_POLL_MS) || 30_000;
|
||||
|
||||
@@ -362,7 +378,8 @@ log; it ends with "### gstack-detach EXIT=<code> ###" (0 passed, 1 failed,
|
||||
|
||||
Backend: a clean HEAD pushed to ${CANONICAL_REPO} dispatches the CI workflow and
|
||||
follows it; anything else runs locally under bin/gstack-detach, capped at
|
||||
ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min (at most 4 h).
|
||||
ceil(1.5 x planned serial seconds / EVALS_JOBS) + 20 min (at most 4 h), with
|
||||
EVALS_RUN_ID set to the log's name unless one is exported.
|
||||
--local / --dispatch force a backend (also GSTACK_EVAL_BG_MODE=local|dispatch);
|
||||
--dispatch refuses instead of falling back
|
||||
--timeout SECS override the computed local cap
|
||||
@@ -417,7 +434,7 @@ async function main(argv: string[]): Promise<number> {
|
||||
const header = `[eval-bg] backend=local lane=${lane} tested=${tested} cap=${cap.seconds}s jobs=${jobs}${timeoutOverride !== null ? ' (--timeout)' : ''} (${choice.reason})`;
|
||||
console.log(header);
|
||||
if (cap.capped) console.log(`[eval-bg] WARNING: planned work exceeds the ${LOCAL_CAP_SECONDS}s local cap; the tail may not run. Dispatch to CI or raise EVALS_JOBS.`);
|
||||
console.log(launchDetached({ log, label, timeoutSeconds: cap.seconds, header, command: ['bun', 'run', LANE_SPECS[lane].script] }));
|
||||
console.log(launchLocal({ log, label, timeoutSeconds: cap.seconds, header, command: ['bun', 'run', LANE_SPECS[lane].script] }));
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ import * as path from 'node:path';
|
||||
import {
|
||||
CANONICAL_REPO, DISPATCH_FOLLOW_SECONDS, EVAL_BG_LANES, LANE_SPECS, LOCAL_CAP_SECONDS, UNRECORDED_SHARD_MS,
|
||||
chooseBackend, conclusionExitCode, dedupeKey, dispatchBlockers, dispatchPlan, findDuplicateRun, launchDetached,
|
||||
localCapSeconds, makeNonce, plannedSerialMs, statusOf, type CheckoutFacts, type RunSummary,
|
||||
launchLocal, localCapSeconds, localRunId, makeNonce, plannedSerialMs, statusOf, type CheckoutFacts, type RunSummary,
|
||||
} from '../scripts/eval-bg';
|
||||
import { buildRunManifest, loadPaidTestDurations, recordedShardMs } from '../scripts/test-paid-shards';
|
||||
|
||||
@@ -158,6 +158,35 @@ describe('completion contract on both backends (DX-2)', () => {
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}, 40_000);
|
||||
|
||||
// GSTA-70: the native-capture cases (qa-callers, ship-docsync, qa-functional)
|
||||
// refuse to start without EVALS_RUN_ID. CI sets it per slice; the local lane
|
||||
// used to launch without it, so 17 of /ship's PR cases failed in 0 s.
|
||||
test('local: the lane exports a safe EVALS_RUN_ID named after its log, and keeps an exported one', async () => {
|
||||
const { dir, env } = sandbox();
|
||||
try {
|
||||
delete env.EVALS_RUN_ID;
|
||||
const probe = (out: string) => [process.execPath, '-e',
|
||||
`const id = process.env.EVALS_RUN_ID ?? ''; require('fs').writeFileSync(${JSON.stringify(out)}, id); process.exit(/^[\\w.-]+$/.test(id) ? 0 : 4)`];
|
||||
const log = path.join(dir, 'evals-pr-gstack-feature+x-20261024-200431-1.log');
|
||||
launchLocal({ log, label: 'evals-test', timeoutSeconds: 60, header: '[eval-bg] backend=local lane=pr tested=abc', env, command: probe(path.join(dir, 'id')) });
|
||||
const text = await waitForExit(log);
|
||||
expect(localRunId(log)).toBe('evals-pr-gstack-feature-x-20261024-200431-1');
|
||||
expect(fs.readFileSync(path.join(dir, 'id'), 'utf8')).toBe(localRunId(log));
|
||||
expect(text.split('\n').slice(0, 2)).toEqual(['[eval-bg] backend=local lane=pr tested=abc', `[eval-bg] EVALS_RUN_ID=${localRunId(log)}`]);
|
||||
expect(text).toMatch(/### gstack-detach EXIT=0 ###/);
|
||||
const kept = path.join(dir, 'kept.log');
|
||||
launchLocal({ log: kept, label: 'evals-test', timeoutSeconds: 60, header: 'h', env: { ...env, EVALS_RUN_ID: 'mine-1' }, command: probe(path.join(dir, 'kept')) });
|
||||
expect(await waitForExit(kept)).toMatch(/### gstack-detach EXIT=0 ###/);
|
||||
expect(fs.readFileSync(path.join(dir, 'kept'), 'utf8')).toBe('mine-1');
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
}, 40_000);
|
||||
|
||||
test('the local backend launches through launchLocal', () => {
|
||||
const source = fs.readFileSync(path.join(ROOT, 'scripts/eval-bg.ts'), 'utf8');
|
||||
expect(source).toMatch(/console\.log\(launchLocal\(\{ log, label, timeoutSeconds: cap\.seconds, header, command: \['bun', 'run', LANE_SPECS\[lane\]\.script\] \}\)\)/);
|
||||
expect(source.match(/launchDetached\(\{ log, label/g)?.length).toBe(1);
|
||||
});
|
||||
|
||||
for (const [conclusion, code, word] of [['failure', 1, 'failed'], ['cancelled', 130, 'cancelled'], ['success', 0, 'passed']] as const) {
|
||||
test(`dispatch: a followed run that ends ${conclusion} writes EXIT=${code}`, async () => {
|
||||
const { dir, env } = sandbox();
|
||||
|
||||
Reference in new issue
Block a user