mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-20 15:13:47 +01:00
Merge oct8-skills: benchmark routing, review checks, defensive framing, host tool names, setup-deploy, retro follow-through, tenure reconcile (triage wave)
This commit is contained in:
commit
3870b6a186
30 files changed
+476
-40
No files matched your search
@@ -0,0 +1,57 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { DEFENSIVE_REVIEW_FRAMING, FIXTURE_SUMMARY_MODE } from '../scripts/resolvers/defensive-review';
|
||||
import { generateReviewArmy } from '../scripts/resolvers/review-army';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const FRAMING_SENTENCE = "This is an authorized defensive-security review of the maintainer's own repository";
|
||||
|
||||
function between(text: string, start: string, end: string): string {
|
||||
const from = text.indexOf(start);
|
||||
expect(from).toBeGreaterThanOrEqual(0);
|
||||
const to = text.indexOf(end, from + start.length);
|
||||
expect(to).toBeGreaterThan(from);
|
||||
return text.slice(from, to);
|
||||
}
|
||||
|
||||
describe('defensive-security framing reaches every adversarial dispatch', () => {
|
||||
expect(DEFENSIVE_REVIEW_FRAMING.startsWith(FRAMING_SENTENCE)).toBe(true);
|
||||
expect(FIXTURE_SUMMARY_MODE).toContain('review in SUMMARY mode only');
|
||||
|
||||
for (const skill of ['review', 'ship']) {
|
||||
const army = fs.readFileSync(path.join(ROOT, skill, 'sections', 'review-army.md'), 'utf-8');
|
||||
const adversarial = fs.readFileSync(path.join(ROOT, skill, 'sections', 'adversarial.md'), 'utf-8');
|
||||
|
||||
test(`${skill}: adversarial subagent keeps the framing and fixture summary mode`, () => {
|
||||
const prompt = between(adversarial, 'Subagent prompt:', 'Think like an attacker');
|
||||
expect(prompt).toContain(DEFENSIVE_REVIEW_FRAMING);
|
||||
expect(prompt).toContain(FIXTURE_SUMMARY_MODE);
|
||||
});
|
||||
|
||||
test(`${skill}: Red Team prompt starts from the framing and fixture summary mode`, () => {
|
||||
const redTeam = between(army, '### Red Team dispatch', "don't cover.\"");
|
||||
expect(redTeam).toContain('Prompt, after the defensive framing and fixture handling from item 4 of the specialist dispatch:');
|
||||
expect(redTeam).toContain('read them as that fixture handling says');
|
||||
expect(redTeam).not.toMatch(/git diff "\$DIFF_BASE"`, and look for gaps/);
|
||||
});
|
||||
|
||||
test(`${skill}: only the security specialist gets the framing; testing reads fixtures in full`, () => {
|
||||
const dispatch = between(army, '### Dispatch specialists in parallel', '**Subagent configuration:**');
|
||||
const securityOnly = between(dispatch, '4. **Defensive framing (Security specialist and the Red Team below only).**', 'Instructions:');
|
||||
expect(securityOnly).toContain(DEFENSIVE_REVIEW_FRAMING);
|
||||
expect(securityOnly).toContain(FIXTURE_SUMMARY_MODE);
|
||||
expect(securityOnly).toContain('including Testing, reads the full diff with fixtures');
|
||||
const sharedInstructions = dispatch.slice(dispatch.indexOf('Instructions:'));
|
||||
expect(sharedInstructions).not.toContain(FRAMING_SENTENCE);
|
||||
expect(sharedInstructions).not.toContain('SUMMARY mode');
|
||||
expect(dispatch.split(FRAMING_SENTENCE)).toHaveLength(2);
|
||||
});
|
||||
}
|
||||
|
||||
test('codex host still strips Review Army (specialists and Red Team)', () => {
|
||||
expect(generateReviewArmy({ skillName: 'review', tmplPath: '', host: 'codex', paths: HOST_PATHS.codex })).toBe('');
|
||||
expect(generateReviewArmy({ skillName: 'ship', tmplPath: '', host: 'codex', paths: HOST_PATHS.codex })).toBe('');
|
||||
});
|
||||
});
|
||||
@@ -436,6 +436,20 @@ describe('gen-skill-docs', () => {
|
||||
expect(content).not.toContain('## Completeness Principle');
|
||||
});
|
||||
|
||||
test('benchmark routing is scoped to web page performance', () => {
|
||||
const generated = fs.readFileSync(path.join(ROOT, 'benchmark', 'SKILL.md'), 'utf-8');
|
||||
const template = fs.readFileSync(path.join(ROOT, 'benchmark', 'SKILL.md.tmpl'), 'utf-8');
|
||||
for (const content of [generated, template]) {
|
||||
const useWhen = content.match(/Use when:[\s\S]*?(?=\n\n|\(gstack\))/)?.[0] ?? '';
|
||||
expect(useWhen).toContain('"web performance benchmark"');
|
||||
expect(useWhen).not.toMatch(/"benchmark"|"performance"/);
|
||||
expect(content).toContain('/benchmark-models');
|
||||
}
|
||||
const router = fs.readFileSync(path.join(ROOT, 'SKILL.md.tmpl'), 'utf-8');
|
||||
expect(router).toContain('web performance regression → invoke `/benchmark`');
|
||||
expect(router).toContain('model or skill benchmarks → invoke `/benchmark-models`');
|
||||
});
|
||||
|
||||
test('telemetry producer lives in the scripts; render documents the analytics sink', () => {
|
||||
// The skill-usage.jsonl producers moved into the scripts (Phase 1).
|
||||
expect(SKILL_START_SCRIPT).toContain('analytics/skill-usage.jsonl');
|
||||
|
||||
@@ -698,3 +698,111 @@ describe('gstack-developer-profile resources entries do not inflate count/tier/n
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// --reconcile: backfill tenure from office-hours timeline history (#2657)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
describe('gstack-developer-profile --reconcile', () => {
|
||||
const completed = (ts: string, extra: Record<string, unknown> = {}) =>
|
||||
JSON.stringify({ skill: 'office-hours', event: 'completed', ts, ...extra });
|
||||
|
||||
function writeTimeline(slug: string, lines: string[]) {
|
||||
const dir = path.join(tmpHome, 'projects', slug);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'timeline.jsonl'), lines.join('\n') + '\n');
|
||||
}
|
||||
|
||||
function logSession(entry: Record<string, unknown>) {
|
||||
expect(runDev('--log-session', JSON.stringify({ mode: 'builder', signal_count: 0, signals: [], ...entry })).status).toBe(0);
|
||||
}
|
||||
|
||||
const readKey = (out: string, key: string) => out.match(new RegExp(`^${key}: (.*)$`, 'm'))?.[1];
|
||||
|
||||
test('backfills one row per office-hours completed run and counts it for tenure', () => {
|
||||
writeTimeline('alpha', [
|
||||
completed('2026-09-01T10:00:00Z'),
|
||||
JSON.stringify({ skill: 'office-hours', event: 'started', ts: '2026-09-02T09:00:00Z' }),
|
||||
completed('2026-09-02T10:00:00Z'),
|
||||
JSON.stringify({ skill: 'review', event: 'completed', ts: '2026-09-03T10:00:00Z' }),
|
||||
]);
|
||||
writeTimeline('beta', [completed('2026-09-04T10:00:00Z')]);
|
||||
const r = runDev('--reconcile');
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('RECONCILE: 3 backfilled from 3 office-hours run(s)');
|
||||
const sessions = readProfile().sessions as Array<Record<string, unknown>>;
|
||||
expect(sessions).toHaveLength(3);
|
||||
expect(sessions[2]).toEqual({
|
||||
date: '2026-09-04T10:00:00Z', mode: 'unknown', project_slug: 'beta',
|
||||
signal_count: 0, signals: [], backfilled: true,
|
||||
});
|
||||
const read = runDev('--read').stdout;
|
||||
expect(readKey(read, 'SESSION_COUNT')).toBe('3');
|
||||
expect(readKey(read, 'TIER')).toBe('welcome_back');
|
||||
});
|
||||
|
||||
test('a second run adds nothing and leaves the profile file untouched', () => {
|
||||
writeTimeline('alpha', [completed('2026-09-01T10:00:00Z'), completed('2026-09-05T10:00:00Z')]);
|
||||
expect(runDev('--reconcile').stdout).toContain('RECONCILE: 2 backfilled');
|
||||
const file = path.join(tmpHome, 'developer-profile.json');
|
||||
const before = fs.readFileSync(file, 'utf-8');
|
||||
const old = new Date('2026-01-01T00:00:00Z');
|
||||
fs.utimesSync(file, old, old);
|
||||
expect(runDev('--reconcile').stdout).toContain('RECONCILE: 0 backfilled from 2 office-hours run(s)');
|
||||
expect(fs.readFileSync(file, 'utf-8')).toBe(before);
|
||||
expect(fs.statSync(file).mtimeMs).toBe(old.getTime());
|
||||
});
|
||||
|
||||
test('a logged session minutes before the run across an hour boundary covers it, per project', () => {
|
||||
logSession({ date: '2026-09-01T10:55:00Z', ts: '2026-09-01T10:55:00Z', project_slug: 'alpha' });
|
||||
writeTimeline('alpha', [completed('2026-09-01T11:20:00Z')]);
|
||||
writeTimeline('beta', [completed('2026-09-01T11:20:00Z')]);
|
||||
expect(runDev('--reconcile').stdout).toContain('RECONCILE: 1 backfilled from 2 office-hours run(s)');
|
||||
const backfilled = (readProfile().sessions as Array<Record<string, unknown>>).filter(s => s.backfilled);
|
||||
expect(backfilled.map(s => s.project_slug)).toEqual(['beta']);
|
||||
});
|
||||
|
||||
test('a long session logged hours before completion is not double-counted; garbage durations are not trusted', () => {
|
||||
logSession({ date: '2026-09-01T09:00:00Z', ts: '2026-09-01T09:00:00Z', project_slug: 'alpha' });
|
||||
logSession({ date: '2026-09-02T09:00:00Z', ts: '2026-09-02T09:00:00Z', project_slug: 'alpha' });
|
||||
writeTimeline('alpha', [
|
||||
completed('2026-09-01T12:00:00Z', { duration_s: '11000' }),
|
||||
completed('2026-09-02T19:00:00Z', { duration_s: '40000' }),
|
||||
]);
|
||||
expect(runDev('--reconcile').stdout).toContain('RECONCILE: 1 backfilled from 2 office-hours run(s)');
|
||||
const backfilled = (readProfile().sessions as Array<Record<string, unknown>>).filter(s => s.backfilled);
|
||||
expect(backfilled.map(s => s.date)).toEqual(['2026-09-02T19:00:00Z']);
|
||||
});
|
||||
|
||||
test('malformed timeline lines are skipped', () => {
|
||||
writeTimeline('alpha', ['{not json', 'null', completed('not-a-date'), completed('2026-09-01T10:00:00Z')]);
|
||||
const r = runDev('--reconcile');
|
||||
expect(r.status).toBe(0);
|
||||
expect(r.stdout).toContain('RECONCILE: 1 backfilled from 1 office-hours run(s)');
|
||||
});
|
||||
|
||||
test('backfilled rows never surface as LAST_*, DESIGN_* or CROSS_PROJECT', () => {
|
||||
logSession({
|
||||
date: '2026-09-01T10:00:00Z', ts: '2026-09-01T10:00:00Z', project_slug: 'alpha',
|
||||
assignment: 'Talk to five users', design_doc: 'docs/alpha-design.md',
|
||||
});
|
||||
writeTimeline('beta', [completed('2026-09-03T10:00:00Z'), completed('2026-09-04T10:00:00Z')]);
|
||||
expect(runDev('--reconcile').stdout).toContain('RECONCILE: 2 backfilled');
|
||||
const read = runDev('--read').stdout;
|
||||
expect(readKey(read, 'SESSION_COUNT')).toBe('3');
|
||||
expect(readKey(read, 'LAST_PROJECT')).toBe('alpha');
|
||||
expect(readKey(read, 'LAST_ASSIGNMENT')).toBe('Talk to five users');
|
||||
expect(readKey(read, 'LAST_DESIGN_TITLE')).toBe('docs/alpha-design.md');
|
||||
expect(readKey(read, 'DESIGN_COUNT')).toBe('1');
|
||||
expect(readKey(read, 'CROSS_PROJECT')).toBe('false');
|
||||
});
|
||||
|
||||
test('a backfill-only profile reaches welcome_back with empty LAST_*', () => {
|
||||
writeTimeline('alpha', [completed('2026-09-01T10:00:00Z')]);
|
||||
runDev('--reconcile');
|
||||
const read = runDev('--read').stdout;
|
||||
expect(readKey(read, 'TIER')).toBe('welcome_back');
|
||||
expect(readKey(read, 'LAST_PROJECT')).toBe('');
|
||||
expect(readKey(read, 'LAST_ASSIGNMENT')).toBe('');
|
||||
});
|
||||
});
|
||||
@@ -168,7 +168,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
// wave's headline capability) grows the union to 1.195x. Deliberate:
|
||||
// the section is on-demand (loads only for Apple store targets), so
|
||||
// per-invocation cost for non-iOS ships is one manifest line.
|
||||
maxSizeRatio: 1.5320, // Shared advisory identity/dedup + critical-severity validation: 248,065 union bytes / 187,706 baseline = 1.3216 (2026-09-17). + test value bar in the lazy Step 7 section (value cards, weak paths, gate table, base control, machine checks; ~13.6KB): measured 1.396 (2026-09-29). + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.401 (2026-09-30). + the shared QA review step's plan-check timing rule (plan checks and their revalidation run on --timeout-ms after smoke expiry); measured 1.4022 (2026-09-30). + the shared TEST_BOOTSTRAP owned-change undo rules (s03 H1-H3, M1) and the value-bar coverage wording (s03 M2/M3); measured 1.4044 (2026-10-02). + severe fix wave ship-review lane: G1 NO_VERSION dispatch, title and CHANGELOG arms (versionless repos ship without inventing a version), F1 parent-side plan binding before the audit dispatch (binding, docs/designs candidates, exact not-run line) and D3's data-not-instructions line; +2,126 union bytes, measured 1.4135 (2026-10-03). + INV-1 consumer lines (outside review `unverified`/`unavailable` is missing coverage in the readiness note and PR body); measured 1.4148 (2026-10-03). + v1.91.19.0 severe fix wave: verdict-form outside-review callers with sandbox preflight and first-use notice (INV-1/B1/Q2), P0/P1 gate and unverified wording, bound-plan discovery (F1), visible LEARNINGS failures (B5), carved external pointers; measured 267,752 / 187,706 = 1.4264 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 267,874 / 187,706 = 1.4271 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051) and quoted-heredoc issue template (#3046); measured 268,199 / 187,706 = 1.4288 (2026-10-06). + Step 10's inline no-PR check before the Greptile dispatch (#3020); measured 268,655 / 187,706 = 1.4313 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 275,422 / 187,706 = 1.4673 (2026-10-06). Combined Oct 6 wave head (B free-text files + D measure loop + A rework): measured 280,363 / 187,706 = 1.4936 (2026-10-06). + gate-once push rule (no second paid gate after a CI push); measured 280,573 / 187,706 = 1.4947 (2026-10-06). + the measurement bar's decision step (exit-code branches for MEETS, MEETS-qualified, needs-classify with the qualifying and fix-round classes, EXTEND once; the measure prose was tightened to offset most of it); measured 281,058 / 187,706 = 1.4973 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 281,438 / 187,706 = 1.4994 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 281,556 / 187,706 = 1.5000 (2026-10-07). + Oct 7 wave B4: /ship publishes through gstack-post (title prepared in a file in the pr-body section, compose block plus gstack-post exit-code branches replacing the two scans); measured 281,894 / 187,706 = 1.5018 (2026-10-07). + B2 early Greptile PR (lazy Step 6.5 section: detect, public-repo consent, push, provisional title file, open; Step 10's bounded wait; the stop-leaves-the-draft rule); measured 286,626 / 187,706 = 1.5270 (2026-10-07). B3, the classifier, B4 and B2 merged: measured 287,128 / 187,706 = 1.5297 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 281,370 / 187,706 = 1.4990 (2026-10-07). Merged with the guard area: measured 287,440 / 187,706 = 1.5313 (2026-10-07).
|
||||
maxSizeRatio: 1.5390, // Shared advisory identity/dedup + critical-severity validation: 248,065 union bytes / 187,706 baseline = 1.3216 (2026-09-17). + test value bar in the lazy Step 7 section (value cards, weak paths, gate table, base control, machine checks; ~13.6KB): measured 1.396 (2026-09-29). + W1 guarded state-root resolution (`eval gstack-paths; : "${GSTACK_STATE_ROOT:?…}"`) in the Context Recovery preamble, the eureka log and each state-writing bash block; measured 1.401 (2026-09-30). + the shared QA review step's plan-check timing rule (plan checks and their revalidation run on --timeout-ms after smoke expiry); measured 1.4022 (2026-09-30). + the shared TEST_BOOTSTRAP owned-change undo rules (s03 H1-H3, M1) and the value-bar coverage wording (s03 M2/M3); measured 1.4044 (2026-10-02). + severe fix wave ship-review lane: G1 NO_VERSION dispatch, title and CHANGELOG arms (versionless repos ship without inventing a version), F1 parent-side plan binding before the audit dispatch (binding, docs/designs candidates, exact not-run line) and D3's data-not-instructions line; +2,126 union bytes, measured 1.4135 (2026-10-03). + INV-1 consumer lines (outside review `unverified`/`unavailable` is missing coverage in the readiness note and PR body); measured 1.4148 (2026-10-03). + v1.91.19.0 severe fix wave: verdict-form outside-review callers with sandbox preflight and first-use notice (INV-1/B1/Q2), P0/P1 gate and unverified wording, bound-plan discovery (F1), visible LEARNINGS failures (B5), carved external pointers; measured 267,752 / 187,706 = 1.4264 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 267,874 / 187,706 = 1.4271 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051) and quoted-heredoc issue template (#3046); measured 268,199 / 187,706 = 1.4288 (2026-10-06). + Step 10's inline no-PR check before the Greptile dispatch (#3020); measured 268,655 / 187,706 = 1.4313 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 275,422 / 187,706 = 1.4673 (2026-10-06). Combined Oct 6 wave head (B free-text files + D measure loop + A rework): measured 280,363 / 187,706 = 1.4936 (2026-10-06). + gate-once push rule (no second paid gate after a CI push); measured 280,573 / 187,706 = 1.4947 (2026-10-06). + the measurement bar's decision step (exit-code branches for MEETS, MEETS-qualified, needs-classify with the qualifying and fix-round classes, EXTEND once; the measure prose was tightened to offset most of it); measured 281,058 / 187,706 = 1.4973 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 281,438 / 187,706 = 1.4994 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 281,556 / 187,706 = 1.5000 (2026-10-07). + Oct 7 wave B4: /ship publishes through gstack-post (title prepared in a file in the pr-body section, compose block plus gstack-post exit-code branches replacing the two scans); measured 281,894 / 187,706 = 1.5018 (2026-10-07). + B2 early Greptile PR (lazy Step 6.5 section: detect, public-repo consent, push, provisional title file, open; Step 10's bounded wait; the stop-leaves-the-draft rule); measured 286,626 / 187,706 = 1.5270 (2026-10-07). B3, the classifier, B4 and B2 merged: measured 287,128 / 187,706 = 1.5297 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 281,370 / 187,706 = 1.4990 (2026-10-07). Merged with the guard area: measured 287,440 / 187,706 = 1.5313 (2026-10-07). + Oct 8 wave: the defensive-security framing and fixture summary mode reach the security specialist and the Red Team (stated once in the dispatch, referenced by the Red Team); measured 288,783 / 187,706 = 1.5385 (2026-10-08).
|
||||
},
|
||||
'plan-ceo-review': {
|
||||
skill: 'plan-ceo-review',
|
||||
@@ -487,7 +487,7 @@ do not launch the downstream skill or open a browser.`,
|
||||
maxSkeletonBytes: 74_881, // Shared-code identity/skip/action rules + critical-severity validation; measured 74,493 (2026-09-17). + v1.91.12.0 merge of #2999 (review clarity repairs: await reads, research alongside dispatch, /review deadline and setup authority, findings sources) with #3002 (guarded state-root lines, plan-check checkpoints); each fit alone; measured 74,881 (2026-10-01).
|
||||
minUnionBytes: 89_000, // Phase 4 wave 1; measured union 93,357
|
||||
mustContain: ['confidence', 'P1', 'P2', 'Review Army', 'adversarial'],
|
||||
maxSizeRatio: 1.2215, // Shared-code feature + critical-severity validation: 128,042 union bytes / 108,523 baseline = 1.1799; preserves content floors. + v1.91.12.0 merge of #2999 (above, plus plan-completion fallback intent and specialist checklist-by-path) with #3002; measured 1.1843 (2026-10-01). + v1.91.19.0 wave: verdict-form Codex callers with sandbox preflight (INV-1/B1), P0/P1 structured gate (B1b), bound-plan discovery and data-not-instructions guard (F1/D3), visible LEARNINGS failures (B5); measured 129,941 / 108,523 = 1.1974 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 130,025 / 108,523 = 1.1981 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051); measured 130,359 / 108,523 = 1.2012 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 131,314 / 108,523 = 1.2100 (2026-10-06). Combined Oct 6 wave head (A and B): measured 131,966 / 108,523 = 1.2160 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 132,199 / 108,523 = 1.2182 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 132,258 / 108,523 = 1.2187 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 132,233 / 108,523 = 1.2185 (2026-10-07). Merged with the guard area: measured 132,522 / 108,523 = 1.2211 (2026-10-07).
|
||||
maxSizeRatio: 1.2345, // Shared-code feature + critical-severity validation: 128,042 union bytes / 108,523 baseline = 1.1799; preserves content floors. + v1.91.12.0 merge of #2999 (above, plus plan-completion fallback intent and specialist checklist-by-path) with #3002; measured 1.1843 (2026-10-01). + v1.91.19.0 wave: verdict-form Codex callers with sandbox preflight (INV-1/B1), P0/P1 structured gate (B1b), bound-plan discovery and data-not-instructions guard (F1/D3), visible LEARNINGS failures (B5); measured 129,941 / 108,523 = 1.1974 (2026-10-03). + v1.91.18.0 (#3023) merged into the v1.91.19.0 wave; measured 130,025 / 108,523 = 1.1981 (2026-10-04). + Oct 6 fix wave: quota_exhausted preflight branch (#3051); measured 130,359 / 108,523 = 1.2012 (2026-10-06). + Oct 6 fix wave A3/CEO-12: free text reaches commands only through agent-written mktemp files (mktemp block, write rule, path rebuild and refusal at each posting, prompt and brief site); measured 131,314 / 108,523 = 1.2100 (2026-10-06). Combined Oct 6 wave head (A and B): measured 131,966 / 108,523 = 1.2160 (2026-10-06). + Oct 7 wave B3: the Codex probe runs as a command (select-model capture lines, check-sandbox/probe-model/run-with-timeout subcommands) instead of being sourced; measured 132,199 / 108,523 = 1.2182 (2026-10-07). + Oct 7 wave CEO-28/DX-11: review prompts ask for a severity per finding and design proposals use the proposal gate; measured 132,258 / 108,523 = 1.2187 (2026-10-07). + Oct 7 wave guard (CEO-20): conditional `run_in_background` request plus the background-run recovery at each foreground dispatch site; measured 132,233 / 108,523 = 1.2185 (2026-10-07). Merged with the guard area: measured 132,522 / 108,523 = 1.2211 (2026-10-07). + Oct 8 wave: the defensive-security framing and fixture summary mode reach the security specialist and the Red Team (stated once in the dispatch, referenced by the Red Team); measured 133,902 / 108,523 = 1.2339 (2026-10-08).
|
||||
},
|
||||
codex: {
|
||||
skill: 'codex',
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
} from '../hosts/index';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
import { RESOLVERS } from '../scripts/resolvers';
|
||||
import { runGeneration } from '../scripts/gen-skill-docs';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const RESOLVER_NAMES = new Set(Object.keys(RESOLVERS));
|
||||
@@ -790,3 +791,29 @@ describe('host renders name the host\'s own tools and identities (#2626, #2015,
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('agent-runtime hosts never leak the Claude "Agent tool" name', () => {
|
||||
const OUT = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-agent-tool-'));
|
||||
afterAll(() => fs.rmSync(OUT, { recursive: true, force: true }));
|
||||
|
||||
test.each([
|
||||
['hermes', 'delegate_task'],
|
||||
['gbrain', 'sessions_spawn'],
|
||||
['openclaw', 'sessions_spawn'],
|
||||
['factory', 'delegation'],
|
||||
])('%s renders map every Agent tool phrasing to %s', async (host, token) => {
|
||||
const result = await runGeneration({ host: host as 'hermes', outputRoot: path.join(OUT, host), contentLinkRoot: null });
|
||||
expect(result.exitCode).toBe(0);
|
||||
const rendered = result.artifacts.filter(a => a.kind === 'skill' || a.kind === 'section');
|
||||
expect(rendered.length).toBeGreaterThan(20);
|
||||
const leaks: string[] = [];
|
||||
let mentions = 0;
|
||||
for (const artifact of rendered) {
|
||||
const content = fs.readFileSync(path.join(OUT, host, artifact.relativePath), 'utf8');
|
||||
if (/\bAgent tool\b/.test(content)) leaks.push(artifact.relativePath);
|
||||
if (content.includes(`via ${token}`)) mentions++;
|
||||
}
|
||||
expect(leaks).toEqual([]);
|
||||
expect(mentions).toBeGreaterThan(0);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Regression tests for #1834: /retro wrote "3 Things to Improve" as prose only,
|
||||
* so the next retro could not tell whether last week's recommendations were
|
||||
* acted on. The Step 13 snapshot now saves them and Step 12 scores them.
|
||||
*/
|
||||
import { describe, expect, test } from "bun:test";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, "..");
|
||||
const SKILL = fs.readFileSync(path.join(ROOT, "retro", "SKILL.md"), "utf-8");
|
||||
const REPORT = fs.readFileSync(path.join(ROOT, "retro", "sections", "report-format.md"), "utf-8");
|
||||
|
||||
function step(heading: string, next: string): string {
|
||||
const start = SKILL.indexOf(heading);
|
||||
expect(start).toBeGreaterThanOrEqual(0);
|
||||
const end = SKILL.indexOf(next, start + heading.length);
|
||||
expect(end).toBeGreaterThan(start);
|
||||
return SKILL.slice(start, end);
|
||||
}
|
||||
|
||||
describe("#1834 retro persists recommendations and detects follow-through", () => {
|
||||
const step12 = step("### Step 12: Load History & Compare", "### Step 13:");
|
||||
const step13 = step("### Step 13: Save Retro History", "### Step 14:");
|
||||
|
||||
test("Step 13 schema always saves the 3 Things to Improve as recommendations", () => {
|
||||
const schema = step13.match(/```json\n(\{[\s\S]*?\n\})\n```/)?.[1] ?? "";
|
||||
const parsed = JSON.parse(schema);
|
||||
expect(Array.isArray(parsed.recommendations)).toBe(true);
|
||||
expect(Object.keys(parsed.recommendations[0]).sort()).toEqual(["category", "text"]);
|
||||
expect(step13).toContain("**Always include `recommendations`:**");
|
||||
expect(step13).toContain("Write `[]` when there are none");
|
||||
expect(step13).toContain('draft the tweetable summary and the "3 Things to Improve" items');
|
||||
});
|
||||
|
||||
test("Step 12 scores prior recommendations and skips older snapshots silently", () => {
|
||||
expect(step12).toContain("**Recommendation follow-through:**");
|
||||
expect(step12).toMatch(/`addressed`, `partial` or `open`/);
|
||||
expect(step12).toContain("2 of 3 prior recommendations addressed");
|
||||
expect(step12).toContain("If the snapshot has no `recommendations` key, skip this silently.");
|
||||
});
|
||||
|
||||
test("report format ties the 3 Things to Improve to the saved items", () => {
|
||||
const improve = REPORT.slice(REPORT.indexOf("### 3 Things to Improve"), REPORT.indexOf("### 3 Habits for Next Week"));
|
||||
expect(improve).toContain("saved as `recommendations` in the Step 13 snapshot");
|
||||
expect(REPORT).toContain("Include the Recommendation follow-through line when Step 12 produced one.");
|
||||
});
|
||||
|
||||
test("global mode snapshot is unchanged", () => {
|
||||
expect(step("## Global Retrospective Mode", "## Compare Mode")).not.toContain("recommendations");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const SKILL = fs.readFileSync(path.resolve(import.meta.dir, '..', 'setup-deploy', 'SKILL.md'), 'utf-8');
|
||||
|
||||
function section(heading: string, next: string): string {
|
||||
const start = SKILL.indexOf(heading);
|
||||
expect(start).toBeGreaterThanOrEqual(0);
|
||||
return SKILL.slice(start, SKILL.indexOf(next, start + heading.length));
|
||||
}
|
||||
|
||||
describe('setup-deploy does not assume Vercel deploys on push', () => {
|
||||
test('auto-deploy is conditional on a Git-connected Vercel project', () => {
|
||||
const vercel = section('#### Vercel', '#### Netlify');
|
||||
expect(vercel).toContain('only when the Vercel project is connected to this Git repository');
|
||||
expect(vercel).toContain('For a CLI-only project, ask how production deploys run and record that command as the deploy trigger');
|
||||
});
|
||||
|
||||
test('the nothing-detected path explains how a CLI-only Vercel project becomes detectable', () => {
|
||||
const manual = section('#### Custom / Manual', '1. **How are deploys triggered?**');
|
||||
expect(manual).toContain('deployed only with the Vercel CLI has no `vercel.json` or `.vercel/`');
|
||||
expect(manual).toContain('`vercel link` creates `.vercel/`');
|
||||
expect(SKILL).not.toMatch(/GoLive/i);
|
||||
});
|
||||
});
|
||||
@@ -939,6 +939,16 @@ describe('Enum & Value Completeness in review checklist', () => {
|
||||
expect(checklist).toContain('allowlist');
|
||||
});
|
||||
|
||||
test('Enum & Value Completeness covers loosened input acceptance and stale user-facing strings', () => {
|
||||
const start = checklist.indexOf('#### Enum & Value Completeness');
|
||||
const section = checklist.slice(start, checklist.indexOf('### Pass 2', start));
|
||||
expect(section).toContain('loosens what an input accepts');
|
||||
expect(section).toContain('Loosened acceptance breaks unchanged consumers');
|
||||
const maintainability = fs.readFileSync(path.join(ROOT, 'review', 'specialists', 'maintainability.md'), 'utf-8');
|
||||
const stale = maintainability.slice(maintainability.indexOf('### Stale Comments & Docstrings'));
|
||||
expect(stale.slice(0, stale.indexOf('\n### ', 5))).toContain('User-facing strings (errors, toasts, labels) whose guarding condition changed');
|
||||
});
|
||||
|
||||
test('Enum & Value Completeness is in the severity classification as CRITICAL', () => {
|
||||
const gateSection = checklist.slice(checklist.indexOf('## Severity Classification'));
|
||||
// The ASCII art has CRITICAL on the left and INFORMATIONAL on the right
|
||||
|
||||
Reference in new issue
Block a user