mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
Merge commit 'dc09764' into garrytan/severe-fix-wave-oct3
This commit is contained in:
commit
10fdb81cec
8 files changed
+142
-150
No files matched your search
@@ -114,7 +114,7 @@ jobs:
|
||||
- name: Emit run manifest (ALL periodic tests minus reasoned excludes)
|
||||
env:
|
||||
EVALS_ALL: "1"
|
||||
run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 540 --jobs 2 --max-parallel 25
|
||||
run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 540 --jobs 2 --max-parallel 26
|
||||
|
||||
# Codex-only slices run on the host-run eval-codex-slices job; the
|
||||
# container executor runs every other slice.
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
fail-fast: false
|
||||
# Every planned slice starts at once; test/evals-workflow-wiring.test.ts
|
||||
# fails when the live plan outgrows this cap.
|
||||
max-parallel: 25
|
||||
max-parallel: 26
|
||||
matrix:
|
||||
slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_slices) }}
|
||||
steps:
|
||||
@@ -308,7 +308,7 @@ jobs:
|
||||
packages: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 25
|
||||
max-parallel: 26
|
||||
matrix:
|
||||
slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}
|
||||
steps:
|
||||
|
||||
@@ -165,6 +165,7 @@
|
||||
"test/skill-e2e-plan-design-finding-floor.test.ts": 155538,
|
||||
"test/skill-e2e-plan-design-plan-mode.test.ts": 106608,
|
||||
"test/skill-e2e-plan-devex-peer-comparison-classification.test.ts": 79786,
|
||||
"test/skill-e2e-plan-eng-artifact.test.ts": 498474,
|
||||
"test/skill-e2e-plan-eng-finding-floor.test.ts": 388666,
|
||||
"test/skill-e2e-plan-eng-multi-finding-batching.test.ts": 597500,
|
||||
"test/skill-e2e-plan-eng-plan-mode.test.ts": 229226,
|
||||
@@ -176,7 +177,6 @@
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review-expansion-energy": 91984,
|
||||
"test/skill-e2e-plan.test.ts#plan-ceo-review-selective": 267212,
|
||||
"test/skill-e2e-plan.test.ts#plan-eng-review": 128506,
|
||||
"test/skill-e2e-plan.test.ts#plan-eng-review-artifact": 164137,
|
||||
"test/skill-e2e-qa-bugs.test.ts": 333847,
|
||||
"test/skill-e2e-qa-workflow.test.ts": 403953,
|
||||
"test/skill-e2e-retro.test.ts": 166711,
|
||||
|
||||
@@ -139,7 +139,7 @@ test('live periodic census fits the declared CI wall including setup', () => {
|
||||
|
||||
test('registered allocation is deterministic and preserves every discovered file', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
expect(files).toHaveLength(106);
|
||||
expect(files).toHaveLength(107);
|
||||
expect(files).toContain('test/skill-e2e-ship-skip.test.ts');
|
||||
const m = livePlan(files);
|
||||
expect(livePlan([...files].reverse())).toEqual(m);
|
||||
|
||||
@@ -10,7 +10,7 @@ import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { runPlanSkillObservation, runPlanSkillCounting, runPlanSkillFloorCheck } from './claude-pty-runner';
|
||||
import { readEngTestPlans, runPlanSkillObservation, runPlanSkillCounting, runPlanSkillFloorCheck } from './claude-pty-runner';
|
||||
import { createFakePtyDriver, type FakeFrame, type FakeSessionContext } from './pty/fake-session';
|
||||
import { FORCING_FLOOR_DEVEX } from '../fixtures/forcing-finding-seeds';
|
||||
|
||||
@@ -228,3 +228,21 @@ describe('runPlanSkillFloorCheck through the fake driver', () => {
|
||||
expect(obs.summary).toBe('agent reached plan_ready without a qualifying finding question');
|
||||
});
|
||||
});
|
||||
|
||||
describe('readEngTestPlans', () => {
|
||||
test('returns only /plan-eng-review QA test plans under projects/<slug>/ of the owned state root', () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'eng-test-plans-'));
|
||||
try {
|
||||
expect(readEngTestPlans(root)).toEqual([]);
|
||||
expect(readEngTestPlans(undefined)).toEqual([]);
|
||||
fs.mkdirSync(path.join(root, 'projects', 'fixture-slug'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'projects', 'fixture-slug', 'runner-main-eng-review-test-plan-20261004-010203.md'), '# Test Plan\nDashboard');
|
||||
fs.writeFileSync(path.join(root, 'projects', 'fixture-slug', 'runner-main-ship-test-plan-20261004-010203.md'), '# Ship plan');
|
||||
fs.writeFileSync(path.join(root, 'projects', 'fixture-slug', 'tasks-eng-review-20261004-010203.jsonl'), '{}');
|
||||
fs.writeFileSync(path.join(root, 'eng-review-test-plan-outside-projects.md'), 'stray');
|
||||
expect(readEngTestPlans(root)).toEqual([{ file: path.join('projects', 'fixture-slug', 'runner-main-eng-review-test-plan-20261004-010203.md'), content: '# Test Plan\nDashboard' }]);
|
||||
} finally {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -24,7 +24,7 @@ export type { ReviewReportAtBottomResult, NativePlanTerminalReview, NativePlanTe
|
||||
export { ceoStep0Boundary, engSetupAUQ, engFirstReviewAUQ, engStep0Boundary, pickDesignFocusAll } from './pty/boundaries';
|
||||
export { runPlanSkillObservation } from './pty/runners/observation';
|
||||
export type { PlanSkillObservation, PlanSkillObservationOptions } from './pty/runners/observation';
|
||||
export { runPlanSkillCounting, countingCapture, isNativeCompletionSummary } from './pty/runners/counting';
|
||||
export type { PlanSkillCountObservation, PlanSkillCountingOptions, CountingRun } from './pty/runners/counting';
|
||||
export { runPlanSkillCounting, countingCapture, isNativeCompletionSummary, readEngTestPlans } from './pty/runners/counting';
|
||||
export type { EngTestPlan, PlanSkillCountObservation, PlanSkillCountingOptions, CountingRun } from './pty/runners/counting';
|
||||
export { planFloorDXPane, planFloorDXReplyInput, runPlanSkillFloorCheck } from './pty/runners/floor';
|
||||
export type { PlanSkillFloorObservation, PlanFloorDXReply } from './pty/runners/floor';
|
||||
@@ -2,6 +2,7 @@
|
||||
* runPlanSkillCounting. Moved from claude-pty-runner.ts.
|
||||
* Import through test/helpers/claude-pty-runner.ts from tests; pty/ modules import siblings directly.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { createPlanCountFixture } from '../../plan-count-fixture';
|
||||
import { createPlanCountSnapshotWriter } from '../../plan-count-artifacts';
|
||||
@@ -62,6 +63,21 @@ export interface PlanSkillCountObservation {
|
||||
reviewCount: number;
|
||||
/** Answered administrative handoffs and artifact rendering, preserved separately. */
|
||||
administrativeCount: number;
|
||||
/** QA test plans (`projects/<slug>/*-eng-review-test-plan-*.md`) in the run's owned state root at the outcome. */
|
||||
engTestPlans: EngTestPlan[];
|
||||
}
|
||||
|
||||
export interface EngTestPlan { file: string; content: string }
|
||||
|
||||
/** The /plan-eng-review QA test-plan artifacts under one state root, read before fixture cleanup. */
|
||||
export function readEngTestPlans(stateRoot: string | undefined): EngTestPlan[] {
|
||||
const projects = stateRoot ? path.join(stateRoot, 'projects') : '';
|
||||
if (!projects || !fs.existsSync(projects)) return [];
|
||||
return fs.readdirSync(projects, { withFileTypes: true }).filter(slug => slug.isDirectory()).flatMap(slug =>
|
||||
fs.readdirSync(path.join(projects, slug.name)).filter(name => /-eng-review-test-plan-.*\.md$/.test(name)).sort().map(name => {
|
||||
const file = path.join('projects', slug.name, name);
|
||||
return { file, content: fs.readFileSync(path.join(stateRoot!, file), 'utf8') };
|
||||
}));
|
||||
}
|
||||
|
||||
/** Options for runPlanSkillCounting. */
|
||||
@@ -94,7 +110,8 @@ export interface PlanSkillCountingOptions {
|
||||
isReviewAUQ?: (fp: AskUserQuestionFingerprint, priorCalls?: readonly NativePlanQuestionCall[]) => boolean;
|
||||
/** Stop a collection-only fixture once its acknowledged inputs are complete.
|
||||
* This is not review completion or a passing verdict; the caller still validates them. */
|
||||
isCollectionComplete?: (transcript: PlanCountTranscript, fingerprints: readonly AskUserQuestionFingerprint[]) => boolean;
|
||||
isCollectionComplete?: (transcript: PlanCountTranscript, fingerprints: readonly AskUserQuestionFingerprint[],
|
||||
engTestPlans: readonly EngTestPlan[]) => boolean;
|
||||
/** Narrow caller-specific selection; null retains the normal answer policy.
|
||||
* The first argument retains full pending metadata for existing callers.
|
||||
* Native-bound selection uses activeCapture, whose metadata is present only
|
||||
@@ -272,7 +289,7 @@ export interface CountingRun {
|
||||
cleanupReserveMs: number;
|
||||
/** Monotonic work cutoff: budget start + timeout - cleanup reserve. */
|
||||
workDeadline: number;
|
||||
fixture: { cwd: string };
|
||||
fixture: { cwd: string; env: Record<string, string> };
|
||||
defaultPick: number;
|
||||
pickerContext: Readonly<{ cwd: string; deadlineAt: number }>;
|
||||
saveSnapshot: ReturnType<typeof createPlanCountSnapshotWriter>;
|
||||
@@ -353,6 +370,7 @@ function countingSnapshot(
|
||||
step0Count: run.step0Count,
|
||||
reviewCount: run.reviewCount,
|
||||
administrativeCount: run.administrativeCount,
|
||||
engTestPlans: readEngTestPlans(run.fixture.env.GSTACK_STATE_ROOT),
|
||||
};
|
||||
const artifacts = countingCapture(run, session, observation);
|
||||
Object.assign(observation, artifacts);
|
||||
@@ -418,7 +436,7 @@ async function countingTick(run: CountingRun, session: ClaudePtySession): Promis
|
||||
if (opts.isCollectionComplete && !pending && transcript.status === 'ready' &&
|
||||
transcript.calls.length > 0 && transcript.calls.every(call => call.answered && !call.failed) &&
|
||||
!unresolvedPlanQuestionCalls(transcript.calls).length && remainingWork(run) > 0 &&
|
||||
opts.isCollectionComplete(transcript, run.fingerprints) && remainingWork(run) > 0) {
|
||||
opts.isCollectionComplete(transcript, run.fingerprints, readEngTestPlans(run.fixture.env.GSTACK_STATE_ROOT)) && remainingWork(run) > 0) {
|
||||
return { done: countingSnapshot(run, session, 'collection_complete', 'Caller-defined native collection is complete; final validation remains required', visible) };
|
||||
}
|
||||
const terminal = await countingTerminal(run, visible, pending);
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* /plan-eng-review writes its QA test-plan artifact when run the way users run
|
||||
* it (periodic, paid, real-PTY).
|
||||
*
|
||||
* Replaces the composing `plan-eng-review-artifact` case in skill-e2e-plan.test.ts.
|
||||
* That case told a `claude -p` session to skip the preamble and questions and
|
||||
* "go straight to the review"; in runs 37151477069, 37158847998 and 37162480720
|
||||
* (and one local CI-image run) the model wrote the whole review into one file
|
||||
* without executing the workflow, so it measured composing, not the skill. A
|
||||
* full interactive run does write the artifact (37158847998's batching PTY run
|
||||
* wrote eng-review-test-plan-20261003-224617.md during Test review).
|
||||
*
|
||||
* This case drives the real interactive /plan-eng-review over the same
|
||||
* dashboard plan and sources, answers each question with its first option, and
|
||||
* stops once the artifact exists or the report completes. The requirement is
|
||||
* unchanged: exactly one new QA test plan under the run's own state root, about
|
||||
* the reviewed dashboard change.
|
||||
*/
|
||||
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { engFirstReviewAUQ, engSetupAUQ, engStep0Boundary, hasCompletePlanReport, runPlanSkillCounting } from './helpers/claude-pty-runner';
|
||||
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
/** Working-plan path in the prompt; rewritten per run to a private temporary path. */
|
||||
const FIXTURE_PLAN_PATH = '/tmp/gstack-test-plan-eng-artifact.md';
|
||||
|
||||
const DASHBOARD_PLAN = [
|
||||
'Proceed directly to the requested engineering review; skip the optional /office-hours prerequisite.',
|
||||
`Please review this plan thoroughly. As you go, write your plan-mode plan to ${FIXTURE_PLAN_PATH} (use Edit/Write to that exact path).`,
|
||||
'',
|
||||
'# Plan: Add Dashboard',
|
||||
'',
|
||||
'## Changes',
|
||||
'1. New `dashboard.ts` with Dashboard component and fetchStats API call',
|
||||
'2. Updated `app.ts` to import and use Dashboard',
|
||||
'',
|
||||
'## Architecture',
|
||||
'- Dashboard fetches from `/api/stats` endpoint',
|
||||
'- Returns user count and revenue metrics',
|
||||
].join('\n');
|
||||
|
||||
const SOURCES = {
|
||||
'app.ts': 'import { Dashboard } from "./dashboard";\nexport function greet() { return "hello"; }\nexport function main() { return Dashboard(); }\n',
|
||||
'dashboard.ts': `export function Dashboard() {
|
||||
const data = fetchStats();
|
||||
return { users: data.users, revenue: data.revenue };
|
||||
}
|
||||
function fetchStats() {
|
||||
return fetch('/api/stats').then(r => r.json());
|
||||
}
|
||||
`,
|
||||
};
|
||||
|
||||
describeE2E('/plan-eng-review QA test-plan artifact (periodic)', () => {
|
||||
test('an interactive review writes one QA test plan about the reviewed change', async () => {
|
||||
const startedAt = Date.now();
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-e2e-plan-eng-artifact-'));
|
||||
const planPath = path.join(tmpDir, 'gstack-test-plan-eng-artifact.md');
|
||||
try {
|
||||
const obs = await runPlanSkillCounting({
|
||||
skillName: 'plan-eng-review',
|
||||
slashCommand: '/plan-eng-review',
|
||||
followUpPrompt: DASHBOARD_PLAN.replaceAll(FIXTURE_PLAN_PATH, planPath),
|
||||
fixtureFiles: SOURCES,
|
||||
permissionPlanPath: planPath,
|
||||
isLastStep0AUQ: engStep0Boundary,
|
||||
isSetupAUQ: engSetupAUQ,
|
||||
isFirstReviewAUQ: engFirstReviewAUQ,
|
||||
// The artifact is the only verdict; stop once it exists or the report is complete.
|
||||
isCollectionComplete: (_transcript, _fingerprints, testPlans) =>
|
||||
testPlans.length > 0 || hasCompletePlanReport(planPath, startedAt, Date.now()),
|
||||
reviewCountCeiling: Infinity,
|
||||
preconfiguredReviewActor: true,
|
||||
timeoutMs: CAPTURE_LONG_MS,
|
||||
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
|
||||
});
|
||||
const plans = obs.engTestPlans;
|
||||
console.log(`plan-eng-review-artifact: outcome=${obs.outcome} elapsed=${obs.elapsedMs}ms review=${obs.reviewCount} test plans=${plans.map(plan => plan.file).join(', ') || 'none'}`);
|
||||
if (!['plan_ready', 'completion_summary', 'collection_complete'].includes(obs.outcome)) {
|
||||
throw new Error(`plan-eng-review-artifact: outcome=${obs.outcome} (${obs.summary})\n--- evidence ---\n${obs.evidence}`);
|
||||
}
|
||||
expect(plans, `expected one new eng-review test plan; outcome=${obs.outcome}`).toHaveLength(1);
|
||||
expect(/dashboard|fetchStats|\/api\/stats/i.test(plans[0]!.content), 'the test plan covers the reviewed dashboard change').toBe(true);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, CAPTURE_LONG_MS + 30_000);
|
||||
});
|
||||
+1
-139
@@ -1,13 +1,12 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS, PTY_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { getHermeticDirs, isHermeticEnabled, seedHermeticGstackHome } from './helpers/hermetic-env';
|
||||
import { EvalCollector } from './helpers/eval-store';
|
||||
import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
describeIfSelected, testConcurrentIfSelected,
|
||||
copyDirSync, setupBrowseShims, logCost, recordE2E,
|
||||
copyDirSync, logCost, recordE2E,
|
||||
finalizeEvalCollector,
|
||||
} from './helpers/e2e-helpers';
|
||||
import { judgePosture } from './helpers/llm-judge';
|
||||
@@ -373,143 +372,6 @@ Focus on architecture, code quality, tests, and performance sections.`,
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Plan-Eng-Review Test-Plan Artifact E2E ---
|
||||
|
||||
describeIfSelected('Plan-Eng-Review Test-Plan Artifact E2E', ['plan-eng-review-artifact'], () => {
|
||||
let planDir: string;
|
||||
let gstackHome: string;
|
||||
let ownsGstackHome = false;
|
||||
let slug: string;
|
||||
let projectDir: string;
|
||||
|
||||
beforeAll(() => {
|
||||
planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-artifact-'));
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: planDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
|
||||
// Create base commit on main
|
||||
fs.writeFileSync(path.join(planDir, 'app.ts'), 'export function greet() { return "hello"; }\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
// Create feature branch with changes
|
||||
run('git', ['checkout', '-b', 'feature/add-dashboard']);
|
||||
fs.writeFileSync(path.join(planDir, 'dashboard.ts'), `export function Dashboard() {
|
||||
const data = fetchStats();
|
||||
return { users: data.users, revenue: data.revenue };
|
||||
}
|
||||
function fetchStats() {
|
||||
return fetch('/api/stats').then(r => r.json());
|
||||
}
|
||||
`);
|
||||
fs.writeFileSync(path.join(planDir, 'app.ts'), `import { Dashboard } from "./dashboard";
|
||||
export function greet() { return "hello"; }
|
||||
export function main() { return Dashboard(); }
|
||||
`);
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'feat: add dashboard']);
|
||||
|
||||
// Plan document
|
||||
fs.writeFileSync(path.join(planDir, 'plan.md'), `# Plan: Add Dashboard
|
||||
|
||||
## Changes
|
||||
1. New \`dashboard.ts\` with Dashboard component and fetchStats API call
|
||||
2. Updated \`app.ts\` to import and use Dashboard
|
||||
|
||||
## Architecture
|
||||
- Dashboard fetches from \`/api/stats\` endpoint
|
||||
- Returns user count and revenue metrics
|
||||
`);
|
||||
run('git', ['add', 'plan.md']);
|
||||
run('git', ['commit', '-m', 'add plan']);
|
||||
|
||||
// Copy plan-eng-review skill
|
||||
fs.mkdirSync(path.join(planDir, 'plan-eng-review'), { recursive: true });
|
||||
fs.copyFileSync(
|
||||
path.join(ROOT, 'plan-eng-review', 'SKILL.md'),
|
||||
path.join(planDir, 'plan-eng-review', 'SKILL.md'),
|
||||
);
|
||||
// Carved skills (v2 plan T9): copy sections/ so the review workflow + report template are present.
|
||||
{ const _sec = path.join(ROOT, 'plan-eng-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(planDir, 'plan-eng-review', 'sections'), { recursive: true }); }
|
||||
|
||||
// Set up remote-slug shim and browse shims (plan-eng-review uses remote-slug for artifact path)
|
||||
setupBrowseShims(planDir);
|
||||
|
||||
// The actor, this observer and cleanup share one isolated state root: the
|
||||
// hermetic run's GSTACK_HOME (or a seeded temporary one when hermetic mode
|
||||
// is off), and the slug gstack-slug resolves for this fixture. Nothing is
|
||||
// read or deleted under the operator's real home.
|
||||
if (isHermeticEnabled()) gstackHome = getHermeticDirs().gstackHome;
|
||||
else {
|
||||
gstackHome = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-plan-artifact-home-'));
|
||||
seedHermeticGstackHome(gstackHome);
|
||||
ownsGstackHome = true;
|
||||
}
|
||||
const slugOut = spawnSync(path.join(ROOT, 'bin', 'gstack-slug'), [], {
|
||||
cwd: planDir, stdio: 'pipe', timeout: 5000, env: { ...process.env, GSTACK_HOME: gstackHome },
|
||||
}).stdout.toString();
|
||||
slug = /^SLUG=(.+)$/m.exec(slugOut)?.[1]?.trim() ?? '';
|
||||
if (!slug) throw new Error(`plan-eng-review-artifact: gstack-slug resolved no slug: ${slugOut}`);
|
||||
projectDir = path.join(gstackHome, 'projects', slug);
|
||||
const realHome = path.resolve(os.homedir());
|
||||
if (projectDir === realHome || projectDir.startsWith(realHome + path.sep) && !projectDir.startsWith(path.resolve(os.tmpdir()) + path.sep)) {
|
||||
throw new Error(`plan-eng-review-artifact: state root ${projectDir} is under the real home directory`);
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
try { fs.rmSync(planDir, { recursive: true, force: true }); } catch {}
|
||||
// Remove only this fixture's project directory inside the isolated state root.
|
||||
try { fs.rmSync(ownsGstackHome ? gstackHome : projectDir, { recursive: true, force: true }); } catch {}
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('plan-eng-review-artifact', async () => {
|
||||
const testPlans = () => fs.existsSync(projectDir)
|
||||
? fs.readdirSync(projectDir).filter(f => /eng-review-test-plan-.*\.md$/.test(f)) : [];
|
||||
const beforeFiles = testPlans();
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `Read plan-eng-review/SKILL.md for the review workflow.
|
||||
Skip the preamble bash block, lake intro, telemetry, and contributor mode sections — go straight to the review.
|
||||
|
||||
Read plan.md — that's the plan to review. This is a standalone plan with source code in app.ts and dashboard.ts.
|
||||
|
||||
Proceed directly to the full review. Skip any AskUserQuestion calls — this is non-interactive.
|
||||
|
||||
In this fixture the gstack state root is GSTACK_HOME=${gstackHome} and the project slug is ${slug}.
|
||||
|
||||
Use ${planDir}/review-output.md as the selected report file. Every other output keeps the destination the skill gives it.`,
|
||||
workingDirectory: planDir,
|
||||
env: { GSTACK_HOME: gstackHome },
|
||||
maxTurns: 25,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
|
||||
timeout: CAPTURE_LONG_MS,
|
||||
testName: 'plan-eng-review-artifact',
|
||||
runId,
|
||||
model: 'claude-opus-4-7',
|
||||
});
|
||||
|
||||
logCost('/plan-eng-review artifact', result);
|
||||
// The QA test plan is a required discovery artifact of the review: exactly
|
||||
// one new file under this fixture's project directory, about this plan.
|
||||
const newFiles = testPlans().filter(f => !beforeFiles.includes(f));
|
||||
const content = newFiles.length === 1 ? fs.readFileSync(path.join(projectDir, newFiles[0]!), 'utf-8') : '';
|
||||
const aboutPlan = /dashboard|fetchStats|\/api\/stats/i.test(content);
|
||||
console.log(`Test-plan artifacts in ${projectDir}: ${newFiles.length} new${newFiles.length ? ` (${newFiles[0]}, ${content.length} chars)` : ''}`);
|
||||
recordE2E(evalCollector, '/plan-eng-review test-plan artifact', 'Plan-Eng-Review Test-Plan Artifact E2E', result, {
|
||||
passed: ['success', 'error_max_turns'].includes(result.exitReason) && newFiles.length === 1 && aboutPlan,
|
||||
});
|
||||
|
||||
expect(['success', 'error_max_turns']).toContain(result.exitReason);
|
||||
expect(newFiles, `expected one new eng-review test plan in ${projectDir}`).toHaveLength(1);
|
||||
expect(aboutPlan, 'the test plan covers the reviewed dashboard change').toBe(true);
|
||||
}, CAPTURE_LONG_MS);
|
||||
});
|
||||
|
||||
// --- Plan Review Report E2E ---
|
||||
// Verifies that plan-eng-review writes a "## GSTACK REVIEW REPORT" section
|
||||
// to the bottom of the plan file (the living review status footer).
|
||||
|
||||
Reference in new issue
Block a user