mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
test(office-hours): a fork whose outer options are the seeded shapes is the Phase 4 question
Census trials 1-2 captured complete Phase 4 forks (A) Server-side B) Client-side C) Hybrid, recommendation with because) whose prose used none of the vocabulary words. Accept two seeded shapes as outer options as Phase 4 specificity; the earlier-phase, nested, fenced and single-shape controls still fail.
This commit is contained in:
1 parent
1751985a21
commit
a487a09bf1
2 files changed
+28
-6
No files matched your search
@@ -96,3 +96,16 @@ test('Phase4 caller fixture retains fork validation under either path convention
|
||||
expect(rejected.thrown).toBeDefined(); expect(rejected.judged).toBe(0); expect(rejected.rows[0].passed).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('Phase4 caller accepts a fork whose outer options are the seeded shapes, without Phase 4 vocabulary', async () => {
|
||||
// Census 36776104571 trials 1-2: a complete Phase 4 fork naming the three seeded shapes as its options.
|
||||
const seeded = (options: string) => `Where should the cross-skill retrieval smarts live?\nThe choice locks in who owns the ranking logic.\n${options}\nRecommendation: C because we can tune the salience signal before freezing a contract.\n`;
|
||||
for (const options of ['A) Server-side — new MCP tools in gbrain\nB) Client-side — a gstack helper\nC) Hybrid — V1 client-side, V1.5 in gbrain',
|
||||
'A) **Server-side** gbrain tools\nB) **Hybrid** promotion later']) {
|
||||
const x = await runCaller(seeded(options)); expect(x.thrown).toBeUndefined(); expect(x.judged).toBe(1); expect(x.rows[0].passed).toBe(true);
|
||||
}
|
||||
for (const options of ['A) Server-side — new MCP tools\nB) Put retrieval on the client', 'A) Server-side\n 1) Client-side\n 2) Hybrid',
|
||||
'A) Use a queue\n```text\nB) Server-side\nC) Client-side\n```', 'A) Ask customers first\nB) Ask later\nServer-side and Client-side are both options.']) {
|
||||
const x = await runCaller(seeded(options)); expect(x.thrown).toBeDefined(); expect(x.judged).toBe(0); expect(x.rows[0].passed).toBe(false);
|
||||
}
|
||||
});
|
||||
@@ -45,8 +45,8 @@ const evalCollector = createEvalCollector('e2e-office-hours-phase4');
|
||||
const BECAUSE_RE = /\bbecause\b/i;
|
||||
// At least 2 numbered/lettered options (A/B or 1/2). Office-hours Phase 4 says
|
||||
// "2-3 distinct alternatives," so 2+ is the minimum bar.
|
||||
function hasTwoAlternatives(text: string): boolean {
|
||||
const options: Array<{ indent: number; label: string }> = [];
|
||||
function outerOptions(text: string): Array<{ label: string; text: string }> {
|
||||
const options: Array<{ indent: number; label: string; text: string }> = [];
|
||||
let fence: { char: string; length: number } | undefined;
|
||||
for (const line of text.split('\n')) {
|
||||
const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
|
||||
@@ -55,13 +55,16 @@ function hasTwoAlternatives(text: string): boolean {
|
||||
continue;
|
||||
}
|
||||
if (marker) { fence = { char: marker[1]![0]!, length: marker[1]!.length }; continue; }
|
||||
const option = /^( {0,3})([A-Z]|[1-9]\d*)\)\s+\S/.exec(line);
|
||||
if (option) options.push({ indent: option[1]!.length, label: option[2]! });
|
||||
const option = /^( {0,3})([A-Z]|[1-9]\d*)\)\s+(\S.*)$/.exec(line);
|
||||
if (option) options.push({ indent: option[1]!.length, label: option[2]!, text: option[3]! });
|
||||
}
|
||||
// Only the outer option list counts; numbered steps inside one option and
|
||||
// Markdown source/quoted blocks do not supply another alternative.
|
||||
const outerIndent = Math.min(...options.map(option => option.indent));
|
||||
const labels = options.filter(option => option.indent === outerIndent).map(option => option.label);
|
||||
return options.filter(option => option.indent === outerIndent);
|
||||
}
|
||||
function hasTwoAlternatives(text: string): boolean {
|
||||
const labels = outerOptions(text).map(option => option.label);
|
||||
return new Set(labels.filter(label => /^[A-Z]$/.test(label))).size >= 2 ||
|
||||
new Set(labels.filter(label => /^\d+$/.test(label))).size >= 2;
|
||||
}
|
||||
@@ -69,6 +72,12 @@ function hasTwoAlternatives(text: string): boolean {
|
||||
// question. Without this, a captured AskUserQuestion from an earlier phase
|
||||
// would false-pass.
|
||||
const PHASE4_VOCAB_RE = /approach|alternative|architectur(?:e|al)|implementation/i;
|
||||
// A question whose outer options are two of the seeded Phase 4 shapes is the
|
||||
// Phase 4 fork even when its prose names none of those words.
|
||||
function offersSeededAlternatives(text: string): boolean {
|
||||
const shapes = outerOptions(text).map(option => /^\**(Server-side|Client-side|Hybrid)\b/i.exec(option.text)?.[1]?.toLowerCase());
|
||||
return new Set(shapes.filter(Boolean)).size >= 2;
|
||||
}
|
||||
|
||||
function setupOfficeHoursDir(): string {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-office-hours-phase4-'));
|
||||
@@ -181,7 +190,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to
|
||||
expect(captured).toMatch(BECAUSE_RE);
|
||||
expect(hasTwoAlternatives(captured)).toBe(true);
|
||||
// Phase-4 specificity: prevents a stray earlier-phase AUQ from false-passing.
|
||||
expect(captured).toMatch(PHASE4_VOCAB_RE);
|
||||
expect(PHASE4_VOCAB_RE.test(captured) || offersSeededAlternatives(captured), 'captured question is not the Phase 4 fork').toBe(true);
|
||||
|
||||
// Recommendation-quality judge: same threshold as plan-format tests.
|
||||
await assertRecommendationQuality({
|
||||
|
||||
Reference in new issue
Block a user