mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-07 20:07:21 +02:00
test(office-hours): a fork whose outer options are the seeded shapes is the Phase 4 question
Census trials 1-2 captured complete Phase 4 forks (A) Server-side B) Client-side C) Hybrid, recommendation with because) whose prose used none of the vocabulary words. Accept two seeded shapes as outer options as Phase 4 specificity; the earlier-phase, nested, fenced and single-shape controls still fail.
This commit is contained in:
1 parent
1751985a21
commit
a487a09bf1
2 files changed
+28
-6
No files matched your search
@@ -96,3 +96,16 @@ test('Phase4 caller fixture retains fork validation under either path convention
|
|||||||
expect(rejected.thrown).toBeDefined(); expect(rejected.judged).toBe(0); expect(rejected.rows[0].passed).toBe(false);
|
expect(rejected.thrown).toBeDefined(); expect(rejected.judged).toBe(0); expect(rejected.rows[0].passed).toBe(false);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('Phase4 caller accepts a fork whose outer options are the seeded shapes, without Phase 4 vocabulary', async () => {
|
||||||
|
// Census 36776104571 trials 1-2: a complete Phase 4 fork naming the three seeded shapes as its options.
|
||||||
|
const seeded = (options: string) => `Where should the cross-skill retrieval smarts live?\nThe choice locks in who owns the ranking logic.\n${options}\nRecommendation: C because we can tune the salience signal before freezing a contract.\n`;
|
||||||
|
for (const options of ['A) Server-side — new MCP tools in gbrain\nB) Client-side — a gstack helper\nC) Hybrid — V1 client-side, V1.5 in gbrain',
|
||||||
|
'A) **Server-side** gbrain tools\nB) **Hybrid** promotion later']) {
|
||||||
|
const x = await runCaller(seeded(options)); expect(x.thrown).toBeUndefined(); expect(x.judged).toBe(1); expect(x.rows[0].passed).toBe(true);
|
||||||
|
}
|
||||||
|
for (const options of ['A) Server-side — new MCP tools\nB) Put retrieval on the client', 'A) Server-side\n 1) Client-side\n 2) Hybrid',
|
||||||
|
'A) Use a queue\n```text\nB) Server-side\nC) Client-side\n```', 'A) Ask customers first\nB) Ask later\nServer-side and Client-side are both options.']) {
|
||||||
|
const x = await runCaller(seeded(options)); expect(x.thrown).toBeDefined(); expect(x.judged).toBe(0); expect(x.rows[0].passed).toBe(false);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -45,8 +45,8 @@ const evalCollector = createEvalCollector('e2e-office-hours-phase4');
|
|||||||
const BECAUSE_RE = /\bbecause\b/i;
|
const BECAUSE_RE = /\bbecause\b/i;
|
||||||
// At least 2 numbered/lettered options (A/B or 1/2). Office-hours Phase 4 says
|
// At least 2 numbered/lettered options (A/B or 1/2). Office-hours Phase 4 says
|
||||||
// "2-3 distinct alternatives," so 2+ is the minimum bar.
|
// "2-3 distinct alternatives," so 2+ is the minimum bar.
|
||||||
function hasTwoAlternatives(text: string): boolean {
|
function outerOptions(text: string): Array<{ label: string; text: string }> {
|
||||||
const options: Array<{ indent: number; label: string }> = [];
|
const options: Array<{ indent: number; label: string; text: string }> = [];
|
||||||
let fence: { char: string; length: number } | undefined;
|
let fence: { char: string; length: number } | undefined;
|
||||||
for (const line of text.split('\n')) {
|
for (const line of text.split('\n')) {
|
||||||
const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
|
const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(line);
|
||||||
@@ -55,13 +55,16 @@ function hasTwoAlternatives(text: string): boolean {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (marker) { fence = { char: marker[1]![0]!, length: marker[1]!.length }; continue; }
|
if (marker) { fence = { char: marker[1]![0]!, length: marker[1]!.length }; continue; }
|
||||||
const option = /^( {0,3})([A-Z]|[1-9]\d*)\)\s+\S/.exec(line);
|
const option = /^( {0,3})([A-Z]|[1-9]\d*)\)\s+(\S.*)$/.exec(line);
|
||||||
if (option) options.push({ indent: option[1]!.length, label: option[2]! });
|
if (option) options.push({ indent: option[1]!.length, label: option[2]!, text: option[3]! });
|
||||||
}
|
}
|
||||||
// Only the outer option list counts; numbered steps inside one option and
|
// Only the outer option list counts; numbered steps inside one option and
|
||||||
// Markdown source/quoted blocks do not supply another alternative.
|
// Markdown source/quoted blocks do not supply another alternative.
|
||||||
const outerIndent = Math.min(...options.map(option => option.indent));
|
const outerIndent = Math.min(...options.map(option => option.indent));
|
||||||
const labels = options.filter(option => option.indent === outerIndent).map(option => option.label);
|
return options.filter(option => option.indent === outerIndent);
|
||||||
|
}
|
||||||
|
function hasTwoAlternatives(text: string): boolean {
|
||||||
|
const labels = outerOptions(text).map(option => option.label);
|
||||||
return new Set(labels.filter(label => /^[A-Z]$/.test(label))).size >= 2 ||
|
return new Set(labels.filter(label => /^[A-Z]$/.test(label))).size >= 2 ||
|
||||||
new Set(labels.filter(label => /^\d+$/.test(label))).size >= 2;
|
new Set(labels.filter(label => /^\d+$/.test(label))).size >= 2;
|
||||||
}
|
}
|
||||||
@@ -69,6 +72,12 @@ function hasTwoAlternatives(text: string): boolean {
|
|||||||
// question. Without this, a captured AskUserQuestion from an earlier phase
|
// question. Without this, a captured AskUserQuestion from an earlier phase
|
||||||
// would false-pass.
|
// would false-pass.
|
||||||
const PHASE4_VOCAB_RE = /approach|alternative|architectur(?:e|al)|implementation/i;
|
const PHASE4_VOCAB_RE = /approach|alternative|architectur(?:e|al)|implementation/i;
|
||||||
|
// A question whose outer options are two of the seeded Phase 4 shapes is the
|
||||||
|
// Phase 4 fork even when its prose names none of those words.
|
||||||
|
function offersSeededAlternatives(text: string): boolean {
|
||||||
|
const shapes = outerOptions(text).map(option => /^\**(Server-side|Client-side|Hybrid)\b/i.exec(option.text)?.[1]?.toLowerCase());
|
||||||
|
return new Set(shapes.filter(Boolean)).size >= 2;
|
||||||
|
}
|
||||||
|
|
||||||
function setupOfficeHoursDir(): string {
|
function setupOfficeHoursDir(): string {
|
||||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-office-hours-phase4-'));
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-office-hours-phase4-'));
|
||||||
@@ -181,7 +190,7 @@ After writing the file with that ONE Phase 4 question, stop. Do not continue to
|
|||||||
expect(captured).toMatch(BECAUSE_RE);
|
expect(captured).toMatch(BECAUSE_RE);
|
||||||
expect(hasTwoAlternatives(captured)).toBe(true);
|
expect(hasTwoAlternatives(captured)).toBe(true);
|
||||||
// Phase-4 specificity: prevents a stray earlier-phase AUQ from false-passing.
|
// Phase-4 specificity: prevents a stray earlier-phase AUQ from false-passing.
|
||||||
expect(captured).toMatch(PHASE4_VOCAB_RE);
|
expect(PHASE4_VOCAB_RE.test(captured) || offersSeededAlternatives(captured), 'captured question is not the Phase 4 fork').toBe(true);
|
||||||
|
|
||||||
// Recommendation-quality judge: same threshold as plan-format tests.
|
// Recommendation-quality judge: same threshold as plan-format tests.
|
||||||
await assertRecommendationQuality({
|
await assertRecommendationQuality({
|
||||||
|
|||||||
Reference in new issue
Block a user