diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 2bb7af627..0a1f7378e 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -399,7 +399,9 @@ jobs: include-hidden-files: true path: | ${{ runner.temp }}/eval-home/.gstack/projects/*/e2e-runs + ${{ runner.temp }}/eval-home/.gstack/projects/*/evals/qa-callers ${{ runner.temp }}/eval-home/.gstack-dev/e2e-runs + ${{ runner.temp }}/eval-home/.gstack-dev/evals/qa-callers if-no-files-found: ignore retention-days: 90 diff --git a/scripts/typecheck-test-baseline.json b/scripts/typecheck-test-baseline.json index 40c48da64..afe50d00d 100644 --- a/scripts/typecheck-test-baseline.json +++ b/scripts/typecheck-test-baseline.json @@ -127,7 +127,6 @@ "test/ceo-split-collection.test.ts\tTS2352\tConversion of type '({ sessionId: string; toolUseId: string; questions: { question: string; header: string; multiSelect: boolean; options: { label: string; description: string; }[]; }[]; answered: boolean; failed: boolean; answers: { ...; }; unansweredQuestionIndices: never[]; answeredAt: string; } | ... 4 more ... | { ...; })[]' to type 'NativePlanQuestionCall[]' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first. Type '{ sessionId: string; toolUseId: string; questions: { question: string; header: string; multiSelect: boolean; options: { label: string; description: string; }[]; }[]; answered: boolean; failed: boolean; answers: { ...; }; unansweredQuestionIndices: never[]; answeredAt: string; } | ... 4 more ... | { ...; }' is not comparable to type 'NativePlanQuestionCall'. Type '{ sessionId: string; toolUseId: string; questions: { question: string; header: string; multiSelect: boolean; options: { label: string; description: string; }[]; }[]; answered: boolean; failed: boolean; answers: { \"D1 \\u2014 Which review mode should govern this scope decision?\\nProject/branch/task: gstack-plan-count-...' is not comparable to type 'NativePlanQuestionCall'. Types of property 'answers' are incompatible. Type '{ \"D1 \\u2014 Which review mode should govern this scope decision?\\nProject/branch/task: gstack-plan-count-sIEkYl on main, deciding which of 5 chat integrations ship this quarter.\\nELI10: Review mode sets my posture for the rest of the session. The plan's own goal is to shrink 5 candidates to 2-3, so the natural fit ...' is not comparable to type 'Record'. Property '\"D1 — Which review mode should govern this scope decision?\\nProject/branch/task: gstack-plan-count-sIEkYl on main, deciding which of 5 chat integrations ship this quarter.\\nELI10: Review mode sets my posture for the rest of the session. The plan's own goal is to shrink 5 candidates to 2-3, so the natural fit is a mode built around deciding what NOT to do. Expansion modes would instead have me pitch extra ideas on top of the five, which is the opposite of the constraint you gave.\\nStakes if we pick wrong: an expansion mode adds noise to a decision that is about subtraction; a hold mode skips the cut/defer analysis you asked for.\\nRecommendation: SCOPE REDUCTION because the plan's stated goal is a bandwidth-capped cut from 5 to 2-3, and all 5 built is an estimated 20-30 files (>15 threshold).\\nNote: options differ in kind, not coverage — no completeness score.\"' is incompatible with index signature. Type 'undefined' is not comparable to type 'string'.": 1, "test/ceo-split-question-policy.test.ts\tTS2345\tArgument of type 'NativePlanQuestion' is not assignable to parameter of type 'NativeQuestion'. Types of property 'multiSelect' are incompatible. Type 'boolean | undefined' is not assignable to type 'boolean'. Type 'undefined' is not assignable to type 'boolean'.": 4, "test/ceo-split-question-policy.test.ts\tTS2352\tConversion of type '({ sessionId: string; toolUseId: string; questions: { question: string; header: string; multiSelect: boolean; options: { label: string; description: string; }[]; }[]; answered: boolean; failed: boolean; answers: { ...; }; unansweredQuestionIndices: never[]; answeredAt: string; } | ... 11 more ... | { ...; })[]' to type 'NativePlanQuestionCall[]' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first. Type '{ sessionId: string; toolUseId: string; questions: { question: string; header: string; multiSelect: boolean; options: { label: string; description: string; }[]; }[]; answered: boolean; failed: boolean; answers: { ...; }; unansweredQuestionIndices: never[]; answeredAt: string; } | ... 11 more ... | { ...; }' is not comparable to type 'NativePlanQuestionCall'. Type '{ sessionId: string; toolUseId: string; questions: { question: string; header: string; multiSelect: boolean; options: { label: string; description: string; }[]; }[]; answered: boolean; failed: boolean; answers: { \"D1.1 \\u2014 E1: Include the Slack DM bot for incident alerts this quarter?\\nProject/branch/task: gstack...' is not comparable to type 'NativePlanQuestionCall'. Types of property 'answers' are incompatible. Type '{ \"D1.1 \\u2014 E1: Include the Slack DM bot for incident alerts this quarter?\\nProject/branch/task: gstack-plan-count-rFjNLS @ main, choosing 2-3 of 5 chat integrations for the quarter.\\nELI10: Slack is where 40% of your customers asked to get incident alerts, and you already have a working Slack login flow to build...' is not comparable to type 'Record'. Property '\"D1.1 — E1: Include the Slack DM bot for incident alerts this quarter?\\nProject/branch/task: gstack-plan-count-rFjNLS @ main, choosing 2-3 of 5 chat integrations for the quarter.\\nELI10: Slack is where 40% of your customers asked to get incident alerts, and you already have a working Slack login flow to build on, so this is the cheapest way to make the most people happy. It takes one of your 2-3 slots (this would be slot 1 of 3). Saying no here means the single biggest customer request waits another quarter.\\nStakes if we pick wrong: Defer or cut and the top Q2 survey request ships late while a Slack-native competitor becomes the default; include and you spend ~2 weeks on the safest bet on the board.\\nRecommendation: A) Include because it is the highest-demand candidate at the second-lowest cost with the only stated code reuse.\\nNote: options differ in kind, not coverage — no completeness score.\\nNet: 40% of demand for ~2 weeks with reusable auth is the strongest ratio on the list; the only reason to say no is if you want all three slots for revenue bets.\"' is incompatible with index signature. Type 'undefined' is not comparable to type 'string'.": 1, - "test/ci-paid-coordination.test.ts\tTS2769\tNo overload matches this call. The last overload gave the following error. Type 'Step | undefined' is not assignable to type 'Step'. Type 'undefined' is not assignable to type 'Step'.": 1, "test/claude-code-runner.test.ts\tTS2741\tProperty '__promisify__' is missing in type '(callback: any, delay: any, ...args: any[]) => Timeout' but required in type 'typeof setTimeout'.": 1, "test/claude-code-runner.test.ts\tTS7006\tParameter 'callback' implicitly has an 'any' type.": 1, "test/claude-code-runner.test.ts\tTS7006\tParameter 'delay' implicitly has an 'any' type.": 1, diff --git a/test/ci-native-evidence.test.ts b/test/ci-native-evidence.test.ts index 3d1b2ed3a..318d5a9f3 100644 --- a/test/ci-native-evidence.test.ts +++ b/test/ci-native-evidence.test.ts @@ -14,6 +14,26 @@ const executors = workflows.flatMap(({ name, value }) => Object.entries(value.jo .filter((step: any) => step.run?.includes('scripts/test-paid-shards.ts') && step.run.includes(' --slice ')) .map((step: any) => ({ name, jobName, job, step })))); +// A host-run executor (evals-periodic eval-codex-slices) starts the CI image with +// `docker run`; the literal `-e NAME=VALUE` flags are the slice's environment. +function dockerEnv(run: string): Record { + return Object.fromEntries([...run.matchAll(/(?:^|\s)-e ([A-Z_][A-Z0-9_]*)=(\S+)/g)].map(match => [match[1]!, match[2]!])); +} +const executorEnv = (job: any, step: any): Record => ({ ...job.env, ...step.env, ...dockerEnv(step.run) }); +// Upload paths on a host-run executor name the host side of its `-v` mounts; +// map them back to the container paths (and the mounted HOME back to `~/`). +function containerPath(step: any, uploadPath: string): string { + const mounts = [...step.run.matchAll(/-v "\$RUNNER_TEMP\/([^:"]+):([^:"]+)(?::ro)?"/g)].map(match => [match[1]!, match[2]!]); + for (const [host, inside] of mounts) { + const prefix = '${{ runner.temp }}/' + host; + if (uploadPath === prefix || uploadPath.startsWith(prefix + '/')) { + const mapped = inside + uploadPath.slice(prefix.length); + return inside === dockerEnv(step.run).HOME ? mapped.replace(inside, '~') : mapped; + } + } + return uploadPath; +} + function render(template: string, fields: Record): string { return template.replace(/\$\{\{\s*([^}]+?)\s*\}\}/g, (_, key) => { if (!(key in fields)) throw new Error(`Unbound CI expression: ${key}`); @@ -23,17 +43,19 @@ function render(template: string, fields: Record): string { test('every direct CI paid executor binds a safe unique run/attempt/job/slice identity', () => { expect(executors.map(({ name, jobName }) => `${name}:${jobName}`)).toEqual([ - 'evals.yml:eval-slices', 'evals-periodic.yml:eval-slices', 'evals-periodic.yml:gate-census', 'evals-marathon.yml:eval-slices', + 'evals.yml:eval-slices', 'evals-periodic.yml:eval-slices', 'evals-periodic.yml:eval-codex-slices', 'evals-periodic.yml:gate-census', 'evals-marathon.yml:eval-slices', ]); const ids = new Set(); for (const [workflowIndex, { job, step }] of executors.entries()) { - expect(job.container.options).toBe('--user runner'); - const env = { ...job.env, ...step.env }; + // Container jobs run as `runner`; the host-run Codex job passes the runner's own uid to docker. + if (job.container) expect(job.container.options).toBe('--user runner'); + else expect(step.run).toContain('--user "$(id -u):$(id -g)"'); + const env = executorEnv(job, step); expect(env.EVALS_RUN_ID).toBeString(); for (const run of ['36302678692', '36302678693']) { for (const attempt of ['1', '2']) { // The planner sizes the matrix; cover more slices than any live plan. - expect(job.strategy.matrix.slice).toMatch(/^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)?slices\) \}\}$/); + expect(job.strategy.matrix.slice).toMatch(/^\$\{\{ fromJSON\(needs\.plan-slices\.outputs\.(?:[a-z]+_)*slices\) \}\}$/); for (let slice = 1; slice <= 64; slice++) { const id = render(env.EVALS_RUN_ID, { 'github.run_id': `${run}${workflowIndex}`, @@ -55,7 +77,7 @@ for (const { name, jobName, job, step } of executors) { const home = path.join(root, 'home'); const bin = path.join(root, 'bin'); fs.mkdirSync(home); fs.mkdirSync(bin); fs.mkdirSync(path.join(root, 'test')); - const configured = { ...job.env, ...step.env }; + const configured = executorEnv(job, step); const runId = render(configured.EVALS_RUN_ID, { 'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', }); @@ -117,7 +139,7 @@ test('native launch plumbing without a model', async () => { expect(record.capture.runId).toBe(runId); expect(snapshot.artifactDir.startsWith(shardDir + path.sep)).toBe(true); } - const upload = job.steps.find((candidate: any) => candidate.with?.path === configured.GSTACK_EVAL_DIR); + const upload = job.steps.find((candidate: any) => candidate.with?.path && containerPath(step, candidate.with.path) === configured.GSTACK_EVAL_DIR); expect(upload.if).toBe('always()'); const captures = job.steps.find((candidate: any) => candidate.name === 'Upload native capture evidence'); expect(captures.if).toBe('always()'); @@ -126,7 +148,7 @@ test('native launch plumbing without a model', async () => { const artifactName = render(captures.with.name, { 'env.EVALS_RUN_ID': runId }); expect(artifactName).toBe(`native-captures-${runId}`); expect(artifactName).not.toMatch(/^(paid-slice|gate-census)-[0-9]/); - const patterns = captures.with.path.trim().split('\n'); + const patterns = captures.with.path.trim().split('\n').map((pattern: string) => containerPath(step, pattern)); expect(patterns).toEqual(['~/.gstack/projects/*/e2e-runs', '~/.gstack/projects/*/evals/qa-callers', '~/.gstack-dev/e2e-runs', '~/.gstack-dev/evals/qa-callers']); const uploaded = patterns.flatMap((pattern: string) => [...new Bun.Glob(`${pattern.replace(/^~\//, '')}/**/*`) diff --git a/test/ci-paid-coordination.test.ts b/test/ci-paid-coordination.test.ts index 42bea3025..1a12e1235 100644 --- a/test/ci-paid-coordination.test.ts +++ b/test/ci-paid-coordination.test.ts @@ -129,15 +129,24 @@ describe('paid CI coordination stays off the eval image', () => { ]); const hiddenUploads = Object.values(jobs).flatMap(job => job.steps).filter(step => step.with?.['include-hidden-files']); const captures = hiddenUploads.filter(step => step.with?.name === 'native-captures-${{ env.EVALS_RUN_ID }}'); - expect(captures).toHaveLength(name === 'evals.yml' ? 1 : 2); + // evals-periodic's host-run Codex job mounts $RUNNER_TEMP/eval-home as the container's HOME. + const hostHome = (pattern: string) => pattern.replace('${{ runner.temp }}/eval-home/', '~/'); + expect(captures).toHaveLength(name === 'evals.yml' ? 1 : name === 'evals-periodic.yml' ? 3 : 2); for (const capture of captures) { expect(capture.if).toBe('always()'); - expect(String(capture.with?.path).trim().split('\n')).toEqual([ + expect(String(capture.with?.path).trim().split('\n').map(hostHome)).toEqual([ '~/.gstack/projects/*/e2e-runs', '~/.gstack/projects/*/evals/qa-callers', '~/.gstack-dev/e2e-runs', '~/.gstack-dev/evals/qa-callers', ]); } - expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs]); + const codexLogs = name === 'evals-periodic.yml' + ? jobs['eval-codex-slices'].steps.find(step => step.with?.name === 'paid-logs-slice-${{ matrix.slice }}-a${{ github.run_attempt }}') : undefined; + if (codexLogs) { + expect(codexLogs.if).toBe('always()'); + // The container sets TMPDIR under HOME, so shard logs land in the mounted HOME's cache. + expect(String(codexLogs.with?.path).trim().split('\n').map(hostHome)).toEqual(['~/.cache/gstack-paid-shard-*.log']); + } + expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual(codexLogs ? [logs!, codexLogs] : [logs!]); }); }