diff --git a/.github/workflows/evals-periodic.yml b/.github/workflows/evals-periodic.yml index 2cfcfdb2e..2bb7af627 100644 --- a/.github/workflows/evals-periodic.yml +++ b/.github/workflows/evals-periodic.yml @@ -98,6 +98,7 @@ jobs: contents: read outputs: periodic_slices: ${{ steps.periodic-matrix.outputs.slices }} + periodic_codex_slices: ${{ steps.periodic-matrix.outputs.codex_slices }} periodic_timeout_minutes: ${{ steps.periodic-matrix.outputs.timeout_minutes }} gate_slices: ${{ steps.gate-matrix.outputs.slices }} gate_timeout_minutes: ${{ steps.gate-matrix.outputs.timeout_minutes }} @@ -115,11 +116,16 @@ jobs: EVALS_ALL: "1" run: EVALS_TIER=periodic bun --no-install run scripts/test-paid-shards.ts --tier periodic --emit-plan /tmp/paid-plan/manifest.json --slice-budget 540 --jobs 2 --max-parallel 25 + # Codex-only slices run on the host-run eval-codex-slices job; the + # container executor runs every other slice. - name: Derive the periodic executor matrix from the plan id: periodic-matrix run: | - echo "slices=$(jq -c '[range(1; .sliceCount + 1)]' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT" - echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/paid-plan/manifest.json)" >> "$GITHUB_OUTPUT" + { + echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)" + echo "codex_slices=$(jq -c '.plan.codexSlices' /tmp/paid-plan/manifest.json)" + echo "timeout_minutes=$(jq -e '.plan.ciTimeoutMinutes' /tmp/paid-plan/manifest.json)" + } >> "$GITHUB_OUTPUT" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -280,6 +286,133 @@ jobs: if-no-files-found: ignore retention-days: 30 + # Codex shards on the host. Codex runs every shell command inside a + # bubblewrap sandbox, which needs an unprivileged user namespace. A job + # container cannot get one: Docker's default seccomp and AppArmor profiles + # block it, and Ubuntu 24.04 hosts set + # kernel.apparmor_restrict_unprivileged_userns=1, which no container option + # can change. This job runs on the runner VM itself (ephemeral, one job), + # lifts that host restriction, and runs the same CI image with both + # profiles unconfined. Only the planner's Codex slices run here + # (scripts/lib/paid-plan.ts codexSlicesOf); slice artifacts use the same + # names as eval-slices, so the report reconciles one manifest. + eval-codex-slices: + runs-on: ubicloud-standard-8 + needs: [build-image, plan-slices] + if: ${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }} + env: + EVALS_RUN_ID: ci-${{ github.run_id }}-${{ github.run_attempt }}-eval-slices-${{ matrix.slice }} + timeout-minutes: ${{ fromJSON(needs.plan-slices.outputs.periodic_timeout_minutes) }} + permissions: + contents: read + packages: read + strategy: + fail-fast: false + max-parallel: 25 + matrix: + slice: ${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: paid-plan + path: ${{ runner.temp }}/paid-plan + + - name: Let Codex's bubblewrap sandbox create user namespaces + run: | + if [ -e /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]; then + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + fi + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # The same steps as eval-slices (bun temp, deps, build, Codex login, + # slice run) inside the CI image. The container's home is a fresh + # runner-temp directory, so logs and captures upload from there. + - name: Run periodic Codex slice ${{ matrix.slice }} + shell: bash + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + EVAL_IMAGE: ${{ needs.build-image.outputs.image-tag }} + SLICE: ${{ matrix.slice }} + EVALS_JOBS: "2" + run: | + mkdir -p "$RUNNER_TEMP/eval-home" "$RUNNER_TEMP/paid-slice-results" + docker run --rm --user "$(id -u):$(id -g)" \ + --security-opt seccomp=unconfined --security-opt apparmor=unconfined \ + -e HOME=/home/runner -e ANTHROPIC_API_KEY -e OPENAI_API_KEY -e GEMINI_API_KEY -e SLICE -e EVALS_JOBS \ + -e EVALS_RUN_ID -e EVALS_PROFILE -e EVALS_FRESH -e EVALS_CACHE_PURPOSE -e CI -e GITHUB_ACTIONS \ + -e GITHUB_RUN_ID -e GITHUB_RUN_ATTEMPT -e GITHUB_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REPOSITORY \ + -e GITHUB_EVENT_NAME -e GITHUB_BASE_REF -e GITHUB_HEAD_REF \ + -e PLAYWRIGHT_BROWSERS_PATH=/opt/playwright-browsers -e EVALS_CONCURRENCY=2 -e GSTACK_EVAL_DIR=/tmp/paid-slice-results \ + -v "$GITHUB_WORKSPACE:/workspace/gstack" -w /workspace/gstack \ + -v "$RUNNER_TEMP/eval-home:/home/runner" \ + -v "$RUNNER_TEMP/paid-plan:/tmp/paid-plan:ro" \ + -v "$RUNNER_TEMP/paid-slice-results:/tmp/paid-slice-results" \ + "$EVAL_IMAGE" bash -c ' + set -euo pipefail + mkdir -p "$HOME/.cache/bun" + export BUN_INSTALL_CACHE_DIR="$HOME/.cache/bun" BUN_TMPDIR="$HOME/.cache/bun" TMPDIR="$HOME/.cache" + if diff -q /opt/node_modules_cache/.bun.lock bun.lock >/dev/null 2>&1; then + rm -rf node_modules && cp -r /opt/node_modules_cache node_modules + else + bun install --frozen-lockfile + fi + bun run build + codex="${GSTACK_CI_CODEX_BIN_DIR:?}/codex" + export CODEX_HOME="$HOME/.gstack-ci-codex" + mkdir -p "$CODEX_HOME" + if [ -n "${OPENAI_API_KEY:-}" ]; then + printenv OPENAI_API_KEY | "$codex" login --with-api-key + "$codex" login status >/dev/null 2>&1 + else + echo "::warning::OPENAI_API_KEY is unavailable; Codex eval cases cannot authenticate and will fail." + fi + export GSTACK_CI_CODEX_HOME="$CODEX_HOME" + unset CODEX_HOME + EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice "$SLICE" + ' + + - name: Upload slice results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: paid-slice-${{ matrix.slice }}-a${{ github.run_attempt }} + path: ${{ runner.temp }}/paid-slice-results + retention-days: 90 + + - name: Upload native capture evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: native-captures-${{ env.EVALS_RUN_ID }} + include-hidden-files: true + path: | + ${{ runner.temp }}/eval-home/.gstack/projects/*/e2e-runs + ${{ runner.temp }}/eval-home/.gstack-dev/e2e-runs + if-no-files-found: ignore + retention-days: 90 + + - name: Upload shard logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: paid-logs-slice-${{ matrix.slice }}-a${{ github.run_attempt }} + include-hidden-files: true + path: ${{ runner.temp }}/eval-home/.cache/gstack-paid-shard-*.log + if-no-files-found: ignore + retention-days: 30 + # Weekly EVALS_ALL gate-tier census: PR lanes are diff-billed, so without # this the full gate census might never execute anywhere and the selector's # blind spots rot invisibly. Census health, not selector correctness — @@ -389,7 +522,7 @@ jobs: report: runs-on: ubicloud-standard-2 - needs: [plan-slices, eval-slices, gate-census] + needs: [plan-slices, eval-slices, eval-codex-slices, gate-census] # !cancelled(): the report must run (and FAIL) when an executor died — a # missing slice artifact reading as green is the class this lane kills — # but a cancelled run stops here. @@ -521,7 +654,7 @@ jobs: # issue (never a new issue per week) with the headline and failure block # of both lanes, and close it on the next green run. - name: Upsert tracking issue on failure - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || needs.gate-census.result != 'success') + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH: ${{ steps.verdict.outputs.redispatch }} @@ -536,7 +669,7 @@ jobs: if [ -n "$REDISPATCH_OF" ]; then echo; echo "This run is the one INFRA/INCOMPLETE re-dispatch of run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${REDISPATCH_OF}; both runs are reported."; fi if [ "$REDISPATCH" = "true" ]; then echo; echo "Every red verdict is machine-classified INFRA/INCOMPLETE: re-dispatching once as a new run (EVAL_POLICY.infraRedispatch). This run stays red and reported."; fi echo - echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }})" + echo "- periodic reconciliation exit: ${{ steps.reconcile.outputs.exit }} (slices job: ${{ needs.eval-slices.result }}, Codex slices job: ${{ needs.eval-codex-slices.result }})" echo "- gate census reconciliation exit: ${{ steps.gate-reconcile.outputs.exit }} (census job: ${{ needs.gate-census.result }})" echo "- pass-rate history gate exit: ${{ steps.pass-rates.outputs.exit }}" echo @@ -573,7 +706,7 @@ jobs: fi - name: Close the tracking issue on a green run - if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && needs.gate-census.result == 'success' + if: always() && steps.reconcile.outputs.exit == '0' && steps.gate-reconcile.outputs.exit == '0' && steps.pass-rates.outputs.exit == '0' && needs.eval-slices.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) && needs.gate-census.result == 'success' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} REDISPATCH_OF: ${{ inputs.redispatch_of }} @@ -588,7 +721,7 @@ jobs: fi - name: Fail the workflow when reconciliation failed - if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || needs.gate-census.result != 'success') + if: always() && (steps.reconcile.outputs.exit != '0' || steps.gate-reconcile.outputs.exit != '0' || steps.pass-rates.outputs.exit != '0' || needs.eval-slices.result != 'success' || !contains(fromJSON('["success","skipped"]'), needs.eval-codex-slices.result) || needs.gate-census.result != 'success') run: exit 1 # The one INFRA/INCOMPLETE re-dispatch (UC-E1). Its own job so the report diff --git a/scripts/lib/paid-plan.ts b/scripts/lib/paid-plan.ts index b46a9449d..7a60c53b2 100644 --- a/scripts/lib/paid-plan.ts +++ b/scripts/lib/paid-plan.ts @@ -55,7 +55,7 @@ export { PAID_TEST_GLOBS, isPaidTestFile }; export { PERIODIC_CI_EXCLUDE }; type E2EShardReuse = NonNullable>; -import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases'; +import { CASE_KEY_SEPARATOR, CASE_SHARDED_FILES, type CaseTrialPlan, caseTrialPlan, codexShardAccess, expandCaseShards, expandTrialShards, isIsolatedCase, partitionCaseExclusions, sameTrialPlan, shardCaseId, shardFile, shardTrial } from './paid-cases'; import { DEFAULT_JOBS, OVERLAY_MAX_ACTIVE_SHARDS, PAID_TIERS, type PaidCaseSelection, type PaidProfile, type PaidShardBudget, type PaidTier, ROOT, type ShardOutcome, type ShardStatus, type ShardTrialRecord, collectPaidTestFiles, computePaidCaseSelection, expectedPrCaseCount, isAllSkippedPass, isOverlayTestFile, paidShardWallUpperBoundMs, partitionShardsByDiffSelection, planPaidShards, prProfileFileSelected, resolvePaidShardBudget, resolvePaidShardTimeoutMs, sameBudget, selectPaidTestFiles, shardSlug, validatedProfile } from '../test-paid-shards'; // ─── Planner / executor / report (the CI re-platform surface) ────────────── @@ -89,6 +89,8 @@ export interface PaidSlicePlan { jobs: number; estimatedSliceMs: number[]; ciTimeoutMinutes: number; + /** Slices that run only Codex shards; CI executes them on the host-run Codex job, where bubblewrap can start. */ + codexSlices?: number[]; } export interface PaidRunManifest { @@ -187,6 +189,16 @@ export function mergePaidTestDurations(seed: Record, results: Sl return Object.fromEntries(Object.entries(merged).sort(([a], [b]) => (a < b ? -1 : 1))); } +/** True when a shard key's file sees the CI Codex install in `tier` (scripts/lib/paid-cases.ts codexShardAccess). */ +export function isCodexShard(key: string, tier: PaidTier): boolean { + return codexShardAccess([shardFile(key)], tier) !== 'none'; +} + +/** Sorted slices holding at least one planned Codex shard. */ +export function codexSlicesOf(tier: PaidTier, entries: readonly ManifestEntry[]): number[] { + return [...new Set(entries.filter(entry => entry.status === 'planned' && isCodexShard(entry.file, tier)).map(entry => entry.slice))].sort((a, b) => a - b); +} + /** Panel identity of a trial shard key (`#`), else null. */ export function trialPanelKey(key: string): string | null { return shardTrial(key) === null ? null : durationKey(key); @@ -228,11 +240,12 @@ export function sliceSupervisedWallMs(files: string[], jobs: number, overrideMs? * within the budget (best fit), else into a new slice. A file with no recorded * wall weighs the whole budget, so unknown cost gets a runner of its own. A * file longer than the budget runs alone. Overlay wrappers keep one shared - * final slice (one wrapper at a time). The CI timeout covers every slice's - * supervised worst case plus the setup allowance. + * final slice (one wrapper at a time). Codex shards never share a slice with + * other shards. The CI timeout covers every slice's supervised worst case plus + * the setup allowance. */ export function packBySliceBudget(files: string[], budgetMs: number, jobs: number, - recorded: Record, timeoutMs?: number): { + recorded: Record, timeoutMs?: number, codexShard: (file: string) => boolean = () => false): { slices: string[][]; estimates: Record; estimatedSliceMs: number[]; ciTimeoutMinutes: number; } { const estimates = Object.fromEntries(files.map(file => [file, recordedShardMs(recorded, file) ?? budgetMs])); @@ -244,6 +257,8 @@ export function packBySliceBudget(files: string[], budgetMs: number, jobs: numbe // Trials of one case never share a runner: independent machines, and // the panel's wall stays one trial long. if (sharesPanel(planned, file)) return; + // Codex shards run on the host-run Codex job; other shards keep the default container sandbox. + if (codexShard(planned[0]!) !== codexShard(file)) return; const ms = estimatedSliceMs([...planned, file], weight, jobs); if (ms <= budgetMs && ms > bestMs) { best = index; bestMs = ms; } }); @@ -339,7 +354,7 @@ export function buildRunManifest(opts: { const entries: ManifestEntry[] = []; if (budgetMode) { const plan = packBySliceBudget(runnable.map(files => files[0]!), opts.sliceBudgetMs!, opts.jobs!, - opts.durations ?? loadPaidTestDurations(rootDir, opts.tier), opts.timeoutMs); + opts.durations ?? loadPaidTestDurations(rootDir, opts.tier), opts.timeoutMs, file => isCodexShard(file, opts.tier)); plan.slices.forEach((files, index) => files.forEach(file => entries.push({ file, slice: index + 1, status: 'planned', estimatedMs: plan.estimates[file]!, ...extras(file), ...(FILE_RETRY_BUDGETS.some(budget => budget.file === shardFile(file)) ? { budget: resolvePaidShardBudget([file], opts.timeoutMs) } : {}) }))); @@ -350,7 +365,8 @@ export function buildRunManifest(opts: { version: 1, tier: opts.tier, evalsAll: opts.evalsAll, sliceCount: plan.slices.length, selectionReason: cases.reason, profile, selection: cases.selection, ...(cases.coverage ? { prCoverage: cases.coverage } : {}), - plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes }, + plan: { sliceBudgetMs: opts.sliceBudgetMs!, jobs: opts.jobs!, estimatedSliceMs: plan.estimatedSliceMs, ciTimeoutMinutes: plan.ciTimeoutMinutes, + codexSlices: codexSlicesOf(opts.tier, entries) }, entries, } satisfies PaidRunManifest)); } @@ -616,6 +632,12 @@ export function parseRunManifest(raw: string): PaidRunManifest { || parsed.entries.some(entry => entry.status === 'planned' && entry.estimatedMs === undefined)) { throw new Error('manifest slice plan malformed'); } + const codexSlices = codexSlicesOf(parsed.tier, parsed.entries); + if (JSON.stringify(plan.codexSlices ?? []) !== JSON.stringify(codexSlices)) { + throw new Error(`manifest Codex slices must be exactly the slices holding Codex shards: ${JSON.stringify(codexSlices)}`); + } + const mixed = parsed.entries.find(entry => entry.status === 'planned' && codexSlices.includes(entry.slice) && !isCodexShard(entry.file, parsed.tier)); + if (mixed) throw new Error(`Codex slice ${mixed.slice} also holds ${mixed.file}; only Codex shards run on the host-run Codex job`); } const keys = parsed.entries.map(entry => normalizeRelativePath(entry.file)); if (new Set(keys).size !== keys.length) throw new Error('Duplicate manifest entry'); diff --git a/test/codex-ci-access.test.ts b/test/codex-ci-access.test.ts index 837e3938e..a311fb90c 100644 --- a/test/codex-ci-access.test.ts +++ b/test/codex-ci-access.test.ts @@ -12,7 +12,7 @@ import { describe, expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; -import { CODEX_CI_CASES, CODEX_CI_ENV, CODEX_CI_FILES, buildRunManifest, codexShardAccess, runPaidShards, scopeCodexAccess, shardFile } from '../scripts/test-paid-shards'; +import { CODEX_CI_CASES, CODEX_CI_ENV, CODEX_CI_FILES, buildRunManifest, codexShardAccess, isCodexShard, packBySliceBudget, parseRunManifest, runPaidShards, scopeCodexAccess, shardFile } from '../scripts/test-paid-shards'; import { buildHermeticEnv } from './helpers/hermetic-env'; import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles'; @@ -135,6 +135,42 @@ describe('codex CI access scope', () => { }, 60_000); }); +describe('Codex slices for the host-run job', () => { + const budgetPlan = () => buildRunManifest({ tier: 'periodic', profile: 'full', sliceBudgetMs: 540_000, jobs: 2, evalsAll: true, env: { EVALS_ALL: '1' } }); + + test('packing never puts a Codex shard on a slice with other shards', () => { + const files = ['test/codex-e2e.test.ts', 'test/skill-e2e-outside-voice.test.ts', 'test/skill-e2e-plan.test.ts', 'test/skill-e2e-review.test.ts']; + const recorded = Object.fromEntries(files.map(file => [file, 60_000])); + const plan = packBySliceBudget(files, 540_000, 2, recorded, undefined, file => isCodexShard(file, 'periodic')); + expect(plan.slices).toEqual([['test/codex-e2e.test.ts', 'test/skill-e2e-outside-voice.test.ts'], ['test/skill-e2e-plan.test.ts', 'test/skill-e2e-review.test.ts']]); + expect(packBySliceBudget(files, 540_000, 2, recorded).slices).toHaveLength(1); + }); + + test('the live periodic plan names exactly the slices holding Codex shards, and they hold nothing else', () => { + const manifest = budgetPlan(); + const planned = manifest.entries.filter(entry => entry.status === 'planned'); + const codexSlices = manifest.plan!.codexSlices!; + expect(codexSlices.length).toBeGreaterThan(0); + expect(new Set(planned.filter(entry => isCodexShard(entry.file, 'periodic')).map(entry => shardFile(entry.file)))) + .toEqual(new Set([...CODEX_CI_FILES, ...Object.keys(CODEX_CI_CASES)])); + for (const entry of planned) expect(codexSlices.includes(entry.slice), entry.file).toBe(isCodexShard(entry.file, 'periodic')); + }); + + test('a manifest that hides a Codex slice or mixes it with another shard fails to parse', () => { + const manifest = budgetPlan(); + const codexSlice = manifest.plan!.codexSlices![0]!; + const hidden = structuredClone(manifest); + hidden.plan!.codexSlices = []; + expect(() => parseRunManifest(JSON.stringify(hidden))).toThrow(/Codex slices must be exactly/); + const omitted = structuredClone(manifest); + delete omitted.plan!.codexSlices; + expect(() => parseRunManifest(JSON.stringify(omitted))).toThrow(/Codex slices must be exactly/); + const mixed = structuredClone(manifest); + mixed.entries.find(entry => entry.status === 'planned' && !isCodexShard(entry.file, 'periodic'))!.slice = codexSlice; + expect(() => parseRunManifest(JSON.stringify(mixed))).toThrow(new RegExp(`Codex slice ${codexSlice} also holds`)); + }); +}); + describe('codex CI access in the image and workflows', () => { test('the image installs Codex off PATH at the directory it advertises', () => { const dockerfile = fs.readFileSync(path.join(ROOT, '.github/docker/Dockerfile.ci'), 'utf8'); @@ -158,9 +194,17 @@ describe('codex CI access in the image and workflows', () => { } } } - expect(steps.length).toBeGreaterThanOrEqual(4); + expect(steps.length).toBeGreaterThanOrEqual(5); for (const run of steps) { - expect(run).toContain('echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"'); + // The host-run Codex job logs in inside its one `docker run` script, so the home stays a shell export there. + if (run.includes('docker run ')) { + const exported = run.indexOf('export GSTACK_CI_CODEX_HOME="$CODEX_HOME"'); + expect(exported).toBeGreaterThan(run.indexOf('login --with-api-key')); + expect(run.indexOf('unset CODEX_HOME')).toBeGreaterThan(exported); + expect(run.indexOf('unset CODEX_HOME')).toBeLessThan(run.indexOf('scripts/test-paid-shards.ts')); + } else { + expect(run).toContain('echo "GSTACK_CI_CODEX_HOME=$CODEX_HOME" >> "$GITHUB_ENV"'); + } expect(run).not.toMatch(/echo "CODEX_HOME=/); expect(run).not.toContain('$HOME/.codex"'); expect(run).toContain('"${GSTACK_CI_CODEX_BIN_DIR:?}/codex"'); diff --git a/test/evals-workflow-wiring.test.ts b/test/evals-workflow-wiring.test.ts index 485f10080..4268c160e 100644 --- a/test/evals-workflow-wiring.test.ts +++ b/test/evals-workflow-wiring.test.ts @@ -22,7 +22,7 @@ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; -import { buildRunManifest, parseCliOptions, sliceExecutionOrder, sliceSupervisedWallMs, CI_SETUP_ALLOWANCE_MINUTES } from '../scripts/test-paid-shards'; +import { buildRunManifest, isCodexShard, parseCliOptions, sliceExecutionOrder, sliceSupervisedWallMs, CI_SETUP_ALLOWANCE_MINUTES } from '../scripts/test-paid-shards'; const ROOT = path.join(import.meta.dir, '..'); const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf-8'); @@ -38,7 +38,7 @@ function plannerSites(source: string): Array<{ manifest: string; budgetSeconds: .map((m) => ({ manifest: m[1]!, budgetSeconds: Number(m[2]), jobs: Number(m[3]) })); } -type Step = { id?: string; name?: string; run?: string; env?: Record; with?: Record }; +type Step = { id?: string; name?: string; run?: string; uses?: string; env?: Record; with?: Record }; type Job = { needs?: string[]; env?: Record; outputs?: Record; 'timeout-minutes': string | number; strategy?: { 'max-parallel': number; matrix: { slice: string } }; steps: Step[] }; @@ -57,7 +57,7 @@ function expectPlannedExecutor(source: string, executorName: string, prefix: str const [stepId] = /^\$\{\{ steps\.([\w-]+)\.outputs\.slices \}\}$/.exec(planner.outputs![`${prefix}slices`]!)!.slice(1); expect(planner.outputs![`${prefix}timeout_minutes`]).toBe(`\${{ steps.${stepId}.outputs.timeout_minutes }}`); const matrixStep = planner.steps.find(step => step.id === stepId)!; - const manifest = /jq -c '\[range\(1; \.sliceCount \+ 1\)\]' (\S+)\)/.exec(matrixStep.run!)![1]!; + const manifest = /jq -c '\[range\(1; \.sliceCount \+ 1\)\](?: - \.plan\.codexSlices)?' (\S+)\)/.exec(matrixStep.run!)![1]!; expect(matrixStep.run).toContain(`jq -e '.plan.ciTimeoutMinutes' ${manifest})`); const emit = planner.steps.filter(step => step.run?.includes(`--emit-plan ${manifest} `)); expect(emit).toHaveLength(1); @@ -201,6 +201,73 @@ describe('evals-periodic.yml sliced-lane wiring', () => { }); }); +describe('evals-periodic.yml host-run Codex slices', () => { + type HostJob = Job & { if?: string; container?: unknown; 'runs-on': string; permissions?: Record; + steps: Array }; + const jobs = (Bun.YAML.parse(periodicYml) as { jobs: Record }).jobs; + const codex = jobs['eval-codex-slices']!; + const container = jobs['eval-slices']!; + const planner = jobs['plan-slices']!; + const uploadNames = (job: HostJob) => job.steps.filter(step => step.uses?.startsWith('actions/upload-artifact@')).map(step => step.with!.name); + + test('the planner gives Codex slices to the host job and every other slice to the container job', () => { + const matrixStep = planner.steps.find(step => step.id === 'periodic-matrix')!; + expect(matrixStep.run).toContain(`echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)"`); + expect(matrixStep.run).toContain(`echo "codex_slices=$(jq -c '.plan.codexSlices' /tmp/paid-plan/manifest.json)"`); + expect(matrixStep.run).toContain('} >> "$GITHUB_OUTPUT"'); + expect(planner.outputs!.periodic_codex_slices).toBe('${{ steps.periodic-matrix.outputs.codex_slices }}'); + expect(codex.needs).toEqual(['build-image', 'plan-slices']); + expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_codex_slices != '[]' }}"); + expect(codex.strategy!.matrix.slice).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}'); + expect(codex['timeout-minutes']).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_timeout_minutes) }}'); + const manifest = buildRunManifest({ tier: 'periodic', profile: 'full', sliceBudgetMs: 540_000, jobs: 2, evalsAll: true, env: { EVALS_ALL: '1' }, rootDir: ROOT }); + const codexSlices = manifest.plan!.codexSlices!; + expect(codexSlices.length).toBeGreaterThan(0); + expect(codex.strategy!['max-parallel']).toBeGreaterThanOrEqual(codexSlices.length); + for (const entry of manifest.entries.filter(entry => entry.status === 'planned')) { + expect(codexSlices.includes(entry.slice), entry.file).toBe(isCodexShard(entry.file, 'periodic')); + } + // Gate plans (evals.yml and the weekly gate census) never give a shard Codex, so they have no Codex slices. + expect(buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 540_000, jobs: 2, evalsAll: true, env: { EVALS_ALL: '1' }, rootDir: ROOT }) + .plan!.codexSlices).toEqual([]); + }); + + test('runs on the runner VM, lifts the user-namespace restriction, then runs the CI image with seccomp and AppArmor unconfined', () => { + expect(codex.container).toBeUndefined(); + expect(codex['runs-on']).toBe(container['runs-on']); + expect(codex.permissions).toEqual({ contents: 'read', packages: 'read' }); + const sysctl = codex.steps.findIndex(step => step.run?.includes('sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0')); + const execute = codex.steps.findIndex(step => step.run?.includes('docker run ')); + expect(sysctl).toBeGreaterThan(-1); + expect(execute).toBeGreaterThan(sysctl); + const step = codex.steps[execute]!; + expect(step.run).toContain('--security-opt seccomp=unconfined --security-opt apparmor=unconfined'); + expect(step.run).toContain('--user "$(id -u):$(id -g)"'); + expect(step.run).not.toMatch(/--privileged|--cap-add|--pid[ =]host|--network[ =]host|docker\.sock/); + expect(step.env!.EVAL_IMAGE).toBe('${{ needs.build-image.outputs.image-tag }}'); + expect(step.run).toContain('"$EVAL_IMAGE" bash -c'); + expect(step.env!.SLICE).toBe('${{ matrix.slice }}'); + expect(step.run).toContain('EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice "$SLICE"'); + expect(step.run).toContain('-v "$RUNNER_TEMP/paid-plan:/tmp/paid-plan:ro"'); + expect(codex.steps.some(other => other.uses?.startsWith('actions/download-artifact@') && other.with?.name === 'paid-plan' + && other.with?.path === '${{ runner.temp }}/paid-plan')).toBe(true); + const containerRun = container.steps.find(other => other.run?.includes('--plan '))!; + expect(step.env!.EVALS_JOBS).toBe(containerRun.env!.EVALS_JOBS); + for (const name of ['EVALS_CONCURRENCY', 'GSTACK_EVAL_DIR', 'PLAYWRIGHT_BROWSERS_PATH']) { + expect(step.run, name).toContain(`-e ${name}=${containerRun.env![name]}`); + } + }); + + test('uploads under the container executor\'s artifact names, and the report waits for it and fails on it', () => { + expect(uploadNames(codex)).toEqual(uploadNames(container)); + const report = jobs.report!; + expect(report.needs).toContain('eval-codex-slices'); + const guards = (report.steps as Array).filter(step => step.if?.includes('needs.eval-slices.result')); + expect(guards.length).toBeGreaterThanOrEqual(3); + for (const guard of guards) expect(guard.if, guard.name).toContain('needs.eval-codex-slices.result'); + }); +}); + describe('evals-marathon.yml non-blocking lane', () => { const workflow = Bun.YAML.parse(marathonYml) as { on: Record; env: Record; jobs: Record };