test(ship-docsync): shard the doc-sync lifecycle by case and drop the duplicate dispatch-only case

ship-docsync ran the same fixture and prompt as ship-docsync-completion and
asserted a subset of it. The file now runs one case per process, so its lane
wall is its longest case instead of half the sum of thirteen.
This commit is contained in:
garrytan committed 2026-09-29 16:47:24 +00:00
1 parent bfd1783fbf
commit 45cdfab2a0
10 files changed
+75 -71

No files matched your search

+12
View File
@@ -71,6 +71,18 @@
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-revalidation": 427000,
"test/skill-e2e-shared-libs.test.ts#shared-libs-unsupported-git": 168000,
"test/skill-e2e-ship-docsync.test.ts": 129000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-completion": 491000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-current": 364000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-failure": 244000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-late-result": 169000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-launch-failure": 109000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-asset": 173000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-marker": 131000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-recovery": 245000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-after": 285000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-before": 210000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-store": 366000,
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-timeout-unsettled": 151000,
"test/skill-e2e-ship-hook-consent.test.ts": 63000,
"test/skill-e2e-ship-hook-refresh.test.ts": 70000,
"test/skill-e2e-skillify.test.ts": 188000,
+1
View File
@@ -136,6 +136,7 @@ export const CASE_SHARDED_FILES: readonly string[] = [
'test/skill-e2e-review-army.test.ts',
'test/skill-e2e-shared-libs-paths.test.ts',
'test/skill-e2e-shared-libs.test.ts',
'test/skill-e2e-ship-docsync.test.ts',
];
/** Bun test names that differ from their E2E id. */
+2 -2
View File
@@ -16,7 +16,7 @@ export const PR_PROFILE_CASE_IDS = [
'auq-format-gate', 'plan-design-review-no-ui-scope', 'office-hours-spec-review',
'tpa-present', 'tpa-absent-linux',
'ship-local-workflow', 'ship-coverage-audit', 'docsync-spawned',
'ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure',
'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before',
'ship-docsync-stale-after', 'ship-docsync-recovery',
@@ -47,7 +47,7 @@ export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
'test/skill-e2e-ship-hook-refresh.test.ts': ['ship-managed-hook-refresh'],
'test/skill-e2e-ship-hook-consent.test.ts': ['ship-unmanaged-hook-consent', 'ship-local-hook-preservation'],
'test/skill-e2e-docsync-spawned.test.ts': ['docsync-spawned'],
'test/skill-e2e-ship-docsync.test.ts': ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store', 'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'],
'test/skill-e2e-ship-docsync.test.ts': ['ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store', 'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'],
'test/skill-e2e-deploy.test.ts': ['setup-deploy-workflow'],
'test/skill-e2e-session-intelligence.test.ts': ['context-restore-loads-latest'],
'test/skill-e2e-plan-tune.test.ts': ['plan-tune-inspect'],
+2 -2
View File
@@ -45,9 +45,9 @@ test('the existing quality and behavior phases retain their complete separate sh
expect(behavior.evalsAll).toBe(true);
expect(qualityFiles).toHaveLength(1);
// 44 files (first-task-scaffold registers no gate case, so the gate lane
// skips it); the five case-sharded files contribute one shard per gate case.
// skips it); the six case-sharded files contribute one shard per gate case.
expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(44);
expect(behaviorFiles).toHaveLength(62);
expect(behaviorFiles).toHaveLength(73);
expect(behaviorFiles).toEqual(expect.arrayContaining([
'test/skill-e2e-qa-callers.test.ts',
'test/skill-e2e-qa-functional-fix.test.ts',
+3 -3
View File
@@ -84,8 +84,8 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
},
}));
await import(path.join(root, 'test/skill-e2e-ship-docsync.test.ts'));
expect(callbacks.size).toBe(13);
for (const name of ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store']) {
expect(callbacks.size).toBe(12);
for (const name of ['ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store']) {
test(name + ' constructs its real native request without launching it', async () => {
const before = launched;
await expect(callbacks.get(name)()).rejects.toBe(stopped);
@@ -101,7 +101,7 @@ for (const name of ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-cur
});
const output = result.stdout.toString() + result.stderr.toString();
expect(result.exitCode, output).toBe(0);
expect(output).toContain('5 pass');
expect(output).toContain('4 pass');
expect(output).toContain('0 fail');
} finally {
fs.rmSync(dir, { recursive: true, force: true });
+1 -1
View File
@@ -184,7 +184,7 @@ test('current detach supervision covers the live-census floor', () => {
const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8'));
const periodicTimeout = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]);
const gateTimeout = Number(pkg.scripts['eval:bg:gate'].match(/--timeout\s+(\d+)/)[1]);
expect(floorFor('gate')).toBe(26_597);
expect(floorFor('gate')).toBe(26_471);
expect(gateTimeout).toBe(49_320);
expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate'));
expect(floorFor('periodic')).toBe(30_797);
+1 -1
View File
@@ -147,7 +147,7 @@ export const FILE_RETRY_BUDGETS = [
...[
{ file: 'test/skill-e2e-qa-callers.test.ts', attemptMs: 5 * (CAPTURE_MS + 15_000), caseMs: CAPTURE_MS + 15_000, configuredRetries: 1 },
{ file: 'test/skill-e2e-shared-libs-paths.test.ts', attemptMs: 3 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS, configuredRetries: 1 },
{ file: 'test/skill-e2e-ship-docsync.test.ts', attemptMs: 5 * CAPTURE_LONG_MS + 8 * CAPTURE_MS, caseMs: CAPTURE_LONG_MS, configuredRetries: 1 },
{ file: 'test/skill-e2e-ship-docsync.test.ts', attemptMs: 4 * CAPTURE_LONG_MS + 8 * CAPTURE_MS, caseMs: CAPTURE_LONG_MS, configuredRetries: 1 },
// Seventeen workflow judges include their 10s recording grace; the other
// seven judges retain 120s. Supervise all 24 and the existing one retry.
{ file: 'test/skill-llm-eval.test.ts', attemptMs: 17 * (JUDGE_MS + 10_000) + 7 * JUDGE_MS, caseMs: JUDGE_MS + 10_000, configuredRetries: 1 },
-3
View File
@@ -803,8 +803,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'ship-triage': [ 'ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts',
'scripts/resolvers/testing.ts'
],
'ship-docsync': [ 'ship/**', 'document-release/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/sections.ts', 'test/skill-e2e-ship-docsync.test.ts',
'scripts/resolvers/testing.ts', 'test/helpers/docsync-*.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
'ship-docsync-completion': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
'ship-docsync-current': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
'ship-docsync-failure': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
@@ -1350,7 +1348,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
'ship-local-hook-preservation': 'gate',
'ship-coverage-audit': 'gate',
'ship-triage': 'gate',
'ship-docsync': 'gate',
'ship-docsync-missing-marker': 'gate',
'ship-docsync-missing-asset': 'gate',
'ship-docsync-launch-failure': 'gate',
+6 -6
View File
@@ -20,7 +20,7 @@ const newBudgets = FILE_RETRY_BUDGETS.filter(row => !FINDING_RETRY_BUDGETS.some(
const expectedWalls = {
'test/skill-e2e-qa-callers.test.ts': 3_270_000,
'test/skill-e2e-shared-libs-paths.test.ts': 1_920_000,
'test/skill-e2e-ship-docsync.test.ts': 5_520_000,
'test/skill-e2e-ship-docsync.test.ts': 4_920_000,
'test/skill-llm-eval.test.ts': 6_220_000,
'test/skill-e2e-auq-consistency.test.ts': 1_080_000,
'test/skill-e2e-auq-matrix.test.ts': 3_720_000,
@@ -172,8 +172,8 @@ test('fixed AUQ count remains strict while mixed-tier files keep ordinary case h
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 0, false],
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 1, true],
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 2, false],
['test/skill-e2e-ship-docsync.test.ts', 5, true],
['test/skill-e2e-ship-docsync.test.ts', 7, true],
['test/skill-e2e-qa-callers.test.ts', 3, true],
['test/skill-e2e-qa-callers.test.ts', 5, true],
// A case shard of a case-sharded registered file executes exactly its case.
[plannedKey('test/skill-e2e-plan.test.ts'), 1, true],
[plannedKey('test/skill-e2e-plan.test.ts'), 2, false],
@@ -227,7 +227,7 @@ test('detached PR fallback and release commands cover their actual default worke
const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce(
(total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0,
)) / 1000 * 1.05);
expect(prFloor).toBe(71_957);
expect(prFloor).toBe(77_501);
expect(prWall).toBe(92_820_000);
expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000);
@@ -246,8 +246,8 @@ test('detached PR fallback and release commands cover their actual default worke
)) / 1000 * 1.05));
}
const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000;
expect(releaseFloors).toEqual([26_597, 30_797]);
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_394);
expect(releaseFloors).toEqual([26_471, 30_797]);
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_268);
expect(detachedReleaseWall).toBe(116_700_000);
expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000);
});
+47 -53
View File
@@ -13,11 +13,8 @@ import { runShipDocsFault } from './helpers/docsync-fault-eval';
const describeE2E = describeE2ETier('gate');
const collector = createEvalCollector('e2e-ship-docsync');
const names = ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled',
'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'];
async function runShipDocs(testName: string, scenario: DocsScenario, dispatchOnly = false) {
async function runShipDocs(testName: string, scenario: DocsScenario) {
if (!process.env.EVALS_RUN_ID) throw Error('Native docs acceptance requires EVALS_RUN_ID');
const deadline = Date.now() + CAPTURE_LONG_MS;
const fixture = fixtureDocs(scenario);
@@ -61,55 +58,51 @@ async function runShipDocs(testName: string, scenario: DocsScenario, dispatchOnl
expect(dispatch).toBeGreaterThanOrEqual(0);
expect(calls[dispatch].input.run_in_background).toBe(false);
if (publishCall >= 0) expect(dispatch).toBeLessThan(publishCall);
if (dispatchOnly) {
expect(['success', 'error_max_turns', 'timeout']).toContain(result.exitReason);
expect(result.exitReason).toBe('success');
expect(docsWriteFailures(observation!, scenario === 'current' || scenario === 'store' ? [] : [DOC_PATH], {
result, fixture, scripts: [publish], readOnly: scenario === 'current' || scenario === 'store',
})).toEqual([]);
expect(docsToolFailures(result, fixture, [publish], scenario === 'current' || scenario === 'store')).toEqual([]);
const output = fs.readFileSync(report, 'utf8');
const after = repoSnapshot(fixture.repo);
const changed = changedFiles(fixture.before, after);
expect(after.head).toBe(fixture.before.head);
expect(after.index).toBe(fixture.before.index);
expect(after.contents['personal-note.txt']).toBe(fixture.before.contents['personal-note.txt']);
expect(fs.readFileSync(path.join(fixture.repo, DOC_PATH), 'utf8')).toContain('User-maintained note: KEEP THIS EXACTLY.');
if (scenario === 'store') {
expect(output).toMatch(/Documentation[\s\S]*blocked/i);
expect(fs.existsSync(receipt)).toBe(false);
expect(publishCall).toBe(-1);
expect(changed).toEqual([]);
expect(output).not.toMatch(/Documentation(?: is|:) current/i);
} else {
expect(result.exitReason).toBe('success');
expect(docsWriteFailures(observation!, scenario === 'current' || scenario === 'store' ? [] : [DOC_PATH], {
result, fixture, scripts: [publish], readOnly: scenario === 'current' || scenario === 'store',
})).toEqual([]);
expect(docsToolFailures(result, fixture, [publish], scenario === 'current' || scenario === 'store')).toEqual([]);
const output = fs.readFileSync(report, 'utf8');
const after = repoSnapshot(fixture.repo);
const changed = changedFiles(fixture.before, after);
expect(after.head).toBe(fixture.before.head);
expect(after.index).toBe(fixture.before.index);
expect(after.contents['personal-note.txt']).toBe(fixture.before.contents['personal-note.txt']);
expect(fs.readFileSync(path.join(fixture.repo, DOC_PATH), 'utf8')).toContain('User-maintained note: KEEP THIS EXACTLY.');
if (scenario === 'legacy' || scenario === 'store') {
expect(output).toMatch(/Documentation[\s\S]*blocked/i);
expect(fs.existsSync(receipt)).toBe(false);
expect(publishCall).toBe(-1);
expect(changed).toEqual(scenario === 'legacy' ? [DOC_PATH] : []);
expect(output).not.toMatch(/Documentation(?: is|:) current/i);
} else {
const raw = JSON.parse(calls[dispatch].output.trimEnd().split('\n').at(-1)!);
const contract = parseDocsCompletion(calls[dispatch].output, raw.audit_id);
expect(JSON.stringify(calls[dispatch].input)).toContain(raw.audit_id);
expect(calls[dispatch].output).toContain('SESSION_KIND: spawned');
expect(sawSpawnedMarker(result)).toBe(true);
vetDocsCompletion(contract, {
settled: result.exitReason === 'success', markerSeen: sawSpawnedMarker(result),
headUnchanged: after.head === fixture.before.head, indexUnchanged: after.index === fixture.before.index,
candidateUnchanged: changed.every(p => p === DOC_PATH), readOnly: false,
changedPaths: changed, allowedDocs: [DOC_PATH],
});
expect(contract.status).toBe(scenario === 'current' ? 'current' : 'updated');
expect(contract.files_reviewed).toContain(DOC_PATH);
expect(docsCompletedRead(result, path.join(fixture.repo, DOC_PATH), fixture, {
source: Buffer.from(fixture.before.contents[DOC_PATH], 'base64').toString('utf8'),
beforeFirstEdit: scenario !== 'current',
})).toBe(true);
expect(output).toContain(contract.documentation_section);
expect(fs.existsSync(receipt)).toBe(true);
expect(publishCall).toBeGreaterThan(dispatch);
expect(changed).toEqual(scenario === 'current' ? [] : [DOC_PATH]);
if (scenario === 'updated') expect(fs.readFileSync(path.join(fixture.repo, DOC_PATH), 'utf8')).toMatch(/Default format: JSON\./i);
}
const actualMutation = calls.filter(call => call.tool === 'Bash').map(call => String(call.input?.command))
.filter(command => /\bgit\s+(?:add|commit|push|reset|checkout|stash|merge|pull|rebase)(?=[\s;&|<>)]|$)/.test(command));
expect(actualMutation).toEqual([]);
const raw = JSON.parse(calls[dispatch].output.trimEnd().split('\n').at(-1)!);
const contract = parseDocsCompletion(calls[dispatch].output, raw.audit_id);
expect(JSON.stringify(calls[dispatch].input)).toContain(raw.audit_id);
expect(calls[dispatch].output).toContain('SESSION_KIND: spawned');
expect(sawSpawnedMarker(result)).toBe(true);
vetDocsCompletion(contract, {
settled: result.exitReason === 'success', markerSeen: sawSpawnedMarker(result),
headUnchanged: after.head === fixture.before.head, indexUnchanged: after.index === fixture.before.index,
candidateUnchanged: changed.every(p => p === DOC_PATH), readOnly: false,
changedPaths: changed, allowedDocs: [DOC_PATH],
});
expect(contract.status).toBe(scenario === 'current' ? 'current' : 'updated');
expect(contract.files_reviewed).toContain(DOC_PATH);
expect(docsCompletedRead(result, path.join(fixture.repo, DOC_PATH), fixture, {
source: Buffer.from(fixture.before.contents[DOC_PATH], 'base64').toString('utf8'),
beforeFirstEdit: scenario !== 'current',
})).toBe(true);
expect(output).toContain(contract.documentation_section);
expect(fs.existsSync(receipt)).toBe(true);
expect(publishCall).toBeGreaterThan(dispatch);
expect(changed).toEqual(scenario === 'current' ? [] : [DOC_PATH]);
if (scenario === 'updated') expect(fs.readFileSync(path.join(fixture.repo, DOC_PATH), 'utf8')).toMatch(/Default format: JSON\./i);
}
const actualMutation = calls.filter(call => call.tool === 'Bash').map(call => String(call.input?.command))
.filter(command => /\bgit\s+(?:add|commit|push|reset|checkout|stash|merge|pull|rebase)(?=[\s;&|<>)]|$)/.test(command));
expect(actualMutation).toEqual([]);
passed = true;
} finally {
recordE2E(collector, testName, 'Ship doc-sync lifecycle', result, { passed });
@@ -120,8 +113,9 @@ async function runShipDocs(testName: string, scenario: DocsScenario, dispatchOnl
}
describeE2E('Ship doc-sync lifecycle E2E (gate)', () => {
describeIfSelected('Ship doc-sync lifecycle', names, () => {
testConcurrentIfSelected('ship-docsync', () => runShipDocs('ship-docsync', 'legacy', true), CAPTURE_LONG_MS);
describeIfSelected('Ship doc-sync lifecycle', ['ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled',
'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'], () => {
testConcurrentIfSelected('ship-docsync-completion', () => runShipDocs('ship-docsync-completion', 'updated'), CAPTURE_LONG_MS);
testConcurrentIfSelected('ship-docsync-current', () => runShipDocs('ship-docsync-current', 'current'), CAPTURE_LONG_MS);
testConcurrentIfSelected('ship-docsync-failure', () => runShipDocsFault('ship-docsync-failure', 'legacy-completion', collector, CAPTURE_LONG_MS), CAPTURE_LONG_MS);