mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 18:06:54 +02:00
test(ship-docsync): shard the doc-sync lifecycle by case and drop the duplicate dispatch-only case
ship-docsync ran the same fixture and prompt as ship-docsync-completion and asserted a subset of it. The file now runs one case per process, so its lane wall is its longest case instead of half the sum of thirteen.
This commit is contained in:
1 parent
bfd1783fbf
commit
45cdfab2a0
10 files changed
+75
-71
No files matched your search
@@ -71,6 +71,18 @@
|
|||||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-revalidation": 427000,
|
"test/skill-e2e-shared-libs.test.ts#shared-libs-review-revalidation": 427000,
|
||||||
"test/skill-e2e-shared-libs.test.ts#shared-libs-unsupported-git": 168000,
|
"test/skill-e2e-shared-libs.test.ts#shared-libs-unsupported-git": 168000,
|
||||||
"test/skill-e2e-ship-docsync.test.ts": 129000,
|
"test/skill-e2e-ship-docsync.test.ts": 129000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-completion": 491000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-current": 364000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-failure": 244000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-late-result": 169000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-launch-failure": 109000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-asset": 173000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-missing-marker": 131000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-recovery": 245000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-after": 285000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-stale-before": 210000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-store": 366000,
|
||||||
|
"test/skill-e2e-ship-docsync.test.ts#ship-docsync-timeout-unsettled": 151000,
|
||||||
"test/skill-e2e-ship-hook-consent.test.ts": 63000,
|
"test/skill-e2e-ship-hook-consent.test.ts": 63000,
|
||||||
"test/skill-e2e-ship-hook-refresh.test.ts": 70000,
|
"test/skill-e2e-ship-hook-refresh.test.ts": 70000,
|
||||||
"test/skill-e2e-skillify.test.ts": 188000,
|
"test/skill-e2e-skillify.test.ts": 188000,
|
||||||
|
|||||||
@@ -136,6 +136,7 @@ export const CASE_SHARDED_FILES: readonly string[] = [
|
|||||||
'test/skill-e2e-review-army.test.ts',
|
'test/skill-e2e-review-army.test.ts',
|
||||||
'test/skill-e2e-shared-libs-paths.test.ts',
|
'test/skill-e2e-shared-libs-paths.test.ts',
|
||||||
'test/skill-e2e-shared-libs.test.ts',
|
'test/skill-e2e-shared-libs.test.ts',
|
||||||
|
'test/skill-e2e-ship-docsync.test.ts',
|
||||||
];
|
];
|
||||||
|
|
||||||
/** Bun test names that differ from their E2E id. */
|
/** Bun test names that differ from their E2E id. */
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ export const PR_PROFILE_CASE_IDS = [
|
|||||||
'auq-format-gate', 'plan-design-review-no-ui-scope', 'office-hours-spec-review',
|
'auq-format-gate', 'plan-design-review-no-ui-scope', 'office-hours-spec-review',
|
||||||
'tpa-present', 'tpa-absent-linux',
|
'tpa-present', 'tpa-absent-linux',
|
||||||
'ship-local-workflow', 'ship-coverage-audit', 'docsync-spawned',
|
'ship-local-workflow', 'ship-coverage-audit', 'docsync-spawned',
|
||||||
'ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
|
'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
|
||||||
'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure',
|
'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure',
|
||||||
'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before',
|
'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before',
|
||||||
'ship-docsync-stale-after', 'ship-docsync-recovery',
|
'ship-docsync-stale-after', 'ship-docsync-recovery',
|
||||||
@@ -47,7 +47,7 @@ export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
|
|||||||
'test/skill-e2e-ship-hook-refresh.test.ts': ['ship-managed-hook-refresh'],
|
'test/skill-e2e-ship-hook-refresh.test.ts': ['ship-managed-hook-refresh'],
|
||||||
'test/skill-e2e-ship-hook-consent.test.ts': ['ship-unmanaged-hook-consent', 'ship-local-hook-preservation'],
|
'test/skill-e2e-ship-hook-consent.test.ts': ['ship-unmanaged-hook-consent', 'ship-local-hook-preservation'],
|
||||||
'test/skill-e2e-docsync-spawned.test.ts': ['docsync-spawned'],
|
'test/skill-e2e-docsync-spawned.test.ts': ['docsync-spawned'],
|
||||||
'test/skill-e2e-ship-docsync.test.ts': ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store', 'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'],
|
'test/skill-e2e-ship-docsync.test.ts': ['ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store', 'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'],
|
||||||
'test/skill-e2e-deploy.test.ts': ['setup-deploy-workflow'],
|
'test/skill-e2e-deploy.test.ts': ['setup-deploy-workflow'],
|
||||||
'test/skill-e2e-session-intelligence.test.ts': ['context-restore-loads-latest'],
|
'test/skill-e2e-session-intelligence.test.ts': ['context-restore-loads-latest'],
|
||||||
'test/skill-e2e-plan-tune.test.ts': ['plan-tune-inspect'],
|
'test/skill-e2e-plan-tune.test.ts': ['plan-tune-inspect'],
|
||||||
|
|||||||
@@ -45,9 +45,9 @@ test('the existing quality and behavior phases retain their complete separate sh
|
|||||||
expect(behavior.evalsAll).toBe(true);
|
expect(behavior.evalsAll).toBe(true);
|
||||||
expect(qualityFiles).toHaveLength(1);
|
expect(qualityFiles).toHaveLength(1);
|
||||||
// 44 files (first-task-scaffold registers no gate case, so the gate lane
|
// 44 files (first-task-scaffold registers no gate case, so the gate lane
|
||||||
// skips it); the five case-sharded files contribute one shard per gate case.
|
// skips it); the six case-sharded files contribute one shard per gate case.
|
||||||
expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(44);
|
expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(44);
|
||||||
expect(behaviorFiles).toHaveLength(62);
|
expect(behaviorFiles).toHaveLength(73);
|
||||||
expect(behaviorFiles).toEqual(expect.arrayContaining([
|
expect(behaviorFiles).toEqual(expect.arrayContaining([
|
||||||
'test/skill-e2e-qa-callers.test.ts',
|
'test/skill-e2e-qa-callers.test.ts',
|
||||||
'test/skill-e2e-qa-functional-fix.test.ts',
|
'test/skill-e2e-qa-functional-fix.test.ts',
|
||||||
|
|||||||
@@ -84,8 +84,8 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
|
|||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
await import(path.join(root, 'test/skill-e2e-ship-docsync.test.ts'));
|
await import(path.join(root, 'test/skill-e2e-ship-docsync.test.ts'));
|
||||||
expect(callbacks.size).toBe(13);
|
expect(callbacks.size).toBe(12);
|
||||||
for (const name of ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store']) {
|
for (const name of ['ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store']) {
|
||||||
test(name + ' constructs its real native request without launching it', async () => {
|
test(name + ' constructs its real native request without launching it', async () => {
|
||||||
const before = launched;
|
const before = launched;
|
||||||
await expect(callbacks.get(name)()).rejects.toBe(stopped);
|
await expect(callbacks.get(name)()).rejects.toBe(stopped);
|
||||||
@@ -101,7 +101,7 @@ for (const name of ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-cur
|
|||||||
});
|
});
|
||||||
const output = result.stdout.toString() + result.stderr.toString();
|
const output = result.stdout.toString() + result.stderr.toString();
|
||||||
expect(result.exitCode, output).toBe(0);
|
expect(result.exitCode, output).toBe(0);
|
||||||
expect(output).toContain('5 pass');
|
expect(output).toContain('4 pass');
|
||||||
expect(output).toContain('0 fail');
|
expect(output).toContain('0 fail');
|
||||||
} finally {
|
} finally {
|
||||||
fs.rmSync(dir, { recursive: true, force: true });
|
fs.rmSync(dir, { recursive: true, force: true });
|
||||||
|
|||||||
@@ -184,7 +184,7 @@ test('current detach supervision covers the live-census floor', () => {
|
|||||||
const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8'));
|
const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8'));
|
||||||
const periodicTimeout = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]);
|
const periodicTimeout = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]);
|
||||||
const gateTimeout = Number(pkg.scripts['eval:bg:gate'].match(/--timeout\s+(\d+)/)[1]);
|
const gateTimeout = Number(pkg.scripts['eval:bg:gate'].match(/--timeout\s+(\d+)/)[1]);
|
||||||
expect(floorFor('gate')).toBe(26_597);
|
expect(floorFor('gate')).toBe(26_471);
|
||||||
expect(gateTimeout).toBe(49_320);
|
expect(gateTimeout).toBe(49_320);
|
||||||
expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate'));
|
expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate'));
|
||||||
expect(floorFor('periodic')).toBe(30_797);
|
expect(floorFor('periodic')).toBe(30_797);
|
||||||
|
|||||||
@@ -147,7 +147,7 @@ export const FILE_RETRY_BUDGETS = [
|
|||||||
...[
|
...[
|
||||||
{ file: 'test/skill-e2e-qa-callers.test.ts', attemptMs: 5 * (CAPTURE_MS + 15_000), caseMs: CAPTURE_MS + 15_000, configuredRetries: 1 },
|
{ file: 'test/skill-e2e-qa-callers.test.ts', attemptMs: 5 * (CAPTURE_MS + 15_000), caseMs: CAPTURE_MS + 15_000, configuredRetries: 1 },
|
||||||
{ file: 'test/skill-e2e-shared-libs-paths.test.ts', attemptMs: 3 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS, configuredRetries: 1 },
|
{ file: 'test/skill-e2e-shared-libs-paths.test.ts', attemptMs: 3 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS, configuredRetries: 1 },
|
||||||
{ file: 'test/skill-e2e-ship-docsync.test.ts', attemptMs: 5 * CAPTURE_LONG_MS + 8 * CAPTURE_MS, caseMs: CAPTURE_LONG_MS, configuredRetries: 1 },
|
{ file: 'test/skill-e2e-ship-docsync.test.ts', attemptMs: 4 * CAPTURE_LONG_MS + 8 * CAPTURE_MS, caseMs: CAPTURE_LONG_MS, configuredRetries: 1 },
|
||||||
// Seventeen workflow judges include their 10s recording grace; the other
|
// Seventeen workflow judges include their 10s recording grace; the other
|
||||||
// seven judges retain 120s. Supervise all 24 and the existing one retry.
|
// seven judges retain 120s. Supervise all 24 and the existing one retry.
|
||||||
{ file: 'test/skill-llm-eval.test.ts', attemptMs: 17 * (JUDGE_MS + 10_000) + 7 * JUDGE_MS, caseMs: JUDGE_MS + 10_000, configuredRetries: 1 },
|
{ file: 'test/skill-llm-eval.test.ts', attemptMs: 17 * (JUDGE_MS + 10_000) + 7 * JUDGE_MS, caseMs: JUDGE_MS + 10_000, configuredRetries: 1 },
|
||||||
|
|||||||
@@ -803,8 +803,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
|||||||
'ship-triage': [ 'ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts',
|
'ship-triage': [ 'ship/**', 'bin/gstack-repo-mode', 'test/skill-e2e-triage.test.ts',
|
||||||
'scripts/resolvers/testing.ts'
|
'scripts/resolvers/testing.ts'
|
||||||
],
|
],
|
||||||
'ship-docsync': [ 'ship/**', 'document-release/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/sections.ts', 'test/skill-e2e-ship-docsync.test.ts',
|
|
||||||
'scripts/resolvers/testing.ts', 'test/helpers/docsync-*.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
|
|
||||||
'ship-docsync-completion': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
|
'ship-docsync-completion': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
|
||||||
'ship-docsync-current': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
|
'ship-docsync-current': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
|
||||||
'ship-docsync-failure': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
|
'ship-docsync-failure': ['ship/**', 'document-release/**', 'test/skill-e2e-ship-docsync.test.ts', 'test/helpers/docsync-*.ts', 'test/helpers/session-runner.ts', 'test/helpers/hermetic-env.ts', 'bin/gstack-skill-start', 'bin/gstack-session-kind', 'scripts/resolvers/sections.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/testing.ts', 'test/helpers/qa-checkpoint-evidence.ts', 'test/helpers/qa-functional-observer.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/qa-evidence-producer.ts', 'test/helpers/qa-functional-fixture.ts'],
|
||||||
@@ -1350,7 +1348,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
|||||||
'ship-local-hook-preservation': 'gate',
|
'ship-local-hook-preservation': 'gate',
|
||||||
'ship-coverage-audit': 'gate',
|
'ship-coverage-audit': 'gate',
|
||||||
'ship-triage': 'gate',
|
'ship-triage': 'gate',
|
||||||
'ship-docsync': 'gate',
|
|
||||||
'ship-docsync-missing-marker': 'gate',
|
'ship-docsync-missing-marker': 'gate',
|
||||||
'ship-docsync-missing-asset': 'gate',
|
'ship-docsync-missing-asset': 'gate',
|
||||||
'ship-docsync-launch-failure': 'gate',
|
'ship-docsync-launch-failure': 'gate',
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ const newBudgets = FILE_RETRY_BUDGETS.filter(row => !FINDING_RETRY_BUDGETS.some(
|
|||||||
const expectedWalls = {
|
const expectedWalls = {
|
||||||
'test/skill-e2e-qa-callers.test.ts': 3_270_000,
|
'test/skill-e2e-qa-callers.test.ts': 3_270_000,
|
||||||
'test/skill-e2e-shared-libs-paths.test.ts': 1_920_000,
|
'test/skill-e2e-shared-libs-paths.test.ts': 1_920_000,
|
||||||
'test/skill-e2e-ship-docsync.test.ts': 5_520_000,
|
'test/skill-e2e-ship-docsync.test.ts': 4_920_000,
|
||||||
'test/skill-llm-eval.test.ts': 6_220_000,
|
'test/skill-llm-eval.test.ts': 6_220_000,
|
||||||
'test/skill-e2e-auq-consistency.test.ts': 1_080_000,
|
'test/skill-e2e-auq-consistency.test.ts': 1_080_000,
|
||||||
'test/skill-e2e-auq-matrix.test.ts': 3_720_000,
|
'test/skill-e2e-auq-matrix.test.ts': 3_720_000,
|
||||||
@@ -172,8 +172,8 @@ test('fixed AUQ count remains strict while mixed-tier files keep ordinary case h
|
|||||||
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 0, false],
|
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 0, false],
|
||||||
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 1, true],
|
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 1, true],
|
||||||
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 2, false],
|
[AUQ_CONSISTENCY_RETRY_BUDGET.file, 2, false],
|
||||||
['test/skill-e2e-ship-docsync.test.ts', 5, true],
|
['test/skill-e2e-qa-callers.test.ts', 3, true],
|
||||||
['test/skill-e2e-ship-docsync.test.ts', 7, true],
|
['test/skill-e2e-qa-callers.test.ts', 5, true],
|
||||||
// A case shard of a case-sharded registered file executes exactly its case.
|
// A case shard of a case-sharded registered file executes exactly its case.
|
||||||
[plannedKey('test/skill-e2e-plan.test.ts'), 1, true],
|
[plannedKey('test/skill-e2e-plan.test.ts'), 1, true],
|
||||||
[plannedKey('test/skill-e2e-plan.test.ts'), 2, false],
|
[plannedKey('test/skill-e2e-plan.test.ts'), 2, false],
|
||||||
@@ -227,7 +227,7 @@ test('detached PR fallback and release commands cover their actual default worke
|
|||||||
const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce(
|
const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce(
|
||||||
(total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0,
|
(total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0,
|
||||||
)) / 1000 * 1.05);
|
)) / 1000 * 1.05);
|
||||||
expect(prFloor).toBe(71_957);
|
expect(prFloor).toBe(77_501);
|
||||||
expect(prWall).toBe(92_820_000);
|
expect(prWall).toBe(92_820_000);
|
||||||
expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000);
|
expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000);
|
||||||
|
|
||||||
@@ -246,8 +246,8 @@ test('detached PR fallback and release commands cover their actual default worke
|
|||||||
)) / 1000 * 1.05));
|
)) / 1000 * 1.05));
|
||||||
}
|
}
|
||||||
const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000;
|
const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000;
|
||||||
expect(releaseFloors).toEqual([26_597, 30_797]);
|
expect(releaseFloors).toEqual([26_471, 30_797]);
|
||||||
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_394);
|
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_268);
|
||||||
expect(detachedReleaseWall).toBe(116_700_000);
|
expect(detachedReleaseWall).toBe(116_700_000);
|
||||||
expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000);
|
expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -13,11 +13,8 @@ import { runShipDocsFault } from './helpers/docsync-fault-eval';
|
|||||||
|
|
||||||
const describeE2E = describeE2ETier('gate');
|
const describeE2E = describeE2ETier('gate');
|
||||||
const collector = createEvalCollector('e2e-ship-docsync');
|
const collector = createEvalCollector('e2e-ship-docsync');
|
||||||
const names = ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
|
|
||||||
'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled',
|
|
||||||
'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'];
|
|
||||||
|
|
||||||
async function runShipDocs(testName: string, scenario: DocsScenario, dispatchOnly = false) {
|
async function runShipDocs(testName: string, scenario: DocsScenario) {
|
||||||
if (!process.env.EVALS_RUN_ID) throw Error('Native docs acceptance requires EVALS_RUN_ID');
|
if (!process.env.EVALS_RUN_ID) throw Error('Native docs acceptance requires EVALS_RUN_ID');
|
||||||
const deadline = Date.now() + CAPTURE_LONG_MS;
|
const deadline = Date.now() + CAPTURE_LONG_MS;
|
||||||
const fixture = fixtureDocs(scenario);
|
const fixture = fixtureDocs(scenario);
|
||||||
@@ -61,55 +58,51 @@ async function runShipDocs(testName: string, scenario: DocsScenario, dispatchOnl
|
|||||||
expect(dispatch).toBeGreaterThanOrEqual(0);
|
expect(dispatch).toBeGreaterThanOrEqual(0);
|
||||||
expect(calls[dispatch].input.run_in_background).toBe(false);
|
expect(calls[dispatch].input.run_in_background).toBe(false);
|
||||||
if (publishCall >= 0) expect(dispatch).toBeLessThan(publishCall);
|
if (publishCall >= 0) expect(dispatch).toBeLessThan(publishCall);
|
||||||
if (dispatchOnly) {
|
expect(result.exitReason).toBe('success');
|
||||||
expect(['success', 'error_max_turns', 'timeout']).toContain(result.exitReason);
|
expect(docsWriteFailures(observation!, scenario === 'current' || scenario === 'store' ? [] : [DOC_PATH], {
|
||||||
|
result, fixture, scripts: [publish], readOnly: scenario === 'current' || scenario === 'store',
|
||||||
|
})).toEqual([]);
|
||||||
|
expect(docsToolFailures(result, fixture, [publish], scenario === 'current' || scenario === 'store')).toEqual([]);
|
||||||
|
const output = fs.readFileSync(report, 'utf8');
|
||||||
|
const after = repoSnapshot(fixture.repo);
|
||||||
|
const changed = changedFiles(fixture.before, after);
|
||||||
|
expect(after.head).toBe(fixture.before.head);
|
||||||
|
expect(after.index).toBe(fixture.before.index);
|
||||||
|
expect(after.contents['personal-note.txt']).toBe(fixture.before.contents['personal-note.txt']);
|
||||||
|
expect(fs.readFileSync(path.join(fixture.repo, DOC_PATH), 'utf8')).toContain('User-maintained note: KEEP THIS EXACTLY.');
|
||||||
|
if (scenario === 'store') {
|
||||||
|
expect(output).toMatch(/Documentation[\s\S]*blocked/i);
|
||||||
|
expect(fs.existsSync(receipt)).toBe(false);
|
||||||
|
expect(publishCall).toBe(-1);
|
||||||
|
expect(changed).toEqual([]);
|
||||||
|
expect(output).not.toMatch(/Documentation(?: is|:) current/i);
|
||||||
} else {
|
} else {
|
||||||
expect(result.exitReason).toBe('success');
|
const raw = JSON.parse(calls[dispatch].output.trimEnd().split('\n').at(-1)!);
|
||||||
expect(docsWriteFailures(observation!, scenario === 'current' || scenario === 'store' ? [] : [DOC_PATH], {
|
const contract = parseDocsCompletion(calls[dispatch].output, raw.audit_id);
|
||||||
result, fixture, scripts: [publish], readOnly: scenario === 'current' || scenario === 'store',
|
expect(JSON.stringify(calls[dispatch].input)).toContain(raw.audit_id);
|
||||||
})).toEqual([]);
|
expect(calls[dispatch].output).toContain('SESSION_KIND: spawned');
|
||||||
expect(docsToolFailures(result, fixture, [publish], scenario === 'current' || scenario === 'store')).toEqual([]);
|
expect(sawSpawnedMarker(result)).toBe(true);
|
||||||
const output = fs.readFileSync(report, 'utf8');
|
vetDocsCompletion(contract, {
|
||||||
const after = repoSnapshot(fixture.repo);
|
settled: result.exitReason === 'success', markerSeen: sawSpawnedMarker(result),
|
||||||
const changed = changedFiles(fixture.before, after);
|
headUnchanged: after.head === fixture.before.head, indexUnchanged: after.index === fixture.before.index,
|
||||||
expect(after.head).toBe(fixture.before.head);
|
candidateUnchanged: changed.every(p => p === DOC_PATH), readOnly: false,
|
||||||
expect(after.index).toBe(fixture.before.index);
|
changedPaths: changed, allowedDocs: [DOC_PATH],
|
||||||
expect(after.contents['personal-note.txt']).toBe(fixture.before.contents['personal-note.txt']);
|
});
|
||||||
expect(fs.readFileSync(path.join(fixture.repo, DOC_PATH), 'utf8')).toContain('User-maintained note: KEEP THIS EXACTLY.');
|
expect(contract.status).toBe(scenario === 'current' ? 'current' : 'updated');
|
||||||
if (scenario === 'legacy' || scenario === 'store') {
|
expect(contract.files_reviewed).toContain(DOC_PATH);
|
||||||
expect(output).toMatch(/Documentation[\s\S]*blocked/i);
|
expect(docsCompletedRead(result, path.join(fixture.repo, DOC_PATH), fixture, {
|
||||||
expect(fs.existsSync(receipt)).toBe(false);
|
source: Buffer.from(fixture.before.contents[DOC_PATH], 'base64').toString('utf8'),
|
||||||
expect(publishCall).toBe(-1);
|
beforeFirstEdit: scenario !== 'current',
|
||||||
expect(changed).toEqual(scenario === 'legacy' ? [DOC_PATH] : []);
|
})).toBe(true);
|
||||||
expect(output).not.toMatch(/Documentation(?: is|:) current/i);
|
expect(output).toContain(contract.documentation_section);
|
||||||
} else {
|
expect(fs.existsSync(receipt)).toBe(true);
|
||||||
const raw = JSON.parse(calls[dispatch].output.trimEnd().split('\n').at(-1)!);
|
expect(publishCall).toBeGreaterThan(dispatch);
|
||||||
const contract = parseDocsCompletion(calls[dispatch].output, raw.audit_id);
|
expect(changed).toEqual(scenario === 'current' ? [] : [DOC_PATH]);
|
||||||
expect(JSON.stringify(calls[dispatch].input)).toContain(raw.audit_id);
|
if (scenario === 'updated') expect(fs.readFileSync(path.join(fixture.repo, DOC_PATH), 'utf8')).toMatch(/Default format: JSON\./i);
|
||||||
expect(calls[dispatch].output).toContain('SESSION_KIND: spawned');
|
|
||||||
expect(sawSpawnedMarker(result)).toBe(true);
|
|
||||||
vetDocsCompletion(contract, {
|
|
||||||
settled: result.exitReason === 'success', markerSeen: sawSpawnedMarker(result),
|
|
||||||
headUnchanged: after.head === fixture.before.head, indexUnchanged: after.index === fixture.before.index,
|
|
||||||
candidateUnchanged: changed.every(p => p === DOC_PATH), readOnly: false,
|
|
||||||
changedPaths: changed, allowedDocs: [DOC_PATH],
|
|
||||||
});
|
|
||||||
expect(contract.status).toBe(scenario === 'current' ? 'current' : 'updated');
|
|
||||||
expect(contract.files_reviewed).toContain(DOC_PATH);
|
|
||||||
expect(docsCompletedRead(result, path.join(fixture.repo, DOC_PATH), fixture, {
|
|
||||||
source: Buffer.from(fixture.before.contents[DOC_PATH], 'base64').toString('utf8'),
|
|
||||||
beforeFirstEdit: scenario !== 'current',
|
|
||||||
})).toBe(true);
|
|
||||||
expect(output).toContain(contract.documentation_section);
|
|
||||||
expect(fs.existsSync(receipt)).toBe(true);
|
|
||||||
expect(publishCall).toBeGreaterThan(dispatch);
|
|
||||||
expect(changed).toEqual(scenario === 'current' ? [] : [DOC_PATH]);
|
|
||||||
if (scenario === 'updated') expect(fs.readFileSync(path.join(fixture.repo, DOC_PATH), 'utf8')).toMatch(/Default format: JSON\./i);
|
|
||||||
}
|
|
||||||
const actualMutation = calls.filter(call => call.tool === 'Bash').map(call => String(call.input?.command))
|
|
||||||
.filter(command => /\bgit\s+(?:add|commit|push|reset|checkout|stash|merge|pull|rebase)(?=[\s;&|<>)]|$)/.test(command));
|
|
||||||
expect(actualMutation).toEqual([]);
|
|
||||||
}
|
}
|
||||||
|
const actualMutation = calls.filter(call => call.tool === 'Bash').map(call => String(call.input?.command))
|
||||||
|
.filter(command => /\bgit\s+(?:add|commit|push|reset|checkout|stash|merge|pull|rebase)(?=[\s;&|<>)]|$)/.test(command));
|
||||||
|
expect(actualMutation).toEqual([]);
|
||||||
passed = true;
|
passed = true;
|
||||||
} finally {
|
} finally {
|
||||||
recordE2E(collector, testName, 'Ship doc-sync lifecycle', result, { passed });
|
recordE2E(collector, testName, 'Ship doc-sync lifecycle', result, { passed });
|
||||||
@@ -120,8 +113,9 @@ async function runShipDocs(testName: string, scenario: DocsScenario, dispatchOnl
|
|||||||
}
|
}
|
||||||
|
|
||||||
describeE2E('Ship doc-sync lifecycle E2E (gate)', () => {
|
describeE2E('Ship doc-sync lifecycle E2E (gate)', () => {
|
||||||
describeIfSelected('Ship doc-sync lifecycle', names, () => {
|
describeIfSelected('Ship doc-sync lifecycle', ['ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
|
||||||
testConcurrentIfSelected('ship-docsync', () => runShipDocs('ship-docsync', 'legacy', true), CAPTURE_LONG_MS);
|
'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled',
|
||||||
|
'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'], () => {
|
||||||
testConcurrentIfSelected('ship-docsync-completion', () => runShipDocs('ship-docsync-completion', 'updated'), CAPTURE_LONG_MS);
|
testConcurrentIfSelected('ship-docsync-completion', () => runShipDocs('ship-docsync-completion', 'updated'), CAPTURE_LONG_MS);
|
||||||
testConcurrentIfSelected('ship-docsync-current', () => runShipDocs('ship-docsync-current', 'current'), CAPTURE_LONG_MS);
|
testConcurrentIfSelected('ship-docsync-current', () => runShipDocs('ship-docsync-current', 'current'), CAPTURE_LONG_MS);
|
||||||
testConcurrentIfSelected('ship-docsync-failure', () => runShipDocsFault('ship-docsync-failure', 'legacy-completion', collector, CAPTURE_LONG_MS), CAPTURE_LONG_MS);
|
testConcurrentIfSelected('ship-docsync-failure', () => runShipDocsFault('ship-docsync-failure', 'legacy-completion', collector, CAPTURE_LONG_MS), CAPTURE_LONG_MS);
|
||||||
|
|||||||
Reference in new issue
Block a user