mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-10 15:09:00 +02:00
Merge remote-tracking branch 'origin/main' into ponytail-inspiration-review
# Conflicts: # CHANGELOG.md # TODOS.md # VERSION # docs/TESTING_INTERNALS.md # package.json # scripts/test-free-shards.ts # test/helpers/llm-judge.ts # test/helpers/touchfiles-data.ts
This commit is contained in:
@@ -0,0 +1,502 @@
|
||||
{
|
||||
"version": 1,
|
||||
"recordedAt": "2026-08-29T05:35:47.316Z",
|
||||
"durations": {
|
||||
"browse/test/activity.test.ts": 90,
|
||||
"browse/test/adversarial-security.test.ts": 79,
|
||||
"browse/test/batch.test.ts": 4451,
|
||||
"browse/test/bridge-chromium-e2e.test.ts": 801,
|
||||
"browse/test/browse-client.test.ts": 123,
|
||||
"browse/test/browser-manager-custom-chromium.test.ts": 416,
|
||||
"browse/test/browser-manager-unit.test.ts": 560,
|
||||
"browse/test/browser-skill-commands.test.ts": 1167,
|
||||
"browse/test/browser-skill-write.test.ts": 106,
|
||||
"browse/test/browser-skills-e2e.test.ts": 124,
|
||||
"browse/test/browser-skills-storage.test.ts": 112,
|
||||
"browse/test/build-command-response.test.ts": 485,
|
||||
"browse/test/build.test.ts": 150,
|
||||
"browse/test/bun-polyfill.test.ts": 17007,
|
||||
"browse/test/busy-daemon-iron-rule.test.ts": 16077,
|
||||
"browse/test/busy-daemon-recovery.test.ts": 147,
|
||||
"browse/test/cdp-allowlist.test.ts": 67,
|
||||
"browse/test/cdp-e2e.test.ts": 548,
|
||||
"browse/test/cdp-inspector-history-cap.test.ts": 72,
|
||||
"browse/test/cdp-mutex.test.ts": 749,
|
||||
"browse/test/cdp-session-cleanup.test.ts": 81,
|
||||
"browse/test/claude-bin.test.ts": 89,
|
||||
"browse/test/cli-lock.test.ts": 74,
|
||||
"browse/test/cli-setsid-daemonize.test.ts": 57,
|
||||
"browse/test/cli-start-final-healthcheck.test.ts": 58,
|
||||
"browse/test/cli-supervisor.test.ts": 68,
|
||||
"browse/test/commands.test.ts": 31392,
|
||||
"browse/test/compare-board.test.ts": 355,
|
||||
"browse/test/config.test.ts": 165,
|
||||
"browse/test/content-security.test.ts": 3676,
|
||||
"browse/test/cookie-import-browser.test.ts": 96,
|
||||
"browse/test/cookie-picker-routes.test.ts": 72,
|
||||
"browse/test/daemon-log-hygiene.test.ts": 77,
|
||||
"browse/test/daemon-mismatch-refuse.test.ts": 263,
|
||||
"browse/test/data-platform.test.ts": 80,
|
||||
"browse/test/domain-skills-e2e.test.ts": 548,
|
||||
"browse/test/domain-skills-storage.test.ts": 68,
|
||||
"browse/test/dual-listener.test.ts": 58,
|
||||
"browse/test/dx-polish.test.ts": 67,
|
||||
"browse/test/error-handling.test.ts": 66,
|
||||
"browse/test/extension-sender-auth.test.ts": 92,
|
||||
"browse/test/extension-token.test.ts": 444,
|
||||
"browse/test/file-drop.test.ts": 73,
|
||||
"browse/test/file-permissions.test.ts": 70,
|
||||
"browse/test/fill-change-event.test.ts": 4572,
|
||||
"browse/test/find-browse.test.ts": 70,
|
||||
"browse/test/findport.test.ts": 470,
|
||||
"browse/test/from-file-path-validation.test.ts": 67,
|
||||
"browse/test/gstack-config.test.ts": 351,
|
||||
"browse/test/gstack-update-check.test.ts": 1106,
|
||||
"browse/test/handoff.test.ts": 5878,
|
||||
"browse/test/launch-signal-flags.test.ts": 80,
|
||||
"browse/test/learnings-injection.test.ts": 101,
|
||||
"browse/test/media-extract-unit.test.ts": 48,
|
||||
"browse/test/memory-command.test.ts": 399,
|
||||
"browse/test/memory-leak-reproducer.test.ts": 403,
|
||||
"browse/test/pair-agent-e2e.test.ts": 857,
|
||||
"browse/test/pair-agent-optin-gate.test.ts": 71,
|
||||
"browse/test/pair-agent-tunnel-eval.test.ts": 1124,
|
||||
"browse/test/path-validation.test.ts": 88,
|
||||
"browse/test/pdf-flags.test.ts": 76,
|
||||
"browse/test/platform.test.ts": 53,
|
||||
"browse/test/playwright-core-patch.test.ts": 70,
|
||||
"browse/test/poisoned-bundle-probe.test.ts": 344,
|
||||
"browse/test/process-liveness-windows.test.ts": 89,
|
||||
"browse/test/proxy-config.test.ts": 76,
|
||||
"browse/test/proxy-redact.test.ts": 43,
|
||||
"browse/test/pty-inject-scan.test.ts": 67,
|
||||
"browse/test/pty-session-lease.test.ts": 54,
|
||||
"browse/test/rebrand-signed-bundle.test.ts": 62,
|
||||
"browse/test/regression-pr1169-pdf-from-file-invalid-json.test.ts": 80,
|
||||
"browse/test/restart-env.test.ts": 68,
|
||||
"browse/test/sanitize.test.ts": 83,
|
||||
"browse/test/screenshot-size-guard.test.ts": 289,
|
||||
"browse/test/security-adversarial-fixes.test.ts": 66,
|
||||
"browse/test/security-adversarial.test.ts": 57,
|
||||
"browse/test/security-audit-r2.test.ts": 94,
|
||||
"browse/test/security-bench.test.ts": 58,
|
||||
"browse/test/security-classifier-download-cleanup.test.ts": 68,
|
||||
"browse/test/security-classifier.test.ts": 66,
|
||||
"browse/test/security-integration.test.ts": 57,
|
||||
"browse/test/security-live-playwright.test.ts": 3613,
|
||||
"browse/test/security-sidecar-client.test.ts": 75,
|
||||
"browse/test/security.test.ts": 52,
|
||||
"browse/test/server-auth.test.ts": 63,
|
||||
"browse/test/server-embedder-terminal-port.test.ts": 3912,
|
||||
"browse/test/server-factory.test.ts": 471,
|
||||
"browse/test/server-flush-trackers.test.ts": 64,
|
||||
"browse/test/server-lock-errors.test.ts": 74,
|
||||
"browse/test/server-no-import-side-effects.test.ts": 730,
|
||||
"browse/test/server-proxy-fail-fast.test.ts": 1433,
|
||||
"browse/test/server-pty-lease-routes.test.ts": 72,
|
||||
"browse/test/server-sanitize-surrogates.test.ts": 57,
|
||||
"browse/test/server-security-surface.test.ts": 63,
|
||||
"browse/test/server-tmp-state-path.test.ts": 54,
|
||||
"browse/test/session-cookie-store.test.ts": 85,
|
||||
"browse/test/session-persist.test.ts": 10686,
|
||||
"browse/test/sidebar-tabs.test.ts": 68,
|
||||
"browse/test/sidebar-ux.test.ts": 70,
|
||||
"browse/test/sidepanel-patient-autoconnect.test.ts": 61,
|
||||
"browse/test/sidepanel-reattach.test.ts": 69,
|
||||
"browse/test/sidepanel-restart-dispose.test.ts": 76,
|
||||
"browse/test/skill-token.test.ts": 68,
|
||||
"browse/test/snapshot.test.ts": 11135,
|
||||
"browse/test/socks-bridge.test.ts": 548,
|
||||
"browse/test/sse-helpers.test.ts": 209,
|
||||
"browse/test/sse-session-cookie.test.ts": 49,
|
||||
"browse/test/state-ttl.test.ts": 53,
|
||||
"browse/test/stealth-extended.test.ts": 48,
|
||||
"browse/test/stealth-layer-c.test.ts": 53,
|
||||
"browse/test/stealth-webdriver.test.ts": 1234,
|
||||
"browse/test/stop-ack-before-shutdown.test.ts": 190,
|
||||
"browse/test/stop-dead-daemon.test.ts": 277,
|
||||
"browse/test/tab-each.test.ts": 91,
|
||||
"browse/test/tab-guardrail.test.ts": 367,
|
||||
"browse/test/tab-isolation.test.ts": 342,
|
||||
"browse/test/tab-session-frame-detach.test.ts": 53,
|
||||
"browse/test/telemetry-optout.test.ts": 161,
|
||||
"browse/test/telemetry.test.ts": 150,
|
||||
"browse/test/terminal-agent-detach-reattach.test.ts": 61,
|
||||
"browse/test/terminal-agent-integration.test.ts": 675,
|
||||
"browse/test/terminal-agent-internal-handler.test.ts": 68,
|
||||
"browse/test/terminal-agent-keepalive.test.ts": 71,
|
||||
"browse/test/terminal-agent-owner-watchdog.test.ts": 5085,
|
||||
"browse/test/terminal-agent-pid-identity.test.ts": 74,
|
||||
"browse/test/terminal-agent-port-range.test.ts": 78,
|
||||
"browse/test/terminal-agent-ring-buffer-runtime.test.ts": 86,
|
||||
"browse/test/terminal-agent-session-routing.test.ts": 61,
|
||||
"browse/test/terminal-agent-watchdog.test.ts": 65,
|
||||
"browse/test/terminal-agent.test.ts": 66,
|
||||
"browse/test/token-registry.test.ts": 69,
|
||||
"browse/test/tunnel-gate-unit.test.ts": 393,
|
||||
"browse/test/tunnel-revoke-cli.test.ts": 1360,
|
||||
"browse/test/url-validation.test.ts": 65,
|
||||
"browse/test/watch.test.ts": 411,
|
||||
"browse/test/watchdog.test.ts": 3538,
|
||||
"browse/test/welcome-page.test.ts": 80,
|
||||
"browse/test/windows-spawn-hide.test.ts": 91,
|
||||
"browse/test/xprotect-heal.test.ts": 861,
|
||||
"browse/test/xvfb.test.ts": 3103,
|
||||
"browser-skills/hackernews-frontpage/script.test.ts": 67,
|
||||
"design/test/auth.test.ts": 65,
|
||||
"design/test/daemon-discovery.test.ts": 15626,
|
||||
"design/test/daemon.test.ts": 86,
|
||||
"design/test/feedback-roundtrip-daemon.test.ts": 410,
|
||||
"design/test/feedback-roundtrip.test.ts": 5645,
|
||||
"design/test/gallery.test.ts": 66,
|
||||
"design/test/image-gen-pairing.test.ts": 67,
|
||||
"design/test/receipted-fetch.test.ts": 67,
|
||||
"design/test/serve.test.ts": 73,
|
||||
"design/test/variants-retry-after.test.ts": 8614,
|
||||
"ios-qa/daemon/test/allowlist.test.ts": 74,
|
||||
"ios-qa/daemon/test/audit.test.ts": 64,
|
||||
"ios-qa/daemon/test/auth-mint.test.ts": 63,
|
||||
"ios-qa/daemon/test/cli-mint.test.ts": 213,
|
||||
"ios-qa/daemon/test/daemon-integration.test.ts": 434,
|
||||
"ios-qa/daemon/test/proxy-classify.test.ts": 70,
|
||||
"ios-qa/daemon/test/session-tokens.test.ts": 59,
|
||||
"ios-qa/daemon/test/single-instance.test.ts": 66,
|
||||
"ios-qa/daemon/test/tailscale-localapi.test.ts": 68,
|
||||
"ios-qa/daemon/test/tunnel-bootstrap.test.ts": 466,
|
||||
"ios-qa/scripts/gen-accessors.test.ts": 114,
|
||||
"make-pdf/test/browseClient.test.ts": 61,
|
||||
"make-pdf/test/cli-args.test.ts": 56,
|
||||
"make-pdf/test/coverage-gaps.test.ts": 81,
|
||||
"make-pdf/test/diagram-prepass.test.ts": 100,
|
||||
"make-pdf/test/e2e/ci-prereqs.test.ts": 75,
|
||||
"make-pdf/test/e2e/combined-gate.test.ts": 2867,
|
||||
"make-pdf/test/e2e/diagram-gate.test.ts": 9739,
|
||||
"make-pdf/test/e2e/emoji-gate.test.ts": 1721,
|
||||
"make-pdf/test/e2e/format-gate.test.ts": 11694,
|
||||
"make-pdf/test/e2e/landscape-gate.test.ts": 12720,
|
||||
"make-pdf/test/image-policy.test.ts": 48,
|
||||
"make-pdf/test/pdftotext.test.ts": 81,
|
||||
"make-pdf/test/render-offline-sanitize.test.ts": 98,
|
||||
"make-pdf/test/render.test.ts": 116,
|
||||
"test/agent-sdk-runner.test.ts": 7272,
|
||||
"test/analytics.test.ts": 75,
|
||||
"test/anthropic-preflight.test.ts": 55,
|
||||
"test/artifacts-allowlist-decisions.test.ts": 55,
|
||||
"test/artifacts-init-migration.test.ts": 204,
|
||||
"test/audit-compliance.test.ts": 90,
|
||||
"test/auq-error-fallback-hook.test.ts": 244,
|
||||
"test/auq-format-always-loaded.test.ts": 78,
|
||||
"test/benchmark-cli.test.ts": 692,
|
||||
"test/benchmark-runner.test.ts": 62,
|
||||
"test/bin-context-windows-slug.test.ts": 625,
|
||||
"test/bin-windows-bun-import-paths.test.ts": 1285,
|
||||
"test/binding-template-drift.test.ts": 85,
|
||||
"test/brain-cache-roundtrip.test.ts": 151,
|
||||
"test/brain-cache-spec.test.ts": 76,
|
||||
"test/brain-preflight.test.ts": 84,
|
||||
"test/brain-sync-windows-paths.test.ts": 55,
|
||||
"test/brain-sync.test.ts": 26537,
|
||||
"test/branch-slug-hygiene.test.ts": 296,
|
||||
"test/build-gbrain-env.test.ts": 69,
|
||||
"test/build-script-shell-compat.test.ts": 62,
|
||||
"test/builder-profile.test.ts": 2131,
|
||||
"test/bun-version-drift.test.ts": 66,
|
||||
"test/cache-concurrent-refresh.test.ts": 105,
|
||||
"test/carve-guard-completeness.test.ts": 83,
|
||||
"test/carve-guards-negative.test.ts": 61,
|
||||
"test/carve-section-ordering.test.ts": 81,
|
||||
"test/catalog-budget.test.ts": 134,
|
||||
"test/catalog-mode-full.test.ts": 272,
|
||||
"test/catalog-trim.test.ts": 94,
|
||||
"test/changed-files-union.test.ts": 376,
|
||||
"test/ci-image-tag-binding.test.ts": 63,
|
||||
"test/claude-provider-keychain.test.ts": 192,
|
||||
"test/code-intelligence-cli.test.ts": 856,
|
||||
"test/code-intelligence.test.ts": 4407,
|
||||
"test/codex-generation-model.test.ts": 119,
|
||||
"test/codex-hardening.test.ts": 1196,
|
||||
"test/codex-model-probe.test.ts": 255,
|
||||
"test/codex-resume-flag-semantics.test.ts": 82,
|
||||
"test/codex-under-codex-detection.test.ts": 79,
|
||||
"test/codex-web-search-flag.test.ts": 214,
|
||||
"test/conductor-env-shim.test.ts": 46,
|
||||
"test/context-bill.test.ts": 142,
|
||||
"test/context-budget-ratchet.test.ts": 203,
|
||||
"test/context-save-hardening.test.ts": 266,
|
||||
"test/cso-preserved.test.ts": 66,
|
||||
"test/cso-spec-taxonomy-alignment.test.ts": 63,
|
||||
"test/declared-annotation.test.ts": 65,
|
||||
"test/design-flag-utils.test.ts": 64,
|
||||
"test/dev-setup-render-isolation.test.ts": 69,
|
||||
"test/diagram-render-drift.test.ts": 130,
|
||||
"test/diff-scope.test.ts": 1699,
|
||||
"test/discover-section-templates.test.ts": 66,
|
||||
"test/distill-apply.test.ts": 577,
|
||||
"test/distill-free-text.test.ts": 646,
|
||||
"test/docs-config-keys.test.ts": 132,
|
||||
"test/document-skills-redaction.test.ts": 59,
|
||||
"test/e2e-harness-audit.test.ts": 64,
|
||||
"test/e2e-tier-alignment.test.ts": 152,
|
||||
"test/egress-lib.test.ts": 409,
|
||||
"test/egress-receipt-wiring.test.ts": 226,
|
||||
"test/egress-receipt.test.ts": 5261,
|
||||
"test/empty-find-fallthrough.test.ts": 358,
|
||||
"test/eval-budgets-policy.test.ts": 78,
|
||||
"test/eval-cli-family.test.ts": 480,
|
||||
"test/eval-detach-timeout-floor.test.ts": 92,
|
||||
"test/eval-list-cli.test.ts": 221,
|
||||
"test/eval-model.test.ts": 57,
|
||||
"test/evals-workflow-matrix.test.ts": 71,
|
||||
"test/evidence.test.ts": 5281,
|
||||
"test/exit-propagation.test.ts": 401,
|
||||
"test/explain-level-config.test.ts": 206,
|
||||
"test/extension-pty-inject-invariant.test.ts": 72,
|
||||
"test/founder-resources-optout.test.ts": 114,
|
||||
"test/free-tests-workflow-wiring.test.ts": 66,
|
||||
"test/fs-atomic.test.ts": 64,
|
||||
"test/fs-utils.test.ts": 203,
|
||||
"test/gate-secret-scan.test.ts": 577,
|
||||
"test/gbrain-cycle-completed.test.ts": 73,
|
||||
"test/gbrain-detect-install.test.ts": 334,
|
||||
"test/gbrain-detect-shape.test.ts": 464,
|
||||
"test/gbrain-detection-override.test.ts": 907,
|
||||
"test/gbrain-dream-stage.test.ts": 167,
|
||||
"test/gbrain-exec-invariant.test.ts": 58,
|
||||
"test/gbrain-guards.test.ts": 76,
|
||||
"test/gbrain-init-rollback.test.ts": 87,
|
||||
"test/gbrain-init-voyage-code-3.test.ts": 71,
|
||||
"test/gbrain-lib-validate-varname.test.ts": 71,
|
||||
"test/gbrain-lib-verify.test.ts": 145,
|
||||
"test/gbrain-local-status.test.ts": 3545,
|
||||
"test/gbrain-refresh-install-render.test.ts": 68,
|
||||
"test/gbrain-repo-policy-client.test.ts": 473,
|
||||
"test/gbrain-repo-policy.test.ts": 828,
|
||||
"test/gbrain-source-gitignore.test.ts": 78,
|
||||
"test/gbrain-source-worktree-advance.test.ts": 525,
|
||||
"test/gbrain-sources-parse.test.ts": 71,
|
||||
"test/gbrain-sources.test.ts": 137,
|
||||
"test/gbrain-spawn-windows-shell.test.ts": 65,
|
||||
"test/gbrain-supabase-provision.test.ts": 162,
|
||||
"test/gbrain-sync-skip.test.ts": 11570,
|
||||
"test/gbrain-sync-voyage-code-3-integration.test.ts": 54,
|
||||
"test/gen-skill-docs-idempotency.test.ts": 1776,
|
||||
"test/gen-skill-docs-import-purity.test.ts": 88,
|
||||
"test/gen-skill-docs-out-dir.test.ts": 1282,
|
||||
"test/gen-skill-docs.test.ts": 3624,
|
||||
"test/global-discover.test.ts": 405,
|
||||
"test/gstack-artifacts-init.test.ts": 2732,
|
||||
"test/gstack-artifacts-url.test.ts": 167,
|
||||
"test/gstack-brain-context-load.test.ts": 445,
|
||||
"test/gstack-codex-session-import.test.ts": 546,
|
||||
"test/gstack-config-defaults.test.ts": 1270,
|
||||
"test/gstack-config-key-locale.test.ts": 104,
|
||||
"test/gstack-config-redact-keys.test.ts": 123,
|
||||
"test/gstack-decision-bins.test.ts": 3316,
|
||||
"test/gstack-decision-semantic.test.ts": 82,
|
||||
"test/gstack-decision.test.ts": 65,
|
||||
"test/gstack-detach.test.ts": 11513,
|
||||
"test/gstack-developer-profile.test.ts": 6859,
|
||||
"test/gstack-egress-cli.test.ts": 533,
|
||||
"test/gstack-gbrain-detect-mcp-mode.test.ts": 15425,
|
||||
"test/gstack-gbrain-mcp-verify.test.ts": 1209,
|
||||
"test/gstack-gbrain-source-wireup.test.ts": 1591,
|
||||
"test/gstack-gbrain-sync.test.ts": 1772,
|
||||
"test/gstack-home-module-scope.test.ts": 102,
|
||||
"test/gstack-learnings-search.test.ts": 282,
|
||||
"test/gstack-memory-helpers.test.ts": 80,
|
||||
"test/gstack-memory-ingest.test.ts": 2274,
|
||||
"test/gstack-next-version.test.ts": 11929,
|
||||
"test/gstack-paths.test.ts": 128,
|
||||
"test/gstack-question-log.test.ts": 1767,
|
||||
"test/gstack-question-preference.test.ts": 4280,
|
||||
"test/gstack-redact-cli.test.ts": 515,
|
||||
"test/gstack-repo-mode.test.ts": 789,
|
||||
"test/gstack-retro-metrics.test.ts": 597,
|
||||
"test/gstack-schema-pack.test.ts": 64,
|
||||
"test/gstack-session-kind.test.ts": 81,
|
||||
"test/gstack-settings-hook-schema-aware.test.ts": 2572,
|
||||
"test/gstack-skill-start.test.ts": 1442,
|
||||
"test/gstack-slug-cwd-walk-up.test.ts": 445,
|
||||
"test/gstack-slug-parity.test.ts": 688,
|
||||
"test/gstack-slug-sanitize.test.ts": 99,
|
||||
"test/gstack-state-root-override.test.ts": 351,
|
||||
"test/gstack-team-init-hook-schema.test.ts": 173,
|
||||
"test/gstack-upgrade-migration-v1_17_0_0.test.ts": 83,
|
||||
"test/gstack-upgrade-migration-v1_37_0_0.test.ts": 120,
|
||||
"test/gstack-upgrade-migration-v1_40_0_0.test.ts": 174,
|
||||
"test/gstack-version-bump.test.ts": 1304,
|
||||
"test/helpers-unit.test.ts": 67,
|
||||
"test/helpers/budget-override.test.ts": 61,
|
||||
"test/helpers/capture-parity-baseline.test.ts": 191,
|
||||
"test/helpers/claude-pty-runner.scope-gate-floor.unit.test.ts": 66,
|
||||
"test/helpers/claude-pty-runner.unit.test.ts": 80,
|
||||
"test/helpers/e2e-gate.unit.test.ts": 61,
|
||||
"test/helpers/eval-store.test.ts": 244,
|
||||
"test/helpers/gemini-session-runner.test.ts": 63,
|
||||
"test/helpers/hermetic-env.test.ts": 66,
|
||||
"test/helpers/observability.test.ts": 158,
|
||||
"test/helpers/providers/gemini.test.ts": 58,
|
||||
"test/helpers/run-bin.test.ts": 67,
|
||||
"test/helpers/session-runner.test.ts": 87,
|
||||
"test/heredoc-pipe-deadlock.test.ts": 121,
|
||||
"test/hermetic-skills-seeding.test.ts": 91,
|
||||
"test/hermetic-wiring.test.ts": 97,
|
||||
"test/hook-scripts.test.ts": 8381,
|
||||
"test/hooks-windows-paths.test.ts": 170,
|
||||
"test/host-config.test.ts": 857,
|
||||
"test/investigate-freeze-path.test.ts": 67,
|
||||
"test/ios-debug-bridge-release-guard.test.ts": 58,
|
||||
"test/ios-qa-regen.test.ts": 272,
|
||||
"test/ios-qa-stateserver-hardening.test.ts": 66,
|
||||
"test/ios-qa-swiftui-tap-regression.test.ts": 61,
|
||||
"test/is-conductor.test.ts": 45,
|
||||
"test/jargon-list.test.ts": 65,
|
||||
"test/jsonl-merge.test.ts": 274,
|
||||
"test/jsonl-store.test.ts": 63,
|
||||
"test/land-and-deploy-postfail.test.ts": 72,
|
||||
"test/learnings-injection.test.ts": 91,
|
||||
"test/learnings.test.ts": 2646,
|
||||
"test/llms-txt-shape.test.ts": 72,
|
||||
"test/memory-cache-injection.test.ts": 360,
|
||||
"test/memory-ingest-include-gitignored.test.ts": 95,
|
||||
"test/memory-ingest-no-put_page.test.ts": 72,
|
||||
"test/memory-ingest-timeout.test.ts": 69,
|
||||
"test/migration-checkpoint-ownership.test.ts": 137,
|
||||
"test/migrations-v1.27.0.0.test.ts": 371,
|
||||
"test/migrations-v1.65.0.0.test.ts": 194,
|
||||
"test/mktemp-portability.test.ts": 75,
|
||||
"test/model-overlay-fable-5.test.ts": 73,
|
||||
"test/model-overlay-gpt-5.6-sol.test.ts": 65,
|
||||
"test/model-overlay-opus-4-7.test.ts": 59,
|
||||
"test/model-overlay-opus-4-8.test.ts": 59,
|
||||
"test/model-overlay-sonnet-5.test.ts": 68,
|
||||
"test/no-quoted-tilde-assignments.test.ts": 79,
|
||||
"test/no-stale-gstack-brain-refs.test.ts": 604,
|
||||
"test/no-suicide-exit.test.ts": 102,
|
||||
"test/onboarding-moved-literals.test.ts": 92,
|
||||
"test/one-way-doors.test.ts": 59,
|
||||
"test/openclaw-native-skills.test.ts": 61,
|
||||
"test/paid-orphan-tripwire.test.ts": 102,
|
||||
"test/paid-selection-propagation.test.ts": 80,
|
||||
"test/paid-shards.test.ts": 1330,
|
||||
"test/pair-agent-token-hygiene.test.ts": 54,
|
||||
"test/parity-baseline-integrity.test.ts": 66,
|
||||
"test/parity-sectioned.test.ts": 67,
|
||||
"test/parity-suite.test.ts": 146,
|
||||
"test/plan-tune-gates.test.ts": 507,
|
||||
"test/plan-tune.test.ts": 655,
|
||||
"test/post-rename-doc-regen.test.ts": 70,
|
||||
"test/pr-title-rewrite.test.ts": 120,
|
||||
"test/pr-title-sync-workflow-safety.test.ts": 75,
|
||||
"test/preamble-compose.test.ts": 62,
|
||||
"test/preamble-first-task-scaffold.test.ts": 837,
|
||||
"test/pty-askuserquestion-single-line.test.ts": 67,
|
||||
"test/pty-skill-seeding-wiring.test.ts": 86,
|
||||
"test/question-log-hook.test.ts": 1144,
|
||||
"test/question-preference-hook.test.ts": 1477,
|
||||
"test/question-tuning-registry-path.test.ts": 55,
|
||||
"test/readme-throughput.test.ts": 151,
|
||||
"test/redact-audit-log.test.ts": 91,
|
||||
"test/redact-doc-resolver.test.ts": 64,
|
||||
"test/redact-engine-autoredact.test.ts": 72,
|
||||
"test/redact-engine.test.ts": 74,
|
||||
"test/redact-parcel-id-false-positive.test.ts": 58,
|
||||
"test/redact-pattern-lint.test.ts": 73,
|
||||
"test/redact-prepush-hook.test.ts": 1705,
|
||||
"test/redact-prepush-rebase-force-push.test.ts": 776,
|
||||
"test/redact-prepush-scan-range.test.ts": 1406,
|
||||
"test/regression-1539-review-self-verify.test.ts": 69,
|
||||
"test/regression-1611-gbrain-sync-resume.test.ts": 83,
|
||||
"test/regression-1624-retro-stale-base.test.ts": 59,
|
||||
"test/regression-issue2091-bsd-mktemp.test.ts": 189,
|
||||
"test/regression-pr1169-build-app-sed.test.ts": 91,
|
||||
"test/regression-pr1169-mktemp-fallbacks.test.ts": 57,
|
||||
"test/relink.test.ts": 2110,
|
||||
"test/required-reads.test.ts": 62,
|
||||
"test/resolver-ask-user-format.test.ts": 70,
|
||||
"test/resolvers-gbrain-put-rewrite.test.ts": 74,
|
||||
"test/resolvers-gbrain-save-results.test.ts": 57,
|
||||
"test/review-log.test.ts": 689,
|
||||
"test/routing-probe.test.ts": 66,
|
||||
"test/run-in-background-guidance.test.ts": 67,
|
||||
"test/run-shard-child.test.ts": 1274,
|
||||
"test/salience-allowlist.test.ts": 98,
|
||||
"test/schema-version-migration.test.ts": 89,
|
||||
"test/secret-sink-harness.test.ts": 105,
|
||||
"test/section-manifest-consistency.test.ts": 69,
|
||||
"test/security-dashboard-fallback.test.ts": 1382,
|
||||
"test/session-runner-timeout.test.ts": 8094,
|
||||
"test/session-update-autostash.test.ts": 169,
|
||||
"test/setup-alias-name-uniqueness.test.ts": 1173,
|
||||
"test/setup-bun-cmd-and-pipe-bugs.test.ts": 67,
|
||||
"test/setup-claude-skill-assets.test.ts": 595,
|
||||
"test/setup-cleanup-orphans.test.ts": 124,
|
||||
"test/setup-codesign.test.ts": 63,
|
||||
"test/setup-codex-model.test.ts": 64,
|
||||
"test/setup-conductor-worktree.test.ts": 69,
|
||||
"test/setup-emoji-font.test.ts": 90,
|
||||
"test/setup-gbrain-bin-invocation-paths.test.ts": 61,
|
||||
"test/setup-gbrain-path4-structure.test.ts": 65,
|
||||
"test/setup-help.test.ts": 81,
|
||||
"test/setup-hook-canonical-paths.test.ts": 67,
|
||||
"test/setup-plan-tune-hooks-noninteractive.test.ts": 218,
|
||||
"test/setup-runtime-lib-command.test.ts": 3765,
|
||||
"test/setup-sections-linking.test.ts": 59,
|
||||
"test/setup-windows-fallback.test.ts": 75,
|
||||
"test/setup-windows-rerun-refresh.test.ts": 131,
|
||||
"test/ship-apple-gate.test.ts": 62,
|
||||
"test/ship-document-release-dispatch.test.ts": 74,
|
||||
"test/ship-plan-completion-invariants.test.ts": 107,
|
||||
"test/ship-review-loop.test.ts": 103,
|
||||
"test/ship-template-redaction.test.ts": 184,
|
||||
"test/ship-test-detection-markers.test.ts": 259,
|
||||
"test/ship-version-sync.test.ts": 464,
|
||||
"test/skill-budget-regression.test.ts": 177,
|
||||
"test/skill-census.test.ts": 67,
|
||||
"test/skill-ceo-section-ordering.test.ts": 68,
|
||||
"test/skill-collision-sentinel.test.ts": 65,
|
||||
"test/skill-coverage-floor.test.ts": 82,
|
||||
"test/skill-coverage-matrix.test.ts": 74,
|
||||
"test/skill-cross-model-recommendation-emit.test.ts": 74,
|
||||
"test/skill-fixture.test.ts": 92,
|
||||
"test/skill-parser.test.ts": 71,
|
||||
"test/skill-preflight-budget.test.ts": 65,
|
||||
"test/skill-size-budget.test.ts": 477,
|
||||
"test/skill-validation.test.ts": 577,
|
||||
"test/slop-diff-cli.test.ts": 408,
|
||||
"test/spec-template-invariants.test.ts": 57,
|
||||
"test/spec-template-sync.test.ts": 224,
|
||||
"test/static-no-legacy-writes.test.ts": 1701,
|
||||
"test/strict-output.test.ts": 57,
|
||||
"test/takes-fence-fallback.test.ts": 50,
|
||||
"test/tasks-section-jq.test.ts": 75,
|
||||
"test/taste-engine.test.ts": 854,
|
||||
"test/team-mode.test.ts": 8780,
|
||||
"test/telemetry-repo-strip.test.ts": 76,
|
||||
"test/telemetry.test.ts": 4253,
|
||||
"test/template-context-parity.test.ts": 67,
|
||||
"test/terse-build.test.ts": 76,
|
||||
"test/test-free-shards.test.ts": 3011,
|
||||
"test/timeline-stop-hook.test.ts": 722,
|
||||
"test/timeline.test.ts": 961,
|
||||
"test/touchfiles-facade.test.ts": 82,
|
||||
"test/touchfiles-map-diff.test.ts": 218,
|
||||
"test/touchfiles.test.ts": 124,
|
||||
"test/tracker-guard-wiring.test.ts": 87,
|
||||
"test/tracker-guard.test.ts": 231,
|
||||
"test/transcript-section-logger.test.ts": 57,
|
||||
"test/uninstall-windows-copies.test.ts": 411,
|
||||
"test/uninstall.test.ts": 2509,
|
||||
"test/update-check-crash-sentinel.test.ts": 358,
|
||||
"test/upgrade-migration-v1.test.ts": 76,
|
||||
"test/upgrade-template-pins.test.ts": 60,
|
||||
"test/user-render-out-dir-install.test.ts": 165,
|
||||
"test/user-slug-fallback.test.ts": 230,
|
||||
"test/v0-dormancy.test.ts": 79,
|
||||
"test/verify-gate.test.ts": 712,
|
||||
"test/version-source.test.ts": 47,
|
||||
"test/workflow-concurrency.test.ts": 73,
|
||||
"test/worktree.test.ts": 506,
|
||||
"test/writing-style-resolver.test.ts": 62
|
||||
}
|
||||
}
|
||||
@@ -78,10 +78,12 @@ scripts/gen-agents-digest.ts, not this file.
|
||||
return { content, bytes: Buffer.byteLength(content, 'utf-8') };
|
||||
}
|
||||
|
||||
export function writeAgentsDigest(opts?: { root?: string }): { outPath: string; bytes: number } {
|
||||
export function writeAgentsDigest(opts?: { root?: string; outRoot?: string }): { outPath: string; bytes: number } {
|
||||
const root = opts?.root ?? ROOT;
|
||||
const { content, bytes } = generateAgentsDigest({ root });
|
||||
const outPath = path.join(root, DIGEST_RELPATH);
|
||||
// outRoot: outputs-only rule for --out-dir renders — inputs (VERSION) read
|
||||
// from root, the artifact lands wherever the caller isolates outputs.
|
||||
const outPath = path.join(opts?.outRoot ?? root, DIGEST_RELPATH);
|
||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||
fs.writeFileSync(outPath, content);
|
||||
return { outPath, bytes };
|
||||
|
||||
+58
-18
@@ -146,13 +146,18 @@ const EXPLAIN_LEVEL: 'default' | 'terse' = (() => {
|
||||
})();
|
||||
|
||||
// ─── Out-dir (dev workspace render isolation) ───────────────
|
||||
// --out-dir <abs-dir> redirects Claude SKILL.md + section output to a separate
|
||||
// (untracked) directory instead of writing in place, AND rewrites the literal
|
||||
// section-base path (`~/.claude/skills/gstack/<skill>/sections/`) inside the
|
||||
// generated content to point at the out-dir, so section Reads resolve to the
|
||||
// rendered copy rather than the global install. Used by bin/dev-setup to render
|
||||
// the gbrain `:user` variant for a Conductor workspace without dirtying tracked
|
||||
// source. Default (unset) = in-place, behavior unchanged. Claude host only.
|
||||
// --out-dir <abs-dir> redirects ALL generated output (Claude SKILL.md +
|
||||
// sections, external-host trees like .agents/.factory, openclaw docs,
|
||||
// gstack/llms.txt) into a separate (untracked) directory instead of writing
|
||||
// in place. OUTPUTS ONLY: inputs (templates, sections/, host configs) are
|
||||
// always read from ROOT. For the Claude host it ALSO rewrites the literal
|
||||
// section-base path (`~/.claude/skills/gstack/<skill>/sections/`) inside
|
||||
// generated content so section Reads resolve to the rendered copy — that
|
||||
// rewrite stays Claude-only (external hosts have their own path grammar).
|
||||
// Consumers: bin/dev-setup (renders the gbrain `:user` variant for a
|
||||
// Conductor workspace — byte-compat pinned by gen-skill-docs-out-dir tests)
|
||||
// and the former TREE_MUTATING tests, which render into a mkdtemp instead
|
||||
// of mutating the live tree. Default (unset) = in-place, unchanged.
|
||||
const OUT_DIR_ARG = process.argv.find(a => a.startsWith('--out-dir'));
|
||||
const OUT_DIR: string | null = (() => {
|
||||
if (!OUT_DIR_ARG) return null;
|
||||
@@ -778,7 +783,8 @@ function processExternalHost(
|
||||
const hostConfig = getHostConfig(host);
|
||||
|
||||
const name = externalSkillName(skillDir === '.' ? '' : skillDir, frontmatterName);
|
||||
const outputDir = path.join(ROOT, hostConfig.hostSubdir, 'skills', name);
|
||||
// --out-dir mirrors the host tree (outputs only; inputs read from ROOT).
|
||||
const outputDir = path.join(OUT_DIR ?? ROOT, hostConfig.hostSubdir, 'skills', name);
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
const outputPath = path.join(outputDir, 'SKILL.md');
|
||||
|
||||
@@ -837,8 +843,8 @@ function processTemplate(tmplPath: string, host: Host = 'claude'): { outputPath:
|
||||
// Determine skill directory relative to ROOT
|
||||
const skillDir = path.relative(ROOT, path.dirname(tmplPath));
|
||||
|
||||
// --out-dir (Claude only): mirror the skill tree into the out-dir instead of
|
||||
// writing in place. External hosts compute their own paths below.
|
||||
// --out-dir: mirror the skill tree into the out-dir instead of writing in
|
||||
// place (external hosts compute their own OUT_DIR-aware paths below).
|
||||
if (OUT_DIR && host === 'claude') {
|
||||
outputPath = path.join(OUT_DIR, skillDir, path.basename(tmplPath).replace(/\.tmpl$/, ''));
|
||||
}
|
||||
@@ -949,7 +955,7 @@ function processSectionTemplate(
|
||||
outputPath = path.join(OUT_DIR || ROOT, skillDir, 'sections', fileName);
|
||||
} else {
|
||||
const externalName = externalSkillName(skillDir, parentName);
|
||||
outputPath = path.join(ROOT, hostConfig.hostSubdir, 'skills', externalName, 'sections', fileName);
|
||||
outputPath = path.join(OUT_DIR ?? ROOT, hostConfig.hostSubdir, 'skills', externalName, 'sections', fileName);
|
||||
}
|
||||
if (!DRY_RUN) fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
return { outputPath, content };
|
||||
@@ -962,6 +968,20 @@ function findTemplates(): string[] {
|
||||
}
|
||||
|
||||
const ALL_HOSTS: Host[] = ALL_HOST_NAMES as Host[];
|
||||
|
||||
/**
|
||||
* The generator's whole executable body. Import-purity contract: importing
|
||||
* this module must NEVER touch the tree — test/gen-skill-docs.test.ts pulls
|
||||
* assertSinglePreamble via require(), test/catalog-trim.test.ts imports
|
||||
* helpers, and before this guard existed every such import regenerated all
|
||||
* 71 SKILL.md in place at module-load time (the root cause of half the
|
||||
* TREE_MUTATING serial shard; hazard class #2532). Pinned by
|
||||
* test/gen-skill-docs-import-purity.test.ts.
|
||||
*
|
||||
* Returns the process exit code. Kept synchronous so the module stays
|
||||
* require()-able (see the llms.txt IIFE note below).
|
||||
*/
|
||||
export function main(): number {
|
||||
const hostsToRun: Host[] = HOST_ARG_VAL === 'all' ? ALL_HOSTS : [HOST];
|
||||
const failures: { host: string; error: Error }[] = [];
|
||||
|
||||
@@ -1049,6 +1069,7 @@ for (const currentHost of hostsToRun) {
|
||||
console.log(`FRESH: ${relOutput}`);
|
||||
}
|
||||
} else {
|
||||
if (OUT_DIR) fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(outputPath, content);
|
||||
console.log(`GENERATED: ${relOutput}`);
|
||||
}
|
||||
@@ -1065,19 +1086,21 @@ for (const currentHost of hostsToRun) {
|
||||
// plain markdown, no placeholder resolution — and are copied byte-for-byte
|
||||
// to openclaw/ at gen time.
|
||||
if (currentHost === 'openclaw' && !DRY_RUN) {
|
||||
const openclawDir = path.join(ROOT, 'openclaw');
|
||||
const openclawTemplatesDir = path.join(openclawDir, 'templates');
|
||||
// Inputs from ROOT, outputs into OUT_DIR when set (outputs-only rule).
|
||||
const openclawTemplatesDir = path.join(ROOT, 'openclaw', 'templates');
|
||||
const openclawOutDir = path.join(OUT_DIR ?? ROOT, 'openclaw');
|
||||
if (OUT_DIR) fs.mkdirSync(openclawOutDir, { recursive: true });
|
||||
for (const variant of ['lite', 'full', 'plan'] as const) {
|
||||
const fileName = `gstack-${variant}-CLAUDE.md`;
|
||||
const content = fs.readFileSync(path.join(openclawTemplatesDir, fileName), 'utf-8');
|
||||
fs.writeFileSync(path.join(openclawDir, fileName), content);
|
||||
fs.writeFileSync(path.join(openclawOutDir, fileName), content);
|
||||
console.log(`GENERATED: openclaw/${fileName}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (DRY_RUN && hasChanges) {
|
||||
console.error(`\nGenerated SKILL.md files are stale (${currentHost} host). Run: bun run gen:skill-docs --host ${currentHost}`);
|
||||
if (HOST_ARG_VAL !== 'all') process.exit(1);
|
||||
if (HOST_ARG_VAL !== 'all') return 1;
|
||||
failures.push({ host: currentHost, error: new Error('Stale files detected') });
|
||||
}
|
||||
|
||||
@@ -1112,7 +1135,7 @@ for (const currentHost of hostsToRun) {
|
||||
// in the same commit" is only a real gate if every host failure is fatal here.
|
||||
if (failures.length > 0 && HOST_ARG_VAL === 'all') {
|
||||
console.error(`\n${failures.length} host(s) failed: ${failures.map(f => f.host).join(', ')}`);
|
||||
process.exit(1);
|
||||
return 1;
|
||||
}
|
||||
// Single host dry-run failure already handled above
|
||||
|
||||
@@ -1138,7 +1161,11 @@ if (!DRY_RUN) {
|
||||
if (!DRY_RUN) {
|
||||
void (async () => {
|
||||
try {
|
||||
const result = await writeLlmsTxt();
|
||||
const result = await writeLlmsTxt(
|
||||
// Outputs-only rule: under --out-dir even this index lands there
|
||||
// (a catalog-mode render must never rewrite the tracked llms.txt).
|
||||
OUT_DIR ? { outputPath: path.join(OUT_DIR, 'gstack', 'llms.txt') } : {},
|
||||
);
|
||||
if (result.warnings.length > 0) {
|
||||
for (const w of result.warnings) console.error(`[gen-llms-txt] WARN: ${w}`);
|
||||
} else {
|
||||
@@ -1153,7 +1180,9 @@ if (!DRY_RUN) {
|
||||
// freshness + byte budget asserted in test/agents-digest.test.ts.
|
||||
try {
|
||||
const { writeAgentsDigest, DIGEST_BYTE_BUDGET } = await import('./gen-agents-digest');
|
||||
const digest = writeAgentsDigest();
|
||||
// Outputs-only rule: under --out-dir the digest lands there too — a
|
||||
// workspace render must never rewrite the tracked committed artifact.
|
||||
const digest = writeAgentsDigest(OUT_DIR ? { outRoot: OUT_DIR } : {});
|
||||
console.log(`[gen-agents-digest] agents-digest/gstack-AGENTS.md: ${digest.bytes} bytes (budget ${DIGEST_BYTE_BUDGET})`);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
@@ -1164,3 +1193,14 @@ if (!DRY_RUN) {
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
// Failure exits are immediate (matching the old top-level process.exit
|
||||
// behavior); success leaves the event loop to drain so the llms.txt
|
||||
// fire-and-forget IIFE inside main() finishes its write.
|
||||
const code = main();
|
||||
if (code !== 0) process.exit(code);
|
||||
}
|
||||
|
||||
+211
-42
@@ -282,12 +282,18 @@ const KNOWN_WINDOWS_SAFE: Array<{ file: string; reason: string }> = [
|
||||
{
|
||||
file: 'browse/test/file-permissions.test.ts',
|
||||
// Trips the POSIX-mode-bitmask pattern, but every `mode & 0o777` assertion
|
||||
// is platform-guarded (win32 returns early / takes the icacls branch).
|
||||
// is platform-guarded: win32-only tests return early, POSIX-only tests
|
||||
// guard the bitmask behind `process.platform !== 'win32'`, and the
|
||||
// symlink-skip regression test both wraps symlinkSync in try/catch
|
||||
// (runners without Developer Mode can't create symlinks) and guards its
|
||||
// bitmask — on win32 it asserts behavior (warns, skips, doesn't throw,
|
||||
// target stays usable), never fake Windows mode bits (dirs stat 0o777
|
||||
// there, so a 0o755 expectation fails on runner semantics, not our code).
|
||||
// This file carries the win32-only icacls-by-SID regression tests, which
|
||||
// can ONLY execute on windows-latest — excluding it here means the
|
||||
// machine-account ACL lockout regression is never exercised on the one
|
||||
// platform it bricks.
|
||||
reason: 'mode-bitmask hits are POSIX-branch only; win32-only ACL regression tests must run on windows-latest',
|
||||
reason: 'every mode-bitmask assertion is guarded off win32 (behavior asserted instead); win32-only ACL regression tests must run on windows-latest',
|
||||
},
|
||||
{
|
||||
file: 'browse/test/terminal-agent-owner-watchdog.test.ts',
|
||||
@@ -326,6 +332,16 @@ export const PER_FILE_WALL_MS = 5_000;
|
||||
export function wallTimeoutForShard(fileCount: number, baseMs = DEFAULT_WALL_TIMEOUT_MS): number {
|
||||
return Math.max(baseMs, fileCount * PER_FILE_WALL_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wall for a duration-packed shard. The count heuristic above assumes count
|
||||
* approximates cost; LPT packing breaks that BY DESIGN (a shard may hold six
|
||||
* slow Playwright files), so packed shards get max(base, predicted x 3) —
|
||||
* generous against seed drift, still bounded.
|
||||
*/
|
||||
export function wallTimeoutForPackedShard(predictedMs: number, baseMs = DEFAULT_WALL_TIMEOUT_MS): number {
|
||||
return Math.max(baseMs, Math.ceil(predictedMs * 3));
|
||||
}
|
||||
/**
|
||||
* Full-suite parallelism: leave RESERVED_CPUS cores for the parent runner +
|
||||
* OS, cap at MAX_FULL_SUITE_JOBS — beyond ~6 concurrent bun processes the
|
||||
@@ -375,46 +391,23 @@ export const WORKER_HOSTILE: Record<string, string> = {
|
||||
|
||||
/**
|
||||
* TREE-SERIAL files: run in ONE serial shard AFTER the parallel shards.
|
||||
* Two kinds live here:
|
||||
* - MUTATORS: tests that regenerate shared repo artifacts in place (skill
|
||||
* SKILL.md files or the .agents/ host outputs). A shard reading those
|
||||
* files concurrently sees a moving target — this family produced an
|
||||
* exactly-doubled catalog estimate, golden-file drift, and a spec-sync
|
||||
* mismatch before serialization.
|
||||
* - RATCHET READERS: tests that MEASURE the shared tree (parity caps,
|
||||
* size budgets). Measuring while any concurrent test regenerates is
|
||||
* undefined behavior — two runs failed with byte-identical inflated
|
||||
* skeletons while the tree was clean before and after, so rather than
|
||||
* hunt every present and future mutator, the measurers get a quiet
|
||||
* tree by construction.
|
||||
* Order within the serial shard is alphabetical (the file census is sorted
|
||||
* and the serial shard is a filter over it) — safety does NOT depend on
|
||||
* mutators-before-readers ordering; it rests on every mutator restoring
|
||||
* default state itself. (CI's --shards matrix is unaffected: each CI shard
|
||||
* has its own checkout.)
|
||||
* EMPTY since the 2026-08 dissolution — kept as a mechanism, not a museum:
|
||||
* a test that must regenerate shared repo artifacts IN PLACE (and cannot
|
||||
* render into an out-dir instead) earns an entry here with a reason, and
|
||||
* the runner will serialize it again.
|
||||
*
|
||||
* How it emptied: gen-skill-docs gained a main() guard (imports stopped
|
||||
* regenerating 71 files at load) and --out-dir grew to every host, so all
|
||||
* eight mutators now render into mkdtemps — the live tree is never written
|
||||
* by the suite (pinned by gen-skill-docs-import-purity + each migrated
|
||||
* file's own porcelain/mtime assertions). With zero mutators, the four
|
||||
* ratchet READERS (parity caps, size budgets, carve parity/ordering) get a
|
||||
* quiet tree by construction in any shard, so they rejoined the parallel
|
||||
* phase — the ~35-40s serial tail on every full-suite run is gone.
|
||||
* Keys are pinned against the live file census by test-free-shards.test.ts —
|
||||
* a renamed file fails the suite instead of silently dropping serialization.
|
||||
*/
|
||||
export const TREE_MUTATING: Record<string, string> = {
|
||||
'test/catalog-mode-full.test.ts': 'regenerates ALL SKILL.md in full-catalog mode, then restores',
|
||||
'test/spec-template-sync.test.ts': 'regenerates all SKILL.md in place to compare spec/SKILL.md',
|
||||
'test/gen-skill-docs-idempotency.test.ts': 'regenerates all SKILL.md twice to prove idempotency',
|
||||
'test/gen-skill-docs.test.ts': 'regenerates .agents/ (codex host) golden artifacts in place',
|
||||
'test/skill-validation.test.ts': 'regenerates .agents/ (codex host) artifacts in place (3 sites)',
|
||||
'test/gbrain-detection-override.test.ts':
|
||||
'regenerates SKILL.md in place with --respect-detection (gbrain variant), then git-restores — readers see inflated skeletons mid-window',
|
||||
'test/host-config.test.ts':
|
||||
'golden tests read .agents/.factory artifacts produced by gen-skill-docs.test.ts, and its beforeAll generates them when missing (#2532) — must not race the parallel readers or run before the mutators window',
|
||||
'test/catalog-trim.test.ts':
|
||||
'imports scripts/gen-skill-docs.ts, whose top-level body regenerates the full claude host at import time (71 files; idempotent on a fresh tree, but a stale tree gets rewritten mid-window) — same hazard class as #2532',
|
||||
'test/gen-skill-docs-out-dir.test.ts':
|
||||
'PORCELAIN READER — its before/after git-status pin needs a quiet tree (a concurrent shard\'s transient fixture write raced it on Windows CI), and its spawned render rewrites llms.txt/agents-digest in place (idempotent on a fresh tree)',
|
||||
// Ratchet readers (measure the tree; need it quiet):
|
||||
'test/parity-suite.test.ts': 'RATCHET READER — parity caps measure live SKILL.md/section bytes',
|
||||
'test/skill-size-budget.test.ts': 'RATCHET READER — per-skill and corpus size budgets measure the live tree',
|
||||
'test/carve-guard-completeness.test.ts': 'RATCHET READER — registry-vs-disk parity reads live sections/manifest.json files',
|
||||
'test/carve-section-ordering.test.ts': 'RATCHET READER — checkOrdering(ROOT) reads live skeletons and sections',
|
||||
};
|
||||
export const TREE_MUTATING: Record<string, string> = {};
|
||||
|
||||
export function normalizeRelativePath(filePath: string): string {
|
||||
return filePath.replace(/\\/g, '/');
|
||||
@@ -536,6 +529,92 @@ export function assignFilesToShards(files: string[], shardCount: number): string
|
||||
return shards.map(filesInShard => filesInShard.sort());
|
||||
}
|
||||
|
||||
// ─── Duration-aware packing (full-suite path ONLY) ─────────────────────────
|
||||
// Hash sharding balances file COUNTS (~1.15x spread) but not cost: the 15
|
||||
// Playwright-launching files land 4/3/4/1/2/1 across 6 shards, giving a
|
||||
// measured 28s–97s shard spread and ~40s of idle tail on every run. LPT
|
||||
// packing over recorded per-file durations reclaims most of it. The `--shard`
|
||||
// CI-matrix path is deliberately untouched — its contract is stable indices
|
||||
// via assignFilesToShards/stableHash (empty shards no-op; see above).
|
||||
//
|
||||
// One store, no overlay: durations come from the committed seed
|
||||
// (scripts/free-test-durations.json), refreshed occasionally via
|
||||
// `--record-durations` (each file timed in its own child — exact, and immune
|
||||
// to bun's stream buffering, where silent passers print no header to
|
||||
// timestamp). GSTACK_FREE_TEST_DURATIONS overrides the path for experiments.
|
||||
// The seed is a HINT, not a contract: missing file → hash-shard fallback;
|
||||
// unknown file → 75th-percentile pessimism (placed early by LPT, bounding
|
||||
// tail risk). Successor note: bun ≥1.3.14 ships native --timings/--shard LPT
|
||||
// scheduling — when the repo unpins 1.3.13, this packer is the code to
|
||||
// replace (keep it swappable).
|
||||
|
||||
export const FREE_TEST_DURATIONS_FILE = 'scripts/free-test-durations.json';
|
||||
|
||||
export function loadFreeTestDurations(rootDir = ROOT): Record<string, number> | null {
|
||||
const file = process.env.GSTACK_FREE_TEST_DURATIONS
|
||||
?? path.join(rootDir, FREE_TEST_DURATIONS_FILE);
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(file, 'utf-8');
|
||||
} catch {
|
||||
return null; // no seed — hash sharding, silently (fresh checkouts are normal)
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { durations?: Record<string, unknown> };
|
||||
const entries = Object.entries(parsed.durations ?? {})
|
||||
.filter((entry): entry is [string, number] =>
|
||||
typeof entry[1] === 'number' && Number.isFinite(entry[1]) && entry[1] >= 0);
|
||||
if (entries.length === 0) return null;
|
||||
return Object.fromEntries(entries);
|
||||
} catch (error) {
|
||||
// A corrupt seed (bad merge) must cost a warning, never the suite.
|
||||
console.error(`[test:free] WARNING: corrupt durations seed ${file} (${(error as Error).message}) — falling back to hash sharding`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface PackedShards {
|
||||
shards: string[][];
|
||||
/** Predicted total per shard, aligned with `shards` — feeds walls + logs. */
|
||||
predictedMs: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Longest-processing-time-first bin packing: files sorted by predicted
|
||||
* duration (desc, path-stable tiebreak) each go to the currently-lightest
|
||||
* shard. Deterministic for a given (files, shardCount, durations).
|
||||
*/
|
||||
export function packShardsByDuration(
|
||||
files: string[],
|
||||
shardCount: number,
|
||||
durations: Record<string, number>,
|
||||
): PackedShards {
|
||||
if (!Number.isInteger(shardCount) || shardCount <= 0) {
|
||||
throw new Error(`Shard count must be a positive integer. Received: ${shardCount}`);
|
||||
}
|
||||
const known = files
|
||||
.map((f) => durations[normalizeRelativePath(f)])
|
||||
.filter((v): v is number => typeof v === 'number')
|
||||
.sort((a, b) => a - b);
|
||||
// Unknown files get the 75th percentile of known durations: pessimistic, so
|
||||
// LPT places them early and a surprise long-runner can't recreate the tail.
|
||||
const fallback = known.length > 0 ? known[Math.min(known.length - 1, Math.floor(known.length * 0.75))] : 1;
|
||||
const predicted = (f: string): number => durations[normalizeRelativePath(f)] ?? fallback;
|
||||
|
||||
const ordered = [...files].sort((a, b) => predicted(b) - predicted(a) || (a < b ? -1 : 1));
|
||||
const shards = Array.from({ length: shardCount }, () => [] as string[]);
|
||||
const loads = new Array<number>(shardCount).fill(0);
|
||||
for (const file of ordered) {
|
||||
let lightest = 0;
|
||||
for (let i = 1; i < shardCount; i += 1) {
|
||||
if (loads[i] < loads[lightest]) lightest = i;
|
||||
}
|
||||
shards[lightest].push(file);
|
||||
loads[lightest] += predicted(file);
|
||||
}
|
||||
return { shards: shards.map((s) => s.sort()), predictedMs: loads };
|
||||
}
|
||||
|
||||
export interface BuildShardArgsOptions {
|
||||
/**
|
||||
* Pass bun's --parallel (worker-per-file, implies --isolate). No production
|
||||
@@ -561,6 +640,7 @@ export function buildShardArgs(files: string[], options: BuildShardArgsOptions =
|
||||
type CliOptions = {
|
||||
dryRun: boolean;
|
||||
listOnly: boolean;
|
||||
recordDurations: boolean;
|
||||
windowsOnly: boolean;
|
||||
verbose: boolean;
|
||||
shardCount: number;
|
||||
@@ -573,6 +653,7 @@ type CliOptions = {
|
||||
function parseCliOptions(argv: string[]): CliOptions {
|
||||
let dryRun = false;
|
||||
let listOnly = false;
|
||||
let recordDurations = false;
|
||||
let windowsOnly = false;
|
||||
let verbose = false;
|
||||
let shardCount = DEFAULT_SHARD_COUNT;
|
||||
@@ -584,6 +665,7 @@ function parseCliOptions(argv: string[]): CliOptions {
|
||||
const arg = argv[index];
|
||||
if (arg === '--dry-run') { dryRun = true; continue; }
|
||||
if (arg === '--list') { listOnly = true; continue; }
|
||||
if (arg === '--record-durations') { recordDurations = true; continue; }
|
||||
if (arg === '--windows-only') { windowsOnly = true; continue; }
|
||||
if (arg === '--verbose') { verbose = true; continue; }
|
||||
if (arg === '--shards') {
|
||||
@@ -611,7 +693,7 @@ function parseCliOptions(argv: string[]): CliOptions {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
return { dryRun, listOnly, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit };
|
||||
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit };
|
||||
}
|
||||
|
||||
function formatShardSummary(shards: string[][]): string[] {
|
||||
@@ -1069,6 +1151,17 @@ export async function runFreeShard(
|
||||
env.TMPDIR = childTmp;
|
||||
env.TEMP = childTmp;
|
||||
env.TMP = childTmp;
|
||||
// Per-shard Chromium profile (same isolation idea as TMPDIR): nine test
|
||||
// files launch in-process persistent contexts or daemons that default to
|
||||
// the SHARED ~/.gstack/chromium-profile, and two concurrent shards on one
|
||||
// profile dir kill each other's browser — observed live on CI once
|
||||
// duration packing recomposed shards (handoff's launchPersistentContext
|
||||
// died "Target page, context or browser has been closed" while a sibling
|
||||
// shard's daemon logged "Chromium process crashed"). Hash sharding had
|
||||
// masked the collision by chance placement. Within a shard, files run
|
||||
// serially, so sharing the per-shard profile is safe; config tests that
|
||||
// assert resolution order save/restore this env around their assertions.
|
||||
env.CHROMIUM_PROFILE = path.join(stateDir, 'chromium-profile');
|
||||
|
||||
const startedAt = Date.now();
|
||||
const child = spawn(command, args, {
|
||||
@@ -1198,6 +1291,63 @@ function exitCodeFor(status: FreeShardStatus): number {
|
||||
return status === 'timed-out' ? 124 : 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* `--record-durations`: time every file in its own child (exact per-file wall,
|
||||
* immune to bun's stream buffering) and write the committed seed atomically.
|
||||
* Occasional + manual by design — CI never records (a hint refreshed by a
|
||||
* human beats per-run churn), and the runtime (~serial suite / jobs) is fine
|
||||
* for an operation run a few times a quarter.
|
||||
*/
|
||||
async function recordFreeTestDurations(files: string[], jobs: number): Promise<number> {
|
||||
const durations: Record<string, number> = {};
|
||||
const failed: string[] = [];
|
||||
let cursor = 0;
|
||||
console.log(`[test:free] recording per-file durations: ${files.length} files across ${jobs} workers`);
|
||||
const worker = async (): Promise<void> => {
|
||||
for (;;) {
|
||||
const index = cursor;
|
||||
cursor += 1;
|
||||
if (index >= files.length) return;
|
||||
const file = files[index];
|
||||
const started = Date.now();
|
||||
const child = spawn('bun', ['test', file, `--timeout=${FREE_TEST_TIMEOUT_MS}`], {
|
||||
cwd: ROOT,
|
||||
stdio: ['ignore', 'ignore', 'ignore'],
|
||||
env: { ...process.env, GSTACK_HEADLESS: '1' },
|
||||
});
|
||||
const code = await new Promise<number>((resolve) => {
|
||||
const timer = setTimeout(() => { child.kill('SIGKILL'); }, wallTimeoutForShard(1));
|
||||
child.on('close', (c) => { clearTimeout(timer); resolve(c ?? 1); });
|
||||
child.on('error', () => { clearTimeout(timer); resolve(1); });
|
||||
});
|
||||
durations[normalizeRelativePath(file)] = Date.now() - started;
|
||||
if (code !== 0) failed.push(file);
|
||||
}
|
||||
};
|
||||
await Promise.all(Array.from({ length: Math.max(1, jobs) }, () => worker()));
|
||||
|
||||
const target = process.env.GSTACK_FREE_TEST_DURATIONS ?? path.join(ROOT, FREE_TEST_DURATIONS_FILE);
|
||||
const payload = {
|
||||
version: 1,
|
||||
recordedAt: new Date().toISOString(),
|
||||
durations: Object.fromEntries(Object.entries(durations).sort(([a], [b]) => (a < b ? -1 : 1))),
|
||||
};
|
||||
// Atomic temp+rename (capture-context-budget's pattern): a killed recorder
|
||||
// must never leave a truncated seed for loadFreeTestDurations to warn on.
|
||||
const tmp = `${target}.tmp-${process.pid}`;
|
||||
fs.writeFileSync(tmp, `${JSON.stringify(payload, null, 2)}\n`);
|
||||
fs.renameSync(tmp, target);
|
||||
console.log(`[test:free] wrote ${Object.keys(durations).length} durations to ${path.relative(ROOT, target)}`);
|
||||
if (failed.length > 0) {
|
||||
// Failures still recorded (a red file's duration is still a real cost),
|
||||
// but surfaced loudly — recording from a broken tree deserves a look.
|
||||
console.error(`[test:free] WARNING: ${failed.length} file(s) failed while recording:`);
|
||||
for (const f of failed) console.error(` ✗ ${f}`);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const options = parseCliOptions(process.argv.slice(2));
|
||||
const allFiles = collectFreeTestFiles();
|
||||
@@ -1225,6 +1375,11 @@ async function main(): Promise<number> {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (options.recordDurations) {
|
||||
const jobs = Math.max(1, Math.min(MAX_FULL_SUITE_JOBS, os.cpus().length - RESERVED_CPUS));
|
||||
return recordFreeTestDurations(files, jobs);
|
||||
}
|
||||
|
||||
if (options.dryRun) {
|
||||
const shards = assignFilesToShards(files, options.shardCount);
|
||||
const occupied = shards.filter((s) => s.length > 0).length;
|
||||
@@ -1268,15 +1423,29 @@ async function main(): Promise<number> {
|
||||
// serial shard, so no concurrent shard ever reads a half-regenerated tree.
|
||||
const mutators = files.filter((f) => f in TREE_MUTATING);
|
||||
const readers = files.filter((f) => !(f in TREE_MUTATING));
|
||||
const shards = assignFilesToShards(readers, jobs);
|
||||
const durations = loadFreeTestDurations();
|
||||
const packed = durations ? packShardsByDuration(readers, jobs, durations) : null;
|
||||
const shards = packed ? packed.shards : assignFilesToShards(readers, jobs);
|
||||
const totalShards = jobs + (mutators.length > 0 ? 1 : 0);
|
||||
console.log(`[test:free] full suite: ${readers.length} files across ${jobs} shard processes`
|
||||
+ (packed ? ' (duration-packed)' : '')
|
||||
+ (mutators.length > 0 ? `, then ${mutators.length} tree-mutating file(s) serially` : ''));
|
||||
if (packed) {
|
||||
// One line per shard so a packing regression is diagnosable from any log.
|
||||
packed.predictedMs.forEach((ms, i) => {
|
||||
console.log(`[test:free] shard ${i + 1}: ${shards[i].length} files, predicted ~${Math.round(ms / 1000)}s`);
|
||||
});
|
||||
}
|
||||
const shardTimeout = (fileCount: number): number =>
|
||||
options.wallTimeoutExplicit ? options.wallTimeoutMs : wallTimeoutForShard(fileCount, options.wallTimeoutMs);
|
||||
const outcomes = await Promise.all(
|
||||
shards.map((shardFiles, index) => runFreeShard(shardFiles, index + 1, totalShards, {
|
||||
wallTimeoutMs: shardTimeout(shardFiles.length),
|
||||
// Packed shards get duration-aware walls: LPT decouples file count from
|
||||
// cost BY DESIGN, so the 5s/file heuristic would undersize a shard
|
||||
// holding few expensive files.
|
||||
wallTimeoutMs: packed && !options.wallTimeoutExplicit
|
||||
? wallTimeoutForPackedShard(packed.predictedMs[index], options.wallTimeoutMs)
|
||||
: shardTimeout(shardFiles.length),
|
||||
verbose: options.verbose,
|
||||
})),
|
||||
);
|
||||
|
||||
+469
-50
@@ -50,20 +50,20 @@
|
||||
* bun run scripts/test-paid-shards.ts --timeout 600 --jobs 2
|
||||
*/
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { normalizeRelativePath } from './test-free-shards';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
exactTestFileSelectors,
|
||||
forwardAndClassify,
|
||||
installChildSignalForwarding,
|
||||
isTerminationRequested,
|
||||
killProcessGroup,
|
||||
runShardChild,
|
||||
strictTestExitCode,
|
||||
} from './test-strict-output';
|
||||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { PERIODIC_CI_EXCLUDE } from '../test/helpers/periodic-exclude-data';
|
||||
import { getProjectEvalDir } from '../test/helpers/eval-store';
|
||||
import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
|
||||
import {
|
||||
@@ -76,6 +76,7 @@ import {
|
||||
} from '../test/helpers/touchfiles';
|
||||
|
||||
export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
export { PERIODIC_CI_EXCLUDE };
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
@@ -143,7 +144,17 @@ export interface TierSelection {
|
||||
export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = ROOT): TierSelection {
|
||||
const selected: string[] = [];
|
||||
const excluded: Array<{ file: string; reason: string }> = [];
|
||||
// Periodic-lane exclusions (documented-red / manual-hardware files): a
|
||||
// known-red weekly shard is triage waste locally AND in CI, so the list
|
||||
// applies to every periodic run, with the reason surfaced per file.
|
||||
const ciExcluded = (file: string): { reason: string; tracking: string } | undefined =>
|
||||
tier === 'periodic' ? PERIODIC_CI_EXCLUDE[normalizeRelativePath(file)] : undefined;
|
||||
for (const file of files) {
|
||||
const exclusion = ciExcluded(file);
|
||||
if (exclusion) {
|
||||
excluded.push({ file, reason: `excluded: ${exclusion.reason} [${exclusion.tracking}]` });
|
||||
continue;
|
||||
}
|
||||
const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
|
||||
const classification = classifyPaidTestFile(source, tier);
|
||||
if (classification.included) selected.push(file);
|
||||
@@ -218,6 +229,26 @@ export function computePaidDiffSelection(
|
||||
return { selectedNames: new Set(selection.selected), reason: selection.reason, totalTests };
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the parent's diff selection for shard children (EVALS_SELECTION_JSON).
|
||||
*
|
||||
* Children's e2e-helpers module-load path adopts this instead of re-deriving
|
||||
* the selection per shard — which, when touchfiles-data.ts is in the diff,
|
||||
* spawned one bun subprocess PER CHILD to evaluate the old data file (the
|
||||
* map-diff path in test/helpers/test-selection.ts, 20s timeout each; 46-68
|
||||
* redundant children per full run). `selected: null` means run-all, mirroring
|
||||
* PaidDiffSelection.selectedNames. The child-side parser lives in
|
||||
* test/helpers/e2e-helpers.ts (parseEvalsSelectionJson); round-trip parity is
|
||||
* pinned by test/paid-selection-propagation.test.ts.
|
||||
*/
|
||||
export function serializePaidDiffSelection(selection: PaidDiffSelection): string {
|
||||
return JSON.stringify({
|
||||
version: 1,
|
||||
selected: selection.selectedNames === null ? null : [...selection.selectedNames].sort(),
|
||||
reason: selection.reason,
|
||||
});
|
||||
}
|
||||
|
||||
export interface ShardSkipDecision {
|
||||
file: string;
|
||||
kept: boolean;
|
||||
@@ -320,11 +351,14 @@ export function buildPaidShardArgs(
|
||||
files: string[],
|
||||
timeoutMs: number,
|
||||
maxConcurrency: number = DEFAULT_WITHIN_SHARD_CONCURRENCY,
|
||||
retries?: number,
|
||||
): string[] {
|
||||
// Explicit --concurrent/--max-concurrency: the legacy path always set one;
|
||||
// omitting it here made within-shard parallelism differ silently between
|
||||
// the two runners (observed: 1.6x sumdur/wall sharded vs 8x legacy).
|
||||
return ['test', ...files, '--retry', '1', '--concurrent', `--max-concurrency=${maxConcurrency}`, `--timeout=${timeoutMs}`];
|
||||
// Retries default to 1; RETRY_OVERRIDES membership (old matrix rows'
|
||||
// earned `retries: 2`) flows through retriesForFiles at the call site.
|
||||
return ['test', ...files, '--retry', String(retries ?? 1), '--concurrent', `--max-concurrency=${maxConcurrency}`, `--timeout=${timeoutMs}`];
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -338,7 +372,17 @@ export function shardSlug(files: string[]): string {
|
||||
.replace(/[^a-zA-Z0-9._+-]/g, '-');
|
||||
}
|
||||
|
||||
export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started' | 'skipped-by-diff';
|
||||
export type ShardStatus =
|
||||
| 'passed'
|
||||
| 'failed'
|
||||
| 'timed-out'
|
||||
| 'never-started'
|
||||
| 'skipped-by-diff'
|
||||
// exit 0 with ZERO executed tests on a run that promised everything
|
||||
// (EVALS_ALL): the hollow-file green the census backstop exists to catch.
|
||||
// Under selective runs, 0-executed passed shards stay 'passed' (in-file
|
||||
// diff/tier self-skips are legitimate there) and get a WARNING line only.
|
||||
| 'passed-empty';
|
||||
|
||||
export interface ShardOutcome {
|
||||
shard: number;
|
||||
@@ -347,6 +391,8 @@ export interface ShardOutcome {
|
||||
exitCode: number | null;
|
||||
elapsedMs: number;
|
||||
groupPid: number | null;
|
||||
/** Tests bun reported executing ("Ran N tests ..."), null when unknown. */
|
||||
executedTests: number | null;
|
||||
}
|
||||
|
||||
export interface ShardCommand {
|
||||
@@ -363,11 +409,43 @@ export interface RunShardsOptions {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** When set, each shard child gets GSTACK_EVAL_DIR=<evalDirBase>/shards/<slug>/. */
|
||||
evalDirBase?: string;
|
||||
/** Directory for the per-shard full-stream log files (default os.tmpdir()). Tests inject. */
|
||||
logDir?: string;
|
||||
/** Override the spawned command. Tests inject fake slow/spinning commands. */
|
||||
commandFor?: (files: string[]) => ShardCommand;
|
||||
log?: (line: string) => void;
|
||||
}
|
||||
|
||||
let shardLogSequence = 0;
|
||||
|
||||
/** Per-shard log path: slug + timestamp; pid + sequence defeat same-ms collisions. */
|
||||
function nextShardLogPath(files: string[], logDir: string): string {
|
||||
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||||
shardLogSequence += 1;
|
||||
return path.join(logDir, `gstack-paid-shard-${shardSlug(files)}-${stamp}-${process.pid}-${shardLogSequence}.log`);
|
||||
}
|
||||
|
||||
/** On-failure console excerpt budget: the last N bytes of the shard's log. */
|
||||
export const FAILURE_TAIL_BYTES = 64 * 1024;
|
||||
|
||||
/** Read back only the tail of a shard log (never the whole 30-min stream). */
|
||||
function readLogTail(logPath: string, maxBytes = FAILURE_TAIL_BYTES): string {
|
||||
try {
|
||||
const size = fs.statSync(logPath).size;
|
||||
const start = Math.max(0, size - maxBytes);
|
||||
const fd = fs.openSync(logPath, 'r');
|
||||
try {
|
||||
const buffer = Buffer.alloc(size - start);
|
||||
fs.readSync(fd, buffer, 0, buffer.length, start);
|
||||
return buffer.toString('utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
} catch {
|
||||
return ''; // a lost tail must never turn a real verdict into an exception
|
||||
}
|
||||
}
|
||||
|
||||
export async function runPaidShard(
|
||||
files: string[],
|
||||
shardNumber: number,
|
||||
@@ -389,6 +467,7 @@ export async function runPaidShard(
|
||||
exactTestFileSelectors(files, rootDir),
|
||||
timeoutMs,
|
||||
options.withinShardConcurrency ?? DEFAULT_WITHIN_SHARD_CONCURRENCY,
|
||||
retriesForFiles(files),
|
||||
),
|
||||
};
|
||||
|
||||
@@ -400,69 +479,90 @@ export async function runPaidShard(
|
||||
const startedAt = Date.now();
|
||||
log(`${label} START ${files.join(' ')} (timeout ${Math.round(timeoutMs / 1000)}s)`);
|
||||
|
||||
const child = spawn(command, args, {
|
||||
cwd: rootDir,
|
||||
env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
detached: process.platform !== 'win32',
|
||||
windowsHide: true,
|
||||
});
|
||||
const groupPid = child.pid ?? null;
|
||||
// Group-kill on parent SIGINT/SIGTERM too, not just on timeout.
|
||||
const forwarding = installChildSignalForwarding({
|
||||
kill: (signal?: NodeJS.Signals | number) => {
|
||||
killProcessGroup(child, (signal as NodeJS.Signals) ?? 'SIGTERM');
|
||||
return true;
|
||||
},
|
||||
// Full-stream spool: EVERY child byte lands on disk (the free runner's
|
||||
// model), never in a whole-run Buffer[] — non-live shards used to hold
|
||||
// their entire 30-min stream-json stdout+stderr in RAM, × concurrent jobs.
|
||||
// Printed at START so a wedged shard is inspectable live, mid-run.
|
||||
const logPath = nextShardLogPath(files, options.logDir ?? os.tmpdir());
|
||||
const logStream = fs.createWriteStream(logPath);
|
||||
let logWriteFailed = false;
|
||||
logStream.on('error', (err) => {
|
||||
if (logWriteFailed) return;
|
||||
logWriteFailed = true;
|
||||
console.error(`${label} could not write the full log at ${logPath}: ${err.message}`);
|
||||
});
|
||||
log(`${label} full log: ${logPath}`);
|
||||
|
||||
const classifier = new BunTestOutputClassifier();
|
||||
const buffered: Buffer[] = [];
|
||||
const sink = (destination: NodeJS.WriteStream): NodeJS.WriteStream => (streamLive
|
||||
? destination
|
||||
: ({ write: (chunk: Buffer | string) => buffered.push(Buffer.from(chunk)) } as unknown as NodeJS.WriteStream));
|
||||
|
||||
let timedOut = false;
|
||||
const killTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
killProcessGroup(child, 'SIGKILL');
|
||||
}, timeoutMs);
|
||||
// Tee: the spool always gets the chunk; live mode (jobs=1) also forwards to
|
||||
// the console. forwardAndClassify feeds the classifier FIRST, so the strict
|
||||
// verdict path is unchanged by where the bytes land afterwards.
|
||||
const sink = (destination: NodeJS.WriteStream): NodeJS.WriteStream => ({
|
||||
write: (chunk: Buffer | string): boolean => {
|
||||
if (!logWriteFailed) logStream.write(chunk);
|
||||
if (streamLive) destination.write(chunk);
|
||||
return true;
|
||||
},
|
||||
} as unknown as NodeJS.WriteStream);
|
||||
|
||||
let exitCode: number | null = null;
|
||||
let timedOut = false;
|
||||
let groupPid: number | null = null;
|
||||
try {
|
||||
const streams: Array<Promise<void>> = [];
|
||||
if (child.stdout) streams.push(forwardAndClassify(child.stdout, sink(process.stdout), classifier, 'stdout'));
|
||||
if (child.stderr) streams.push(forwardAndClassify(child.stderr, sink(process.stderr), classifier, 'stderr'));
|
||||
exitCode = await new Promise<number | null>((resolve, reject) => {
|
||||
child.once('error', reject);
|
||||
child.once('close', (code) => resolve(code));
|
||||
// Shared spawn/detached/group-kill/wall-timer/reap lifecycle.
|
||||
const result = await runShardChild({
|
||||
command,
|
||||
args,
|
||||
cwd: rootDir,
|
||||
env,
|
||||
timeoutMs,
|
||||
hookStreams: (child) => {
|
||||
const streams: Array<Promise<void>> = [];
|
||||
if (child.stdout) streams.push(forwardAndClassify(child.stdout, sink(process.stdout), classifier, 'stdout'));
|
||||
if (child.stderr) streams.push(forwardAndClassify(child.stderr, sink(process.stderr), classifier, 'stderr'));
|
||||
return streams;
|
||||
},
|
||||
});
|
||||
await Promise.all(streams);
|
||||
exitCode = result.exitCode;
|
||||
timedOut = result.timedOut;
|
||||
groupPid = result.groupPid;
|
||||
} finally {
|
||||
clearTimeout(killTimer);
|
||||
forwarding.dispose();
|
||||
// Reap survivors of this shard even on the clean path.
|
||||
killProcessGroup(child, 'SIGKILL');
|
||||
// Close the spool even when the spawn itself failed.
|
||||
await new Promise<void>((resolve) => logStream.end(() => resolve()));
|
||||
}
|
||||
|
||||
const summary = classifier.end();
|
||||
if (!streamLive && buffered.length > 0) process.stdout.write(Buffer.concat(buffered));
|
||||
|
||||
// Pass expectedFiles so a shard whose bun child ran fewer files than planned
|
||||
// (or zero, all self-skipped) with exit 0 is NOT recorded 'passed' — the
|
||||
// invisible-non-execution class this runner exists to kill. bun prints
|
||||
// "Ran N tests across M files" with M = selected files even when every test
|
||||
// self-skips, so terminalFileCounts must include files.length. Only enforced
|
||||
// on the real bun path: an injected commandFor (tests) isn't bun and emits no
|
||||
// terminal summary, so there's no file count to check against.
|
||||
const expectedFiles = options.commandFor ? undefined : files.length;
|
||||
// self-skips, so terminalFileCounts must include files.length. Enforced for
|
||||
// injected commandFor (tests) too, matching the free runner — fake passing
|
||||
// commands must print a synthetic `Ran N tests across M files. [Xms]` line,
|
||||
// so tests can pin the summary-missing => failure backstop.
|
||||
const expectedFiles = files.length;
|
||||
const status: ShardStatus = timedOut
|
||||
? 'timed-out'
|
||||
: strictTestExitCode(exitCode ?? 1, summary, expectedFiles) === 0 ? 'passed' : 'failed';
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
log(`${label} ${status.toUpperCase()} in ${Math.round(elapsedMs / 1000)}s (exit ${exitCode ?? 'signal'})`);
|
||||
|
||||
return { shard: shardNumber, files, status, exitCode, elapsedMs, groupPid };
|
||||
// Failure debuggability without the RAM cost: read back only the log's
|
||||
// tail. Live mode already streamed everything, so no re-print there.
|
||||
if (status !== 'passed' && !streamLive) {
|
||||
const tail = readLogTail(logPath);
|
||||
if (tail.length > 0) {
|
||||
process.stdout.write(`${label} last ${Math.min(tail.length, FAILURE_TAIL_BYTES)} bytes of ${logPath}:\n`);
|
||||
process.stdout.write(tail.endsWith('\n') ? tail : `${tail}\n`);
|
||||
}
|
||||
}
|
||||
const logSuffix = status === 'passed' ? '' : ` — full log: ${logPath}`;
|
||||
log(`${label} ${status.toUpperCase()} in ${Math.round(elapsedMs / 1000)}s (exit ${exitCode ?? 'signal'})${logSuffix}`);
|
||||
|
||||
const executedTests = summary.terminalTestCounts.length > 0
|
||||
? summary.terminalTestCounts.reduce((a, b) => a + b, 0)
|
||||
: null;
|
||||
return { shard: shardNumber, files, status, exitCode, elapsedMs, groupPid, executedTests };
|
||||
}
|
||||
|
||||
export interface RunSummary {
|
||||
@@ -483,7 +583,7 @@ export function summarize(outcomes: ShardOutcome[]): RunSummary {
|
||||
total: outcomes.length,
|
||||
executed: outcomes.length - count('never-started') - count('skipped-by-diff'),
|
||||
passed: count('passed'),
|
||||
failed: count('failed'),
|
||||
failed: count('failed') + count('passed-empty'),
|
||||
timedOut: count('timed-out'),
|
||||
neverStarted: count('never-started'),
|
||||
skippedByDiff: count('skipped-by-diff'),
|
||||
@@ -491,6 +591,28 @@ export function summarize(outcomes: ShardOutcome[]): RunSummary {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Hollow-shard guard. Under EVALS_ALL (the run promised EVERY test), a
|
||||
* passed shard whose bun summary reported 0 executed tests is not a pass —
|
||||
* it is the zero-execution class one layer down (file selected, every test
|
||||
* inside self-skipped, exit 0). Selective runs keep those shards 'passed'
|
||||
* (in-file diff/tier self-skips are legitimate) and only warn.
|
||||
*/
|
||||
export function applyHollowShardGuard(
|
||||
outcomes: ShardOutcome[],
|
||||
opts: { evalsAll: boolean; warn?: (line: string) => void },
|
||||
): ShardOutcome[] {
|
||||
const warn = opts.warn ?? ((line: string) => console.error(line));
|
||||
return outcomes.map((outcome) => {
|
||||
if (outcome.status !== 'passed' || outcome.executedTests !== 0) return outcome;
|
||||
if (!opts.evalsAll) {
|
||||
warn(`[test:paid] WARNING: shard ${outcome.shard} passed with 0 executed tests (${outcome.files.join(' ')}) — legitimate under selection, hollow under EVALS_ALL`);
|
||||
return outcome;
|
||||
}
|
||||
return { ...outcome, status: 'passed-empty' };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Exit code for a finished run: skipped-by-diff shards are successes (the
|
||||
* parent proved none of their tests were selected); everything else must
|
||||
@@ -513,6 +635,7 @@ export async function runPaidShards(
|
||||
exitCode: null,
|
||||
elapsedMs: 0,
|
||||
groupPid: null,
|
||||
executedTests: null,
|
||||
}));
|
||||
|
||||
let next = 0;
|
||||
@@ -535,6 +658,7 @@ export async function runPaidShards(
|
||||
exitCode: null,
|
||||
elapsedMs: 0,
|
||||
groupPid: null,
|
||||
executedTests: null,
|
||||
};
|
||||
console.error(`[test:paid] shard ${index + 1} could not run: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
@@ -562,6 +686,153 @@ export function formatSummary(summary: RunSummary): string[] {
|
||||
return lines;
|
||||
}
|
||||
|
||||
// ─── Planner / executor / report (the CI re-platform surface) ──────────────
|
||||
// One PLANNER computes selection and the slice plan ONCE; K executor jobs
|
||||
// consume it; a REPORT reconciles results against the plan. This kills two
|
||||
// classes at the root: per-slice selector divergence (one slice failing
|
||||
// merge-base resolution and running a different partition than its siblings)
|
||||
// and hollow lanes (a missing/failed slice that artifact-presence aggregation
|
||||
// would read as green). CI wiring: evals.yml planner job → K-way matrix of
|
||||
// `--plan manifest.json --slice i` → report job running `--report <dir>`.
|
||||
|
||||
export interface ManifestEntry {
|
||||
file: string;
|
||||
/** 1-based executor slice for planned entries; 0 for skipped/excluded. */
|
||||
slice: number;
|
||||
status: 'planned' | 'skipped-by-diff' | 'excluded';
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface PaidRunManifest {
|
||||
version: 1;
|
||||
tier: PaidTier;
|
||||
evalsAll: boolean;
|
||||
sliceCount: number;
|
||||
selectionReason: string;
|
||||
entries: ManifestEntry[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Files whose old evals.yml matrix rows carried `retries: 2`, with the
|
||||
* receipts that earned them (see the deleted rows' comments). The runner
|
||||
* default stays --retry 1; membership here is a literals map so retry
|
||||
* parity with the matrix is explicit, not folklore.
|
||||
*/
|
||||
export const RETRY_OVERRIDES: Record<string, number> = {
|
||||
'test/skill-e2e-workflow.test.ts': 2,
|
||||
'test/skill-e2e-office-hours-auto-mode.test.ts': 2,
|
||||
'test/skill-e2e-plan-mode-no-op.test.ts': 2,
|
||||
};
|
||||
|
||||
export function retriesForFiles(files: string[]): number {
|
||||
return Math.max(1, ...files.map((f) => RETRY_OVERRIDES[normalizeRelativePath(f)] ?? 1));
|
||||
}
|
||||
|
||||
/** Round-robin the RUNNABLE (sorted) shard plan across K slices — deterministic. */
|
||||
export function buildRunManifest(opts: {
|
||||
tier: PaidTier;
|
||||
sliceCount: number;
|
||||
evalsAll: boolean;
|
||||
discovered?: string[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
rootDir?: string;
|
||||
}): PaidRunManifest {
|
||||
if (!Number.isInteger(opts.sliceCount) || opts.sliceCount <= 0) {
|
||||
throw new Error(`--slices needs a positive integer. Received: ${opts.sliceCount}`);
|
||||
}
|
||||
const rootDir = opts.rootDir ?? ROOT;
|
||||
const discovered = opts.discovered ?? collectPaidTestFiles(rootDir);
|
||||
const { selected, excluded } = selectPaidTestFiles(discovered, opts.tier, rootDir);
|
||||
const shards = planPaidShards(selected, { maxFilesPerShard: 1 });
|
||||
const diffSelection = computePaidDiffSelection(opts.env ?? process.env);
|
||||
const { runnable, skipped } = partitionShardsByDiffSelection(shards, diffSelection.selectedNames);
|
||||
|
||||
const entries: ManifestEntry[] = [];
|
||||
runnable.forEach((files, index) => {
|
||||
entries.push({ file: files[0], slice: (index % opts.sliceCount) + 1, status: 'planned' });
|
||||
});
|
||||
for (const s of skipped) entries.push({ file: s.files[0], slice: 0, status: 'skipped-by-diff', reason: s.reason });
|
||||
for (const e of excluded) entries.push({ file: e.file, slice: 0, status: 'excluded', reason: e.reason });
|
||||
entries.sort((a, b) => (a.file < b.file ? -1 : 1));
|
||||
|
||||
return {
|
||||
version: 1,
|
||||
tier: opts.tier,
|
||||
evalsAll: opts.evalsAll,
|
||||
sliceCount: opts.sliceCount,
|
||||
selectionReason: diffSelection.reason,
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseRunManifest(raw: string): PaidRunManifest {
|
||||
const parsed = JSON.parse(raw) as PaidRunManifest;
|
||||
if (parsed.version !== 1) throw new Error(`unsupported manifest version: ${(parsed as { version?: unknown }).version}`);
|
||||
if (parsed.tier !== 'gate' && parsed.tier !== 'periodic') throw new Error(`manifest tier invalid: ${parsed.tier}`);
|
||||
if (!Number.isInteger(parsed.sliceCount) || parsed.sliceCount <= 0) throw new Error('manifest sliceCount invalid');
|
||||
if (!Array.isArray(parsed.entries)) throw new Error('manifest entries missing');
|
||||
for (const entry of parsed.entries) {
|
||||
if (typeof entry.file !== 'string' || !Number.isInteger(entry.slice)) throw new Error('manifest entry malformed');
|
||||
if (!['planned', 'skipped-by-diff', 'excluded'].includes(entry.status)) throw new Error(`manifest entry status invalid: ${entry.status}`);
|
||||
if (entry.status === 'planned' && (entry.slice < 1 || entry.slice > parsed.sliceCount)) {
|
||||
throw new Error(`planned entry ${entry.file} has out-of-range slice ${entry.slice}`);
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export interface SliceResult {
|
||||
version: 1;
|
||||
tier: PaidTier;
|
||||
sliceIndex: number;
|
||||
sliceCount: number;
|
||||
outcomes: Array<Pick<ShardOutcome, 'files' | 'status' | 'exitCode' | 'elapsedMs' | 'executedTests'>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile slice results against the manifest — the fail-closed aggregation.
|
||||
* Problems (any → non-zero): a slice index missing entirely (a cancelled or
|
||||
* crashed executor whose artifact never landed), a planned entry no slice
|
||||
* reported, an entry reported by the wrong/duplicate slice, or any reported
|
||||
* outcome that is not a pass.
|
||||
*/
|
||||
export function verifySliceResults(
|
||||
manifest: PaidRunManifest,
|
||||
results: SliceResult[],
|
||||
): { ok: boolean; problems: string[] } {
|
||||
const problems: string[] = [];
|
||||
const byIndex = new Map<number, SliceResult>();
|
||||
for (const result of results) {
|
||||
if (result.version !== 1) { problems.push(`slice result with unsupported version: ${String(result.version)}`); continue; }
|
||||
if (result.tier !== manifest.tier) problems.push(`slice ${result.sliceIndex} ran tier ${result.tier}, manifest says ${manifest.tier}`);
|
||||
if (byIndex.has(result.sliceIndex)) problems.push(`duplicate result for slice ${result.sliceIndex}`);
|
||||
byIndex.set(result.sliceIndex, result);
|
||||
}
|
||||
for (let index = 1; index <= manifest.sliceCount; index += 1) {
|
||||
if (!byIndex.has(index)) problems.push(`slice ${index}/${manifest.sliceCount} reported NO result — cancelled/crashed executor, not a pass`);
|
||||
}
|
||||
|
||||
const reported = new Map<string, { slice: number; status: ShardStatus }>();
|
||||
for (const result of results) {
|
||||
for (const outcome of result.outcomes) {
|
||||
const file = normalizeRelativePath(outcome.files[0] ?? '');
|
||||
if (reported.has(file)) problems.push(`${file} reported by two slices`);
|
||||
reported.set(file, { slice: result.sliceIndex, status: outcome.status });
|
||||
}
|
||||
}
|
||||
for (const entry of manifest.entries) {
|
||||
if (entry.status !== 'planned') continue;
|
||||
const got = reported.get(normalizeRelativePath(entry.file));
|
||||
if (!got) {
|
||||
if (byIndex.has(entry.slice)) problems.push(`planned ${entry.file} (slice ${entry.slice}) was never reported`);
|
||||
continue; // the missing-slice problem above already covers it
|
||||
}
|
||||
if (got.slice !== entry.slice) problems.push(`${entry.file} planned for slice ${entry.slice} but reported by slice ${got.slice}`);
|
||||
if (got.status !== 'passed') problems.push(`${entry.file}: ${got.status}`);
|
||||
}
|
||||
return { ok: problems.length === 0, problems };
|
||||
}
|
||||
|
||||
type CliOptions = {
|
||||
tier: PaidTier;
|
||||
listOnly: boolean;
|
||||
@@ -569,6 +840,16 @@ type CliOptions = {
|
||||
jobs: number;
|
||||
withinShardConcurrency: number;
|
||||
maxFilesPerShard: number;
|
||||
/** Planner mode: write the run manifest here and exit. */
|
||||
emitPlanPath: string | null;
|
||||
/** Slice count for --emit-plan. */
|
||||
slices: number;
|
||||
/** Executor mode: consume this manifest... */
|
||||
planPath: string | null;
|
||||
/** ...running only this 1-based slice. */
|
||||
sliceIndex: number | null;
|
||||
/** Report mode: reconcile manifest.json + slice-*.json under this dir. */
|
||||
reportDir: string | null;
|
||||
};
|
||||
|
||||
function parsePositiveInt(value: string | undefined, flag: string): number {
|
||||
@@ -605,6 +886,11 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
|
||||
? parsePositiveInt(env.EVALS_CONCURRENCY, 'EVALS_CONCURRENCY')
|
||||
: DEFAULT_WITHIN_SHARD_CONCURRENCY,
|
||||
maxFilesPerShard: DEFAULT_MAX_FILES_PER_SHARD,
|
||||
emitPlanPath: null,
|
||||
slices: 1,
|
||||
planPath: null,
|
||||
sliceIndex: null,
|
||||
reportDir: null,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
@@ -619,6 +905,23 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
|
||||
if (arg === '--timeout') { options.timeoutMs = parsePositiveInt(argv[index += 1], '--timeout') * 1000; continue; }
|
||||
if (arg === '--jobs') { options.jobs = parsePositiveInt(argv[index += 1], '--jobs'); continue; }
|
||||
if (arg === '--files-per-shard') { options.maxFilesPerShard = parsePositiveInt(argv[index += 1], '--files-per-shard'); continue; }
|
||||
if (arg === '--emit-plan') {
|
||||
const value = argv[index += 1];
|
||||
if (!value) throw new Error('--emit-plan needs a file path');
|
||||
options.emitPlanPath = value; continue;
|
||||
}
|
||||
if (arg === '--slices') { options.slices = parsePositiveInt(argv[index += 1], '--slices'); continue; }
|
||||
if (arg === '--plan') {
|
||||
const value = argv[index += 1];
|
||||
if (!value) throw new Error('--plan needs a manifest path');
|
||||
options.planPath = value; continue;
|
||||
}
|
||||
if (arg === '--slice') { options.sliceIndex = parsePositiveInt(argv[index += 1], '--slice'); continue; }
|
||||
if (arg === '--report') {
|
||||
const value = argv[index += 1];
|
||||
if (!value) throw new Error('--report needs a directory');
|
||||
options.reportDir = value; continue;
|
||||
}
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
return options;
|
||||
@@ -626,9 +929,111 @@ export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const options = parseCliOptions(process.argv.slice(2));
|
||||
|
||||
// ── Planner mode: compute selection + the slice plan ONCE, write it, exit.
|
||||
if (options.emitPlanPath) {
|
||||
const manifest = buildRunManifest({
|
||||
tier: options.tier,
|
||||
sliceCount: options.slices,
|
||||
evalsAll: process.env.EVALS_ALL === '1',
|
||||
});
|
||||
fs.mkdirSync(path.dirname(path.resolve(options.emitPlanPath)), { recursive: true });
|
||||
fs.writeFileSync(options.emitPlanPath, `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
const planned = manifest.entries.filter((e) => e.status === 'planned').length;
|
||||
const skipped = manifest.entries.filter((e) => e.status === 'skipped-by-diff').length;
|
||||
const excludedCount = manifest.entries.filter((e) => e.status === 'excluded').length;
|
||||
console.log(
|
||||
`[test:paid] plan: tier=${manifest.tier} evalsAll=${manifest.evalsAll} — `
|
||||
+ `${planned} planned across ${manifest.sliceCount} slice(s), ${skipped} skipped by diff, `
|
||||
+ `${excludedCount} excluded (${manifest.selectionReason})`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// ── Report mode: reconcile slice artifacts against the manifest. Fail-closed:
|
||||
// a slice whose artifact never landed is a FAILURE, not an absence.
|
||||
if (options.reportDir) {
|
||||
const manifest = parseRunManifest(fs.readFileSync(path.join(options.reportDir, 'manifest.json'), 'utf-8'));
|
||||
const results: SliceResult[] = fs.readdirSync(options.reportDir)
|
||||
.filter((name) => /^slice-\d+\.json$/.test(name))
|
||||
.map((name) => JSON.parse(fs.readFileSync(path.join(options.reportDir, name), 'utf-8')) as SliceResult);
|
||||
const verdict = verifySliceResults(manifest, results);
|
||||
const planned = manifest.entries.filter((e) => e.status === 'planned').length;
|
||||
console.log(`[test:paid] report: ${results.length}/${manifest.sliceCount} slices, ${planned} planned shards, tier=${manifest.tier}`);
|
||||
for (const result of results.sort((a, b) => a.sliceIndex - b.sliceIndex)) {
|
||||
for (const outcome of result.outcomes) {
|
||||
console.log(` slice ${result.sliceIndex} ${outcome.status.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s ${outcome.files.join(' ')}`);
|
||||
}
|
||||
}
|
||||
if (!verdict.ok) {
|
||||
console.error(`[test:paid] report: ${verdict.problems.length} problem(s):`);
|
||||
for (const problem of verdict.problems) console.error(` ✗ ${problem}`);
|
||||
return 1;
|
||||
}
|
||||
console.log('[test:paid] report: every planned shard accounted and passed');
|
||||
return 0;
|
||||
}
|
||||
|
||||
const discovered = collectPaidTestFiles();
|
||||
if (discovered.length === 0) throw new Error('No paid test files were discovered.');
|
||||
|
||||
// ── Executor mode: consume the planner's manifest; never self-select.
|
||||
if (options.planPath || options.sliceIndex !== null) {
|
||||
if (!options.planPath || options.sliceIndex === null) {
|
||||
throw new Error('--plan and --slice must be used together');
|
||||
}
|
||||
const manifest = parseRunManifest(fs.readFileSync(options.planPath, 'utf-8'));
|
||||
if (manifest.tier !== options.tier) {
|
||||
throw new Error(`manifest tier ${manifest.tier} != requested tier ${options.tier} — refusing a cross-tier run`);
|
||||
}
|
||||
if (options.sliceIndex > manifest.sliceCount) {
|
||||
throw new Error(`--slice ${options.sliceIndex} exceeds manifest sliceCount ${manifest.sliceCount}`);
|
||||
}
|
||||
const mine = manifest.entries.filter((e) => e.status === 'planned' && e.slice === options.sliceIndex);
|
||||
const shards = mine.map((e) => [e.file]);
|
||||
console.log(`[test:paid] slice ${options.sliceIndex}/${manifest.sliceCount}: ${shards.length} shard(s), tier=${manifest.tier}, evalsAll=${manifest.evalsAll}`);
|
||||
|
||||
const evalDirBase = process.env.GSTACK_EVAL_DIR || getProjectEvalDir();
|
||||
let summary: RunSummary;
|
||||
if (shards.length === 0) {
|
||||
summary = summarize([]);
|
||||
} else {
|
||||
preflightAnthropicApi(process.env);
|
||||
summary = await runPaidShards(shards, {
|
||||
timeoutMs: options.timeoutMs,
|
||||
jobs: options.jobs,
|
||||
withinShardConcurrency: options.withinShardConcurrency,
|
||||
env: {
|
||||
...process.env,
|
||||
EVALS: '1',
|
||||
EVALS_TIER: options.tier,
|
||||
...(manifest.evalsAll ? { EVALS_ALL: '1' } : {}),
|
||||
EVALS_PREFLIGHT_OK: '1',
|
||||
// The manifest IS the selection: children must not re-derive a
|
||||
// possibly-different one from their own git view.
|
||||
EVALS_SELECTION_JSON: JSON.stringify({ version: 1, selected: null, reason: `manifest slice ${options.sliceIndex}: ${manifest.selectionReason}` }),
|
||||
},
|
||||
evalDirBase,
|
||||
});
|
||||
}
|
||||
const guarded = applyHollowShardGuard(summary.outcomes, { evalsAll: manifest.evalsAll });
|
||||
summary = summarize(guarded);
|
||||
const sliceResult: SliceResult = {
|
||||
version: 1,
|
||||
tier: manifest.tier,
|
||||
sliceIndex: options.sliceIndex,
|
||||
sliceCount: manifest.sliceCount,
|
||||
outcomes: guarded.map(({ files, status, exitCode, elapsedMs, executedTests }) =>
|
||||
({ files, status, exitCode, elapsedMs, executedTests })),
|
||||
};
|
||||
fs.mkdirSync(evalDirBase, { recursive: true });
|
||||
const sliceResultPath = path.join(evalDirBase, `slice-${options.sliceIndex}.json`);
|
||||
fs.writeFileSync(sliceResultPath, `${JSON.stringify(sliceResult, null, 2)}\n`);
|
||||
console.log(`[test:paid] slice result: ${sliceResultPath}`);
|
||||
for (const line of formatSummary(summary)) console.log(line);
|
||||
return summaryExitCode(summary);
|
||||
}
|
||||
|
||||
const { selected, excluded } = selectPaidTestFiles(discovered, options.tier);
|
||||
const shards = planPaidShards(selected, { maxFilesPerShard: options.maxFilesPerShard });
|
||||
|
||||
@@ -676,7 +1081,17 @@ async function main(): Promise<number> {
|
||||
timeoutMs: options.timeoutMs,
|
||||
jobs: options.jobs,
|
||||
withinShardConcurrency: options.withinShardConcurrency,
|
||||
env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier, EVALS_PREFLIGHT_OK: '1' },
|
||||
env: {
|
||||
...process.env,
|
||||
EVALS: '1',
|
||||
EVALS_TIER: options.tier,
|
||||
EVALS_PREFLIGHT_OK: '1',
|
||||
// The parent's selection, computed once above — children's e2e-helpers
|
||||
// module load adopts it instead of re-deriving per shard (which spawned
|
||||
// a bun subprocess per child on the touchfiles-data map-diff path).
|
||||
// Children fall back to local derivation on any parse failure.
|
||||
EVALS_SELECTION_JSON: serializePaidDiffSelection(diffSelection),
|
||||
},
|
||||
evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(),
|
||||
});
|
||||
const skippedOutcomes: ShardOutcome[] = skipped.map((s, index) => ({
|
||||
@@ -686,8 +1101,12 @@ async function main(): Promise<number> {
|
||||
exitCode: null,
|
||||
elapsedMs: 0,
|
||||
groupPid: null,
|
||||
executedTests: null,
|
||||
}));
|
||||
const summary = summarize([...runSummary.outcomes, ...skippedOutcomes]);
|
||||
const guardedOutcomes = applyHollowShardGuard(runSummary.outcomes, {
|
||||
evalsAll: process.env.EVALS_ALL === '1',
|
||||
});
|
||||
const summary = summarize([...guardedOutcomes, ...skippedOutcomes]);
|
||||
for (const line of formatSummary(summary)) console.log(line);
|
||||
return summaryExitCode(summary);
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
* future strict wrapper around `bun test`.
|
||||
*/
|
||||
|
||||
import { type ChildProcess } from 'node:child_process';
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import { StringDecoder } from 'node:string_decoder';
|
||||
import * as path from 'node:path';
|
||||
|
||||
@@ -19,7 +19,7 @@ const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const ANSI_ESCAPE = /\u001B\[[0-?]*[ -/]*[@-~]/g;
|
||||
const BUN_FAIL_RESULT = /^\(fail\) .+ \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
|
||||
const BUN_BETWEEN_TESTS_ERROR = '# Unhandled error between tests';
|
||||
const BUN_TERMINAL_SUMMARY = /^Ran \d+ tests? across (\d+) files?\. \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
|
||||
const BUN_TERMINAL_SUMMARY = /^Ran (\d+) tests? across (\d+) files?\. \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
|
||||
|
||||
export type BunTestOutputFinding = 'failed-test' | 'unhandled-between-tests';
|
||||
|
||||
@@ -27,6 +27,8 @@ export interface BunTestOutputSummary {
|
||||
failedTests: number;
|
||||
unhandledBetweenTests: number;
|
||||
terminalFileCounts: number[];
|
||||
/** Test counts from the same terminal lines — feeds the hollow-shard guard. */
|
||||
terminalTestCounts: number[];
|
||||
}
|
||||
|
||||
export type ForwardedTerminationSignal = 'SIGINT' | 'SIGTERM';
|
||||
@@ -196,9 +198,15 @@ export function classifyBunTestOutputLine(rawLine: string): BunTestOutputFinding
|
||||
}
|
||||
|
||||
export function parseBunTerminalSummaryLine(rawLine: string): number | null {
|
||||
return parseBunTerminalSummary(rawLine)?.files ?? null;
|
||||
}
|
||||
|
||||
export function parseBunTerminalSummary(rawLine: string): { tests: number; files: number } | null {
|
||||
const line = stripAnsiLine(rawLine);
|
||||
const match = BUN_TERMINAL_SUMMARY.exec(line);
|
||||
return match ? Number.parseInt(match[1], 10) : null;
|
||||
return match
|
||||
? { tests: Number.parseInt(match[1], 10), files: Number.parseInt(match[2], 10) }
|
||||
: null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -220,6 +228,7 @@ export class BunTestOutputClassifier {
|
||||
private failedTests = 0;
|
||||
private unhandledBetweenTests = 0;
|
||||
private terminalFileCounts: number[] = [];
|
||||
private terminalTestCounts: number[] = [];
|
||||
|
||||
write(chunk: Uint8Array | string, origin: ClassifierOrigin = 'stdout'): void {
|
||||
this.pending[origin] += typeof chunk === 'string'
|
||||
@@ -242,6 +251,7 @@ export class BunTestOutputClassifier {
|
||||
failedTests: this.failedTests,
|
||||
unhandledBetweenTests: this.unhandledBetweenTests,
|
||||
terminalFileCounts: [...this.terminalFileCounts],
|
||||
terminalTestCounts: [...this.terminalTestCounts],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -258,8 +268,11 @@ export class BunTestOutputClassifier {
|
||||
const finding = classifyBunTestOutputLine(line);
|
||||
if (finding === 'failed-test') this.failedTests += 1;
|
||||
if (finding === 'unhandled-between-tests') this.unhandledBetweenTests += 1;
|
||||
const terminalFileCount = parseBunTerminalSummaryLine(line);
|
||||
if (terminalFileCount !== null) this.terminalFileCounts.push(terminalFileCount);
|
||||
const terminal = parseBunTerminalSummary(line);
|
||||
if (terminal !== null) {
|
||||
this.terminalFileCounts.push(terminal.files);
|
||||
this.terminalTestCounts.push(terminal.tests);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -298,3 +311,88 @@ export function forwardAndClassify(
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
// --- Shared shard-child lifecycle ---
|
||||
|
||||
export interface RunShardChildOptions {
|
||||
command: string;
|
||||
args: string[];
|
||||
cwd: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
/** External wall-clock deadline; on expiry the child's process GROUP is SIGKILLed. */
|
||||
timeoutMs: number;
|
||||
/**
|
||||
* Hook the freshly-spawned child's stdout/stderr. Stream POLICY (classifier
|
||||
* tees, log spooling, console forwarding, reporters) is entirely the
|
||||
* caller's. Runs synchronously right after spawn; the returned promises are
|
||||
* awaited AFTER the child closes, so trailing output is fully drained
|
||||
* before the caller reads its classifier/reporter state.
|
||||
*/
|
||||
hookStreams: (child: ChildProcess) => Array<Promise<void>>;
|
||||
}
|
||||
|
||||
export interface ShardChildResult {
|
||||
exitCode: number | null;
|
||||
/** True when the wall timer fired and SIGKILLed the group. */
|
||||
timedOut: boolean;
|
||||
/** The child's pid — the process-GROUP id on POSIX (detached spawn). */
|
||||
groupPid: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The child lifecycle both sharded runners need, extracted from
|
||||
* scripts/test-paid-shards.ts runPaidShard (scripts/test-free-shards.ts
|
||||
* runFreeShard duplicates the same ~35 lines verbatim today and is designed
|
||||
* to migrate here in a later change):
|
||||
*
|
||||
* - spawn detached on POSIX so the child owns its process group,
|
||||
* - forward parent SIGINT/SIGTERM to the whole group (not just the child),
|
||||
* - arm an EXTERNAL wall-clock timer that SIGKILLs the group — a spinning
|
||||
* child main thread never fires its own in-process timer,
|
||||
* - in EVERY exit path: disarm the timer, detach the signal forwarder, and
|
||||
* reap group survivors with SIGKILL.
|
||||
*
|
||||
* Caller-side cleanup that must run even on a spawn failure (log streams,
|
||||
* reporters, temp dirs) belongs in the caller's own try/finally around this
|
||||
* call: a spawn 'error' event THROWS from here after the finally block runs,
|
||||
* preserving the runners' existing could-not-run handling.
|
||||
*/
|
||||
export async function runShardChild(options: RunShardChildOptions): Promise<ShardChildResult> {
|
||||
const child = spawn(options.command, options.args, {
|
||||
cwd: options.cwd,
|
||||
env: options.env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
detached: process.platform !== 'win32',
|
||||
windowsHide: true,
|
||||
});
|
||||
const groupPid = child.pid ?? null;
|
||||
// Group-kill on parent SIGINT/SIGTERM too, not just on timeout.
|
||||
const forwarding = installChildSignalForwarding({
|
||||
kill: (signal?: NodeJS.Signals | number) => {
|
||||
killProcessGroup(child, (signal as NodeJS.Signals) ?? 'SIGTERM');
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
let timedOut = false;
|
||||
const killTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
killProcessGroup(child, 'SIGKILL');
|
||||
}, options.timeoutMs);
|
||||
|
||||
let exitCode: number | null = null;
|
||||
try {
|
||||
const streams = options.hookStreams(child);
|
||||
exitCode = await new Promise<number | null>((resolve, reject) => {
|
||||
child.once('error', reject);
|
||||
child.once('close', (code) => resolve(code));
|
||||
});
|
||||
await Promise.all(streams);
|
||||
} finally {
|
||||
clearTimeout(killTimer);
|
||||
forwarding.dispose();
|
||||
// Reap survivors of this shard even on the clean path.
|
||||
killProcessGroup(child, 'SIGKILL');
|
||||
}
|
||||
return { exitCode, timedOut, groupPid };
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user