mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 14:14:37 +01:00
gstack's router no longer gives bug reports a special check-the-cause, fix-it-test-first rule, and its description no longer lists debugging, so the model fixes a bug the way it would without gstack. This keeps gstack to the parts that measurably help, and on 26 hard real bugs the rule gave no clear gain. Nothing to do. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> Review-Finding: p1/red-team#1 reader=red-team
127 lines
5.9 KiB
TypeScript
127 lines
5.9 KiB
TypeScript
/**
|
|
* Agent host selection for paid E2E cases (GSTA-49 TM2). EVALS_HOST picks the
|
|
* agent every runSkillTest session runs on: `claude` (default) or `codex`
|
|
* (codex exec, test/helpers/codex-host-runner.ts). E2E_HOSTS is the per-case
|
|
* host metadata over every gate case (classification in the GSTA-49 TM2
|
|
* scoping): `both` runs on either host with the shared adapter; every other
|
|
* value is the reason the case is Claude-only. On the Codex host a Claude-only
|
|
* case is SKIPPED with its reason, planned as excluded and deselected by the
|
|
* report with zero credit, never counted as failed. A case without metadata
|
|
* (every non-gate case) is Claude-only.
|
|
*/
|
|
|
|
export type EvalHost = 'claude' | 'codex';
|
|
export const EVAL_HOSTS: readonly EvalHost[] = ['claude', 'codex'];
|
|
|
|
export function resolveEvalHost(env: NodeJS.ProcessEnv = process.env): EvalHost {
|
|
const raw = (env.EVALS_HOST ?? '').trim();
|
|
if (!raw) return 'claude';
|
|
if ((EVAL_HOSTS as readonly string[]).includes(raw)) return raw as EvalHost;
|
|
throw new Error(`EVALS_HOST must be one of ${EVAL_HOSTS.join(', ')}; got "${raw}"`);
|
|
}
|
|
|
|
export const CLAUDE_ONLY_REASONS = {
|
|
pty: 'drives the interactive Claude Code TUI (plan mode, native AskUserQuestion) through the PTY harness',
|
|
sdk: 'uses the Claude Agent SDK (canUseTool answers and PreToolUse hooks)',
|
|
subagents: 'asserts Claude Agent/Task subagent events',
|
|
'skill-tool': 'depends on the Claude Skill tool load and registration',
|
|
hermetic: 'a claude -p hermetic canary; the Codex host needs its own canary twin',
|
|
'self-host': 'the Claude host calling /codex; on the Codex host /codex is the agent itself',
|
|
rework: 'parses Claude stream-json tool events or prescribes native Claude tools; not ported to the Codex host',
|
|
'host-neutral': 'runs no model session, so a second host adds nothing',
|
|
} as const;
|
|
|
|
export type ClaudeOnlyReason = keyof typeof CLAUDE_ONLY_REASONS;
|
|
export type HostSupport = 'both' | ClaudeOnlyReason;
|
|
|
|
/** Adapter-only cases: final output, files, git state, Bash commands, server logs or browseErrors. */
|
|
const ADAPTER_ONLY: readonly string[] = [
|
|
'benchmark-workflow',
|
|
'browse-basic',
|
|
'browse-snapshot',
|
|
'canary-workflow',
|
|
'context-recovery-artifacts',
|
|
'context-restore-loads-latest',
|
|
'context-save-writes-file',
|
|
'cso-diff-mode',
|
|
'design-review-detector-shim',
|
|
'design-review-detector-shim-dom',
|
|
'design-review-plugin-handoff',
|
|
'diagram-triplet',
|
|
'document-release',
|
|
'gstack-upgrade-happy-path',
|
|
'land-and-deploy-first-run',
|
|
'land-and-deploy-review-gate',
|
|
'land-and-deploy-workflow',
|
|
'learnings-show',
|
|
'operational-learning',
|
|
'plan-design-review-no-ui-scope',
|
|
'plan-eng-coverage-audit',
|
|
'plan-review-report',
|
|
'plan-tune-inspect',
|
|
'qa-bootstrap',
|
|
'qa-quick',
|
|
'retro-base-branch',
|
|
'review-army-delivery-audit',
|
|
'review-army-json-findings',
|
|
'review-army-migration-safety',
|
|
'review-army-quality-score',
|
|
'review-base-branch',
|
|
'review-coverage-audit',
|
|
'review-enum-completeness',
|
|
'review-sql-injection',
|
|
'review-test-value',
|
|
'safety-pair-agent-block',
|
|
'safety-ship-stale-evidence',
|
|
'setup-deploy-workflow',
|
|
'ship-base-branch',
|
|
'ship-coverage-audit',
|
|
'ship-coverage-value',
|
|
'ship-local-workflow',
|
|
'ship-triage',
|
|
'skillmd-no-local-binary',
|
|
'skillmd-outside-git',
|
|
'skillmd-setup-discovery',
|
|
'test-audit-report-only',
|
|
'tpa-absent-darwin',
|
|
'tpa-absent-linux',
|
|
'tpa-apple-ban',
|
|
'tpa-broken',
|
|
'tpa-present'
|
|
];
|
|
|
|
const CLAUDE_ONLY: Readonly<Record<ClaudeOnlyReason, readonly string[]>> = {
|
|
pty: ['autoplan-guard-pty', 'office-hours-auto-mode', 'plan-ceo-finding-floor', 'plan-ceo-review-plan-mode',
|
|
'plan-design-with-ui-scope', 'plan-devex-finding-floor', 'plan-devex-review-plan-mode', 'plan-mode-no-op'],
|
|
sdk: ['auq-format-gate',
|
|
'shared-libs-review-index-flags', 'shared-libs-review-lifecycle', 'shared-libs-review-path-eligibility',
|
|
'shared-libs-review-prior-coverage', 'shared-libs-review-revalidation', 'ship-frozen-scope-convergence', 'ship-local-hook-preservation',
|
|
'ship-managed-hook-refresh', 'ship-skipped-queued-finding', 'ship-unmanaged-hook-consent'],
|
|
subagents: ['docsync-spawned', 'review-army-perf-n-plus-one'],
|
|
'skill-tool': ['skillify-approval-reject', 'skillify-happy-path', 'skillify-provenance-refusal'],
|
|
hermetic: ['hermetic-canary', 'hermetic-sentinel'],
|
|
'self-host': ['safety-codex-consult-embed'],
|
|
rework: ['qa-functional-cli-fix', 'qa-functional-cli-report', 'qa-functional-webhook-fix', 'qa-functional-webhook-report',
|
|
'qa-only-no-fix', 'review-dashboard-via', 'review-exploratory-small-cli', 'review-gate-integrity',
|
|
'shared-libs-read-only', 'shared-libs-unsupported-git', 'ship-docsync-completion', 'ship-docsync-current',
|
|
'ship-docsync-failure', 'ship-docsync-late-result', 'ship-docsync-launch-failure', 'ship-docsync-missing-asset',
|
|
'ship-docsync-missing-marker', 'ship-docsync-recovery', 'ship-docsync-stale-after', 'ship-docsync-stale-before',
|
|
'ship-docsync-store', 'ship-docsync-timeout-unsettled', 'ship-exploratory-late-input', 'ship-exploratory-plan-checks',
|
|
'ship-exploratory-small-cli', 'ship-exploratory-unavailable'],
|
|
'host-neutral': ['timeline-event-flow'],
|
|
};
|
|
|
|
export const E2E_HOSTS: Readonly<Record<string, HostSupport>> = Object.freeze(Object.fromEntries([
|
|
...ADAPTER_ONLY.map(id => [id, 'both'] as const),
|
|
...(Object.entries(CLAUDE_ONLY) as Array<[ClaudeOnlyReason, readonly string[]]>).flatMap(([reason, ids]) => ids.map(id => [id, reason] as const)),
|
|
]));
|
|
|
|
/** Why a case does not run on `host`, or null when it does. */
|
|
export function hostSkipReason(id: string, host: EvalHost, hosts: Readonly<Record<string, HostSupport>> = E2E_HOSTS): string | null {
|
|
if (host === 'claude') return null;
|
|
const support = hosts[id];
|
|
if (support === 'both') return null;
|
|
if (support === undefined) return `Claude-only on the ${host} host: no Codex host metadata (E2E_HOSTS covers the gate cases)`;
|
|
return `Claude-only on the ${host} host: ${CLAUDE_ONLY_REASONS[support]}`;
|
|
}
|