mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-01 22:45:02 +01:00
test: follow the review-bot rename in the remote free tier's three reds
The cross-skill consistency judge describes review-bot-history.md and its record helper; the tracker-guard exemption for the removed raw Greptile fetch in review/greptile-triage.md is dropped (the fetch now lives in gstack-review-bots, whose bodies still reach context only through gstack-issue-guard); the Step 10 PR-check test expects 'Review bots: not run'. HARNESS_VERSION stays 6: skill-fixture.ts and llm-judge.ts are recorded non-behavioral for every other case. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
b9777a1d9b
commit
4f05f56c00
4 files changed
+7
-13
No files matched your search
@@ -29,7 +29,7 @@
|
||||
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
|
||||
"test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
|
||||
"test/helpers/llm-judge.ts": {"blob":"095f9c85dd0ae29da8ba10fed91510313a4442a4","since":6,"nonBehavioral":"callJudge emits one llm-judge-aborted diagnostic line (wrapper attempts, HTTP request and status counts seen through a pass-through fetch, 429 waits, connect and event timings, block kind, character counts, message and request ids, a capped reason; no content) when its caller's signal aborts it; observers swallow their own errors, and the request, retries, parsing, scores and the thrown abort reason are unchanged, so no case's pass or fail changes"},
|
||||
"test/helpers/llm-judge.ts": {"blob":"f14f3ffc1fed5a0311158f26ca8bbfef6593a8bf","since":6,"nonBehavioral":"GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading ('Check for review bot comments') so the review E2E cases extract the same section, now the skill's own reworded bytes; llm-judge.ts updates only the cross-skill consistency judge's intended architecture to the renamed review-bot-history.md and its record helper. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own new series"},
|
||||
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
|
||||
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
|
||||
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
|
||||
@@ -48,7 +48,7 @@
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
|
||||
"test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"8a654dae6b0a396d839b02179119ae3e3bb13f69","since":6,"nonBehavioral":"extractSkillBody and extractSkillSections follow the shared-preamble hoist (pointer plus sections/preamble.md); every fixture they build is byte-identical to the pre-hoist output (compared for 8 body and 4 section extractions), so no case's input or verdict changes; the /ship review then reworded only comments and two error messages, so no fixture byte changes"},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"ce7609eacc7a8265a4a211358d1b4fb49ea85bc5","since":6,"nonBehavioral":"GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading ('Check for review bot comments') so the review E2E cases extract the same section, now the skill's own reworded bytes; llm-judge.ts updates only the cross-skill consistency judge's intended architecture to the renamed review-bot-history.md and its record helper. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own new series"},
|
||||
"test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"},
|
||||
"test/helpers/test-selection.ts": {"blob":"e1b54ad5e3d3eadd8027de1b44e57554d6d99a41","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"},
|
||||
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6}
|
||||
|
||||
@@ -741,11 +741,11 @@ export function buildCrossSkillConsistencyJudgePrompt(collected: string): string
|
||||
return `You are evaluating whether multiple skill configuration files implement the same data architecture consistently.
|
||||
|
||||
INTENDED ARCHITECTURE:
|
||||
- greptile-history has TWO paths: per-project (~/.gstack/projects/{slug}/greptile-history.md) and global (~/.gstack/greptile-history.md)
|
||||
- review-bot-history.md has TWO paths: per-project (~/.gstack/projects/{slug}/review-bot-history.md) and global (~/.gstack/review-bot-history.md); each row names its bot, and rows in the older greptile-history.md files count as Greptile's
|
||||
- /review and /ship WRITE to BOTH paths (per-project for suppressions, global for retro aggregation)
|
||||
- /review and /ship delegate write mechanics to greptile-triage.md
|
||||
- /retro READS from the GLOBAL path only (it aggregates across all projects)
|
||||
- REMOTE_SLUG derivation should be consistent across files that use it
|
||||
- /review and /ship delegate write mechanics to greptile-triage.md (its History File Writes, through the gstack-review-bots record helper)
|
||||
- /retro READS the GLOBAL history only (it aggregates across all projects, per bot)
|
||||
- Greptile-only behavior (escalation tiers, severity re-rank) is labeled Greptile-only wherever it appears
|
||||
|
||||
Below are greptile-related lines extracted from each skill file:
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ describe('/ship Step 10 Greptile PR check', () => {
|
||||
expect(dispatchIdx).toBeGreaterThan(checkIdx);
|
||||
const between = SECTION.slice(checkIdx, dispatchIdx);
|
||||
expect(between).toContain('Only `PR: exists` dispatches.');
|
||||
expect(between).toContain('"Greptile: not run (<reason>); runs on the PR once it exists"');
|
||||
expect(between).toContain('"Review bots: not run (<reason>); runs on the PR once it exists"');
|
||||
expect(SECTION).not.toContain('PR: unknown');
|
||||
});
|
||||
|
||||
|
||||
@@ -47,12 +47,6 @@ const SCANNER_EXEMPT: { file: string; pattern: string; reason: string }[] = [
|
||||
reason:
|
||||
'F1 plan binding: only the first `Plan: <path>` token survives (one whitespace-free path, CR and backticks stripped); the body never reaches model context, and the path is then read as a file the parent confirms',
|
||||
},
|
||||
{
|
||||
file: 'review/greptile-triage.md',
|
||||
pattern: 'gh comment-body api read',
|
||||
reason:
|
||||
'raw fetch lands in /tmp json FILES (metadata/body split); body text is read into context only via the gstack-issue-guard --stdin pipes documented in the same file',
|
||||
},
|
||||
{
|
||||
file: 'document-release/sections/release-body.md.tmpl',
|
||||
pattern: 'gh pr body read',
|
||||
|
||||
Reference in new issue
Block a user