feat: QA restructure, browser ref staleness, eval efficiency metrics (v0.4.0) (#83)

* feat: browser ref staleness detection via async count() validation

resolveRef() now checks element count to detect stale refs after page
mutations (e.g. SPA navigation). RefEntry stores role+name metadata
for better diagnostics. 3 new snapshot tests for staleness detection.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: qa-only skill, qa fix loop, plan-to-QA artifact flow

Add /qa-only (report-only, Edit tool blocked), restructure /qa with
find-fix-verify cycle, add {{QA_METHODOLOGY}} DRY placeholder for
shared methodology. /plan-eng-review now writes test-plan artifacts
to ~/.gstack/projects/<slug>/ for QA consumption.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* feat: eval efficiency metrics — turns, duration, commentary across all surfaces

Add generateCommentary() for natural-language delta interpretation,
per-test turns/duration in comparison and summary output, judgePassed
unit tests, 3 new E2E tests (qa-only, qa fix loop, plan artifact).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* chore: bump version and changelog (v0.4.0)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* docs: update ARCHITECTURE, BROWSER, CONTRIBUTING, README for v0.4.0

- ARCHITECTURE: add ref staleness detection section, update RefEntry type
- BROWSER: add ref staleness paragraph to snapshot system docs
- CONTRIBUTING: update eval tool descriptions with commentary feature
- README: fix missing qa-only in project-local uninstall command

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* docs: add user-facing benefit descriptions to v0.4.0 changelog

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-15 23:55:39 -05:00
committed by GitHub
co-authored by Claude Opus 4.6
parent bb46ca6b21
commit f3ee0ee28a
30 changed files with 2317 additions and 354 deletions
+20 -7
View File
@@ -18,6 +18,12 @@
import { chromium, type Browser, type BrowserContext, type Page, type Locator } from 'playwright';
import { addConsoleEntry, addNetworkEntry, addDialogEntry, networkBuffer, type DialogEntry } from './buffers';
export interface RefEntry {
locator: Locator;
role: string;
name: string;
}
export class BrowserManager {
private browser: Browser | null = null;
private context: BrowserContext | null = null;
@@ -31,7 +37,7 @@ export class BrowserManager {
public serverPort: number = 0;
// ─── Ref Map (snapshot → @e1, @e2, @c1, @c2, ...) ────────
private refMap: Map<string, Locator> = new Map();
private refMap: Map<string, RefEntry> = new Map();
// ─── Snapshot Diffing ─────────────────────────────────────
// NOT cleared on navigation — it's a text baseline for diffing
@@ -169,7 +175,7 @@ export class BrowserManager {
}
// ─── Ref Map ──────────────────────────────────────────────
setRefMap(refs: Map<string, Locator>) {
setRefMap(refs: Map<string, RefEntry>) {
this.refMap = refs;
}
@@ -181,16 +187,23 @@ export class BrowserManager {
* Resolve a selector that may be a @ref (e.g., "@e3", "@c1") or a CSS selector.
* Returns { locator } for refs or { selector } for CSS selectors.
*/
resolveRef(selector: string): { locator: Locator } | { selector: string } {
async resolveRef(selector: string): Promise<{ locator: Locator } | { selector: string }> {
if (selector.startsWith('@e') || selector.startsWith('@c')) {
const ref = selector.slice(1); // "e3" or "c1"
const locator = this.refMap.get(ref);
if (!locator) {
const entry = this.refMap.get(ref);
if (!entry) {
throw new Error(
`Ref ${selector} not found. Page may have changed — run 'snapshot' to get fresh refs.`
`Ref ${selector} not found. Run 'snapshot' to get fresh refs.`
);
}
return { locator };
const count = await entry.locator.count();
if (count === 0) {
throw new Error(
`Ref ${selector} (${entry.role} "${entry.name}") is stale — element no longer exists. ` +
`Run 'snapshot' for fresh refs.`
);
}
return { locator: entry.locator };
}
return { selector };
}