gstack/scripts/gen-qm-edition.ts

#!/usr/bin/env -S bun --no-env-file --config=/dev/null
/**
 * Generate the QM edition (hosts/qm.ts) into qm/sandbox/skills/<id>/ and qm/edition.json.
 *
 * Each skill is a QM-authored header plus sections cut by heading from its
 * gstack sources, so source edits flow in on the next run. Generation fails
 * closed: a kept heading, skipped heading or dropped block that no longer
 * exists, an unmapped {{RESOLVER}}, a file over the size limit, forbidden
 * text, or a command QM's sandbox can't run stops it.
 *
 *   bun run scripts/gen-qm-edition.ts           write the edition
 *   bun run scripts/gen-qm-edition.ts --check   exit 1 when the committed edition is stale
 */

import * as fs from 'node:fs';
import * as path from 'node:path';
import { QM_EDITION, QM_IMAGE_COMMANDS, QM_POSIX_COMMANDS, type QmPart, type QmSkill } from '../hosts/qm';

const ROOT = path.resolve(import.meta.dir, '..');
const HEADING = /^(#{1,6}) +(.+?) *$/;
const FENCE = /^ *(```|~~~)/;
const GENERATED = '<!-- Generated by scripts/gen-qm-edition.ts';
const SHELL_FENCE = /^ *```(sh|bash|shell|zsh)\b/;

interface Line { text: string; level: number; title: string }

/** Lines with their heading level; a # line inside a code fence is never a heading. */
function scanLines(text: string): Line[] {
  let open = false;
  return text.split('\n').map(line => {
    if (FENCE.test(line)) open = !open;
    const m = !open && !FENCE.test(line) ? HEADING.exec(line) : null;
    return { text: line, level: m ? m[1]!.length : 0, title: m ? m[2]! : '' };
  });
}

function sectionRange(lines: Line[], title: string, where: string): [number, number] {
  const hits = lines.flatMap((l, i) => (l.title === title ? [i] : []));
  if (hits.length !== 1) throw new Error(`${where}: heading "${title}" found ${hits.length} times (want 1)`);
  const start = hits[0]!;
  let end = start + 1;
  while (end < lines.length && !(lines[end]!.level && lines[end]!.level <= lines[start]!.level)) end++;
  return [start, end];
}

/** Blocks: a fenced block, a list item with its continuation, a heading, or a paragraph; blank lines kept as ''. */
function splitBlocks(text: string): string[] {
  const blocks: string[] = [];
  let current: string[] = [];
  let inFence = false;
  const flush = () => { if (current.length) blocks.push(current.join('\n')); current = []; };
  for (const line of text.split('\n')) {
    if (inFence) {
      current.push(line);
      if (FENCE.test(line)) { inFence = false; flush(); }
    } else if (FENCE.test(line)) {
      flush(); current.push(line); inFence = true;
    } else if (line.trim() === '') {
      flush(); blocks.push('');
    } else {
      if (/^ {0,3}([-*]|\d+\.) /.test(line) || HEADING.test(line)) flush();
      current.push(line);
    }
  }
  flush();
  const merged: string[] = [];
  for (const block of blocks) {
    const prev = merged.findLastIndex(b => b !== '');
    const continues = /^\s+\S/.test(block) && prev >= 0 && /^ {0,3}([-*]|\d+\.) |^\s+\S/.test(merged[prev]!);
    if (continues) merged.splice(prev, merged.length - prev, [merged[prev], ...merged.slice(prev + 1), block].join('\n'));
    else merged.push(block);
  }
  return merged;
}

function cutPart(part: QmPart, where: string): string {
  const at = `${where} ${part.file}`;
  const lines = scanLines(fs.readFileSync(path.join(ROOT, part.file), 'utf-8'));
  const [start, end] = sectionRange(lines, part.heading, at);
  const keep = lines.slice(start, end);
  for (const skip of part.skipHeadings ?? []) {
    const [s, e] = sectionRange(keep, skip, `${at} skipHeadings`);
    keep.splice(s, e - s);
  }
  const shift = part.level - keep[0]!.level;
  let text = keep.map(l => (l.level ? `${'#'.repeat(l.level + shift)} ${l.title}` : l.text)).join('\n');
  for (const needle of part.dropBlocks ?? []) {
    const blocks = splitBlocks(text);
    const kept = blocks.filter(b => !b.includes(needle));
    if (kept.length === blocks.length) throw new Error(`${at}: dropBlocks "${needle}" matched nothing`);
    text = kept.join('\n');
  }
  for (const [from, to] of part.replaceText ?? []) {
    if (!text.includes(from)) throw new Error(`${at}: replaceText "${from}" matched nothing`);
    text = text.split(from).join(to);
  }
  return text;
}

function applyResolvers(text: string, where: string): string {
  return text.replace(/\{\{([A-Z_]+)(:[^}]*)?\}\}/g, (match, name: string) => {
    if (!QM_EDITION.dropResolvers.includes(name)) {
      throw new Error(`${where}: unmapped placeholder ${match}; add it to QM_EDITION.dropResolvers or drop its block`);
    }
    return '';
  });
}

function tidy(text: string): string {
  return text.split('\n').map(l => l.replace(/[ \t]+$/, '')).join('\n').replace(/\n{3,}/g, '\n\n').trim() + '\n';
}

/** One skill's files, keyed by path inside its folder. */
export function renderSkill(skill: QmSkill): Map<string, string> {
  const where = `qm edition ${skill.id}`;
  let body = applyResolvers(skill.parts.map(p => cutPart(p, where)).join('\n\n'), where);
  for (const [from, to] of QM_EDITION.rewrites) body = body.split(from).join(to);
  const sources = [...new Set(skill.parts.map(p => p.file))].join(', ');
  const files = new Map<string, string>();
  files.set('SKILL.md', tidy([
    '---',
    `name: ${skill.id}`,
    `description: "${skill.description}"`,
    '---',
    `${GENERATED} from hosts/qm.ts and ${sources}. Do not edit. -->`,
    '',
    skill.header,
    '',
    body,
  ].join('\n')));
  for (const [name, src] of Object.entries(skill.scripts)) {
    files.set(`scripts/${name}`, fs.readFileSync(path.join(ROOT, QM_EDITION.scriptSourceDir, src), 'utf-8'));
  }
  return files;
}

const KEYWORDS = new Set(['if', 'then', 'else', 'elif', 'do', 'while', 'until', '!', '{', 'time']);
const CLOSERS = new Set(['fi', 'done', 'esac', '}']);
const NOT_QM = new Set(['bun', 'bunx', 'node', 'npm', 'npx', 'pnpm', 'yarn', 'python', 'python3', 'pip', 'jq', 'curl', 'gh', 'rg', 'claude', 'codex', 'make']);
const ALLOWED = new Set([...QM_POSIX_COMMANDS, ...QM_IMAGE_COMMANDS]);

/** The words of every simple command in a shell snippet: pipeline stages, list members and $( ) substitutions. */
export function commandWords(snippet: string): { functions: string[]; commands: string[][] } {
  const extra: string[] = [];
  const functions: string[] = [];
  let text = snippet.replace(/(^|\s)#.*$/gm, '$1').replace(/'[^']*'/g, "''").replace(/\d*>&\d*-?/g, ' ');
  text = text.replace(/"((?:[^"\\]|\\.)*)"/g, (_m, inner: string) => {
    for (const sub of inner.matchAll(/\$\(([^()]*)\)/g)) extra.push(sub[1]!);
    return '""';
  });
  text = text.replace(/^\s*([A-Za-z_]\w*)\s*\(\)/gm, (_m, name: string) => { functions.push(name); return ''; });
  const commands: string[][] = [];
  for (const segment of [text, ...extra].join('\n').split(/\|\||&&|\$\(|[|;&()`\n]/)) {
    const words = segment.trim().split(/\s+/).filter(Boolean);
    while (words.length && (KEYWORDS.has(words[0]!) || /^[A-Za-z_]\w*=/.test(words[0]!))) words.shift();
    if (!words.length || CLOSERS.has(words[0]!) || words[0] === 'for' || words[0] === 'case') continue;
    commands.push(words);
  }
  return { functions, commands };
}

/** Why the files of skill `id` name something QM's sandbox can't run or the edition doesn't ship; empty when clean. */
export function qmCommandViolations(id: string, files: Map<string, string>): string[] {
  const shipped = new Set([...files.keys()].filter(f => f.startsWith('scripts/')).map(f => f.slice('scripts/'.length)));
  const ids = new Set(QM_EDITION.skills.map(s => s.id));
  const problems: string[] = [];
  const check = (snippet: string, where: string) => {
    const { functions, commands } = commandWords(snippet);
    for (const [cmd, arg] of commands) {
      if (cmd === 'sh') {
        const script = (arg ?? '').replace(/^"|"$/g, '');
        const prefix = `skills/${id}/scripts/`;
        if (!script.startsWith(prefix) || !shipped.has(script.slice(prefix.length))) {
          problems.push(`${where}: sh runs "${script}", not a script shipped in ${prefix}`);
        }
      } else if (!ALLOWED.has(cmd!) && !functions.includes(cmd!)) {
        problems.push(`${where}: "${cmd}" is neither POSIX, an image binary (${QM_IMAGE_COMMANDS.join(', ')}) nor a shipped script`);
      }
    }
  };
  for (const [file, content] of files) {
    const where = `${id}/${file}`;
    const prose = content.split('\n').filter(l => !l.startsWith(GENERATED)).join('\n');
    for (const token of QM_EDITION.forbiddenText) {
      if (prose.includes(token)) problems.push(`${where}: names "${token}"`);
    }
    for (const m of prose.matchAll(/\bgstack-[a-z0-9-]+/g)) {
      if (!ids.has(m[0])) problems.push(`${where}: names "${m[0]}", a gstack command QM doesn't have`);
    }
    if (file.startsWith('scripts/')) { check(content, where); continue; }
    let fence: string[] | null = null;
    for (const line of content.split('\n')) {
      if (fence) {
        if (FENCE.test(line)) { check(fence.join('\n'), `${where} shell block`); fence = null; } else fence.push(line);
        continue;
      }
      if (SHELL_FENCE.test(line)) { fence = []; continue; }
      for (const span of line.matchAll(/`([^`]+)`/g)) {
        const first = span[1]!.trim().split(/\s+/)[0]!;
        if (ALLOWED.has(first) || NOT_QM.has(first)) check(span[1]!, `${where} inline`);
      }
    }
  }
  return problems;
}

/** Every file of the edition, keyed by path relative to the gstack root; throws when any rule fails. */
export function renderQmEdition(): Map<string, string> {
  const out = new Map<string, string>();
  const problems: string[] = [];
  const { fileChars, descriptionChars, indexChars } = QM_EDITION.limits;
  let index = 0;
  const manifest = QM_EDITION.skills.map(skill => {
    const files = renderSkill(skill);
    problems.push(...qmCommandViolations(skill.id, files));
    const d = skill.description;
    if (d.length > descriptionChars) problems.push(`${skill.id}: description is ${d.length} chars (limit ${descriptionChars})`);
    if (d.includes('"') || d.includes('\n')) problems.push(`${skill.id}: description must be one line without double quotes`);
    const skillMd = files.get('SKILL.md')!;
    const firstDerived = skillMd.indexOf(skill.parts[0]!.heading);
    const step = skillMd.indexOf(skill.firstStep.command);
    if (step < 0 || step > firstDerived) problems.push(`${skill.id}: first step "${skill.firstStep.command}" is not in the header`);
    if (!new RegExp(skill.firstStep.marker).test(skill.firstStep.command)) problems.push(`${skill.id}: marker does not match the first step`);
    index += skill.id.length + 3 + d.length;
    for (const [file, content] of files) {
      if (content.length > fileChars) problems.push(`${skill.id}/${file}: ${content.length} chars (limit ${fileChars})`);
      out.set(path.posix.join(QM_EDITION.outDir, skill.id, file), content);
    }
    return { id: skill.id, source: skill.source, files: [...files.keys()], first_step: skill.firstStep.command, first_step_marker: skill.firstStep.marker };
  });
  if (index > indexChars) problems.push(`skills index entries total ${index} chars (limit ${indexChars})`);
  if (problems.length) throw new Error(`QM edition is invalid:\n  ${problems.join('\n  ')}`);
  out.set(QM_EDITION.manifest, JSON.stringify({ edition: 'gstack-qm', layer_root: QM_EDITION.outDir, skills: manifest }, null, 2) + '\n');
  return out;
}

function onDisk(): string[] {
  const dir = path.join(ROOT, QM_EDITION.outDir);
  const files = fs.existsSync(dir)
    ? (fs.readdirSync(dir, { recursive: true }) as string[]).map(f => path.posix.join(QM_EDITION.outDir, f.split(path.sep).join('/')))
    : [];
  return [...files, QM_EDITION.manifest].filter(f => fs.existsSync(path.join(ROOT, f)) && fs.statSync(path.join(ROOT, f)).isFile());
}

/** Paths whose committed bytes differ from the generator's, or that only one side has. */
export function staleQmFiles(): string[] {
  const want = renderQmEdition();
  const stale = onDisk().filter(f => !want.has(f));
  for (const [f, content] of want) {
    const abs = path.join(ROOT, f);
    if (!fs.existsSync(abs) || fs.readFileSync(abs, 'utf-8') !== content) stale.push(f);
  }
  return stale.sort();
}

if (import.meta.main) {
  if (process.argv.includes('--check')) {
    const stale = staleQmFiles();
    if (stale.length) {
      console.error(`QM edition is stale (${stale.join(', ')}). Run: bun run scripts/gen-qm-edition.ts`);
      process.exit(1);
    }
    console.log('QM edition: fresh');
  } else {
    const want = renderQmEdition();
    for (const f of onDisk()) if (!want.has(f)) fs.rmSync(path.join(ROOT, f));
    for (const [f, content] of want) {
      fs.mkdirSync(path.dirname(path.join(ROOT, f)), { recursive: true });
      fs.writeFileSync(path.join(ROOT, f), content);
    }
    console.log(`QM edition: wrote ${want.size} files`);
  }
}