initial: refactor web frontend from monorepo with WASM SQLite3 engine

src/lib/client.ts

@@ -0,0 +1,15 @@
+export async function fetchText(url: string | URL): Promise<string> {
+  const r = await fetch(url);
+  if (!r.ok) {
+    throw new Error(`Failed to fetch ${url}: ${r.status} ${r.statusText}`);
+  }
+  return r.text();
+}
+
+function splitLines(text: string): string[] {
+  return text.split(/\r?\n/).filter((l) => l.trim().length > 0);
+}
+
+export async function fetchLines(url: string | URL): Promise<string[]> {
+  return splitLines(await fetchText(url));
+}

src/lib/engine/index.ts

@@ -0,0 +1,36 @@
+import type { Engine } from "./types";
+import { WASMEngine } from "./wasm";
+import { KVEngine } from "./kv";
+
+let wasmSupported: boolean | null = null;
+
+async function checkWASMSupport(): Promise<boolean> {
+  if (wasmSupported !== null) return wasmSupported;
+
+  try {
+    if (typeof WebAssembly === "undefined") {
+      wasmSupported = false;
+      return false;
+    }
+
+    await WebAssembly.instantiate(Uint8Array.of(0x0, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00));
+
+    await import("@sqlite.org/sqlite-wasm");
+
+    wasmSupported = true;
+    return true;
+  } catch {
+    wasmSupported = false;
+    return false;
+  }
+}
+
+export async function createEngine(group: string): Promise<Engine> {
+  const supported = await checkWASMSupport();
+  if (supported) {
+    return new WASMEngine();
+  }
+  return new KVEngine(group);
+}
+
+export { KVEngine, WASMEngine };

src/lib/engine/kv.ts

@@ -0,0 +1,113 @@
+import type { Engine } from "./types";
+import type { OS } from "@/lib/types";
+import { addBasePath, dataBaseURL } from "@/lib/env";
+import { fetchText, fetchLines } from "@/lib/client";
+
+interface KVRecord {
+  key: string;
+  offset: number;
+  length: number;
+}
+
+class KVStore {
+  #index: Map<string, [number, number]> = new Map();
+  #blobsURL: string;
+
+  constructor(records: KVRecord[], blobsURL: string) {
+    for (const { key, offset, length } of records) {
+      if (this.#index.has(key)) {
+        throw new Error(`invalid data source, duplicate key: ${key}`);
+      }
+      this.#index.set(key, [offset, length]);
+    }
+    this.#blobsURL = blobsURL;
+  }
+
+  async get(key: string): Promise<string> {
+    const pair = this.#index.get(key);
+    if (!pair) {
+      throw new Error(`key not found: ${key}`);
+    }
+    const [offset, length] = pair;
+
+    return fetch(this.#blobsURL, {
+      headers: {
+        Range: `bytes=${offset}-${offset + length - 1}`,
+      },
+    }).then((r) => {
+      if (!r.ok) {
+        throw new Error(`failed to fetch blob for key: ${key}`);
+      }
+      return r.text();
+    });
+  }
+
+  *keys(): IterableIterator<string> {
+    yield* this.#index.keys();
+  }
+}
+
+export class KVEngine implements Engine {
+  #baseURL: string;
+
+  constructor(group: string) {
+    this.#baseURL = `${dataBaseURL()}/${group}`;
+  }
+
+  async listOS(): Promise<OS[]> {
+    const list = await fetchText(addBasePath(`${this.#baseURL}/list.json`));
+    return JSON.parse(list);
+  }
+
+  async getPaths(build: string): Promise<string[]> {
+    const os = await this.findOS(build);
+    const tag = `${os.version}_${build}`;
+    return fetchLines(addBasePath(`${this.#baseURL}/${tag}/paths.txt`));
+  }
+
+  async getBinaryXML(build: string, path: string): Promise<string> {
+    const os = await this.findOS(build);
+    const tag = `${os.version}_${build}`;
+    const reader = await this.openKV(`${this.#baseURL}/${tag}/blobs`);
+    const blob = await reader.get(path);
+
+    const location = blob.search(/<\/plist>\s*{/i);
+    if (location === -1) {
+      return blob;
+    }
+    return blob.substring(0, location + 8);
+  }
+
+  async getKeys(build: string): Promise<string[]> {
+    const os = await this.findOS(build);
+    const tag = `${os.version}_${build}`;
+    const reader = await this.openKV(`${this.#baseURL}/${tag}/keys`);
+    return [...reader.keys()];
+  }
+
+  async getPathsForKey(build: string, key: string): Promise<string[]> {
+    const os = await this.findOS(build);
+    const tag = `${os.version}_${build}`;
+    const reader = await this.openKV(`${this.#baseURL}/${tag}/keys`);
+    const lines = await reader.get(key);
+    return lines.split("\n").filter(Boolean);
+  }
+
+  #osCache: OS[] | null = null;
+
+  private async findOS(build: string): Promise<OS> {
+    if (!this.#osCache) {
+      this.#osCache = await this.listOS();
+    }
+    const os = this.#osCache.find((o) => o.build === build);
+    if (!os) throw new Error(`OS not found for build: ${build}`);
+    return os;
+  }
+
+  private async openKV(baseURL: string): Promise<KVStore> {
+    const recordsURL = baseURL + ".index.json";
+    const blobsURL = baseURL + ".txt";
+    const records = await fetch(recordsURL).then((r) => r.json());
+    return new KVStore(records, blobsURL);
+  }
+}

src/lib/engine/types.ts

@@ -0,0 +1,9 @@
+import type { OS } from "@/lib/types";
+
+export interface Engine {
+  listOS(): Promise<OS[]>;
+  getPaths(build: string): Promise<string[]>;
+  getBinaryXML(build: string, path: string): Promise<string>;
+  getKeys(build: string): Promise<string[]>;
+  getPathsForKey(build: string, key: string): Promise<string[]>;
+}

src/lib/engine/wasm.ts

@@ -0,0 +1,106 @@
+import type { Engine } from "./types";
+import type { OS } from "@/lib/types";
+import { dataBaseURL } from "@/lib/env";
+
+type SQLite3API = {
+  Database: new (data: ArrayLike<number | bigint>) => {
+    exec: (
+      sql: string,
+      bind?: unknown[]
+    ) => {
+      columns: string[];
+      rows: unknown[][];
+    }[];
+    close: () => void;
+  };
+};
+
+let sqlite3Module: SQLite3API | null = null;
+let dbInstance: InstanceType<SQLite3API["Database"]> | null = null;
+let dbReady = false;
+
+async function loadSQLite(): Promise<SQLite3API> {
+  if (sqlite3Module) return sqlite3Module;
+
+  const module = await import("@sqlite.org/sqlite-wasm");
+  sqlite3Module = module.default || module;
+  return sqlite3Module;
+}
+
+async function getDB(): Promise<InstanceType<SQLite3API["Database"]>> {
+  if (dbReady && dbInstance) return dbInstance;
+
+  const sqlite3 = await loadSQLite();
+
+  const response = await fetch(`${dataBaseURL()}/ent.db`);
+  const buffer = await response.arrayBuffer();
+
+  dbInstance = new sqlite3.Database(new Uint8Array(buffer));
+  dbReady = true;
+  return dbInstance;
+}
+
+export class WASMEngine implements Engine {
+  async listOS(): Promise<OS[]> {
+    const db = await getDB();
+    const results = db.exec(
+      "SELECT name, version, build, devices FROM os ORDER BY version DESC"
+    );
+    if (!results.length) return [];
+
+    const cols = results[0].columns;
+    const nameIdx = cols.indexOf("name");
+    const versionIdx = cols.indexOf("version");
+    const buildIdx = cols.indexOf("build");
+    const devicesIdx = cols.indexOf("devices");
+
+    return results[0].rows.map((row) => ({
+      name: row[nameIdx] as string,
+      version: row[versionIdx] as string,
+      build: row[buildIdx] as string,
+      devices: JSON.parse(row[devicesIdx] as string),
+    }));
+  }
+
+  async getPaths(build: string): Promise<string[]> {
+    const db = await getDB();
+    const results = db.exec(
+      `SELECT path FROM bin JOIN os ON bin.osid=os.id WHERE os.build=?`,
+      [build]
+    );
+    if (!results.length) return [];
+    return results[0].rows.map((row) => row[0] as string);
+  }
+
+  async getBinaryXML(build: string, path: string): Promise<string> {
+    const db = await getDB();
+    const results = db.exec(
+      `SELECT xml FROM bin JOIN os ON bin.osid=os.id WHERE os.build=? AND bin.path=?`,
+      [build, path]
+    );
+    if (!results.length || !results[0].rows.length) {
+      throw new Error(`Binary not found: ${path}`);
+    }
+    return results[0].rows[0][0] as string;
+  }
+
+  async getKeys(build: string): Promise<string[]> {
+    const db = await getDB();
+    const results = db.exec(
+      `SELECT DISTINCT key FROM pair JOIN bin ON pair.binid=bin.id JOIN os ON bin.osid=os.id WHERE os.build=?`,
+      [build]
+    );
+    if (!results.length) return [];
+    return results[0].rows.map((row) => row[0] as string);
+  }
+
+  async getPathsForKey(build: string, key: string): Promise<string[]> {
+    const db = await getDB();
+    const results = db.exec(
+      `SELECT path FROM bin JOIN pair ON bin.id=pair.binid JOIN os ON bin.osid=os.id WHERE os.build=? AND pair.key=?`,
+      [build, key]
+    );
+    if (!results.length) return [];
+    return results[0].rows.map((row) => row[0] as string);
+  }
+}

src/lib/env.ts

@@ -0,0 +1,11 @@
+export const basePath = process.env.NEXT_PUBLIC_BASE_PATH || "";
+
+export function addBasePath(path: string) {
+  let prefixed = path;
+  if (!prefixed.startsWith("/")) prefixed = `/${prefixed}`;
+  return basePath + prefixed;
+}
+
+export function dataBaseURL(): string {
+  return addBasePath("/data");
+}

src/lib/tree.ts

@@ -0,0 +1,63 @@
+interface SimpleTree {
+  [key: string]: SimpleTree;
+}
+
+function toTree(list: string[]): SimpleTree {
+  const root: SimpleTree = {};
+  for (const path of list) {
+    if (!path.startsWith("/")) {
+      continue;
+    }
+    const parts = path.split("/").slice(1);
+    let node = root;
+    for (const part of parts) {
+      if (!node[part]) {
+        node[part] = {};
+      }
+      node = node[part];
+    }
+  }
+  return root;
+}
+
+function shake(tree: SimpleTree) {
+  const result: SimpleTree = {};
+  for (const key in tree) {
+    const child = tree[key];
+    const shakenChild = shake(child);
+    const childKeys = Object.keys(shakenChild);
+    if (childKeys.length === 1) {
+      const grandChildKey = childKeys[0];
+      result[`${key}/${grandChildKey}`] = shakenChild[grandChildKey];
+    } else {
+      result[key] = shakenChild;
+    }
+  }
+  return result;
+}
+
+export interface TreeWithFullPath {
+  [key: string]: TreeWithFullPath | string;
+}
+
+function finalize(tree: SimpleTree, prefix = ""): TreeWithFullPath {
+  const keys = Object.keys(tree);
+  const result: TreeWithFullPath = {};
+  for (const key of keys) {
+    const path = prefix + "/" + key;
+    const child = tree[key];
+
+    result[key] =
+      Object.keys(child).length === 0
+        ? path
+        : (finalize(tree[key], path) as TreeWithFullPath);
+  }
+  return result;
+}
+
+export default function filesToTree(list: string[]): TreeWithFullPath {
+  const tree1 = toTree(list);
+  const tree2 = shake(tree1);
+  const tree3 = finalize(tree2);
+  return tree3;
+}

src/lib/types.ts

@@ -0,0 +1,11 @@
+export interface OS {
+  name: string;
+  build: string;
+  version: string;
+  devices: string[];
+}
+
+export interface Group {
+  name: string;
+  list: OS[];
+}

src/lib/utils.ts

@@ -0,0 +1,6 @@
+import { clsx, type ClassValue } from "clsx"
+import { twMerge } from "tailwind-merge"
+
+export function cn(...inputs: ClassValue[]) {
+  return twMerge(clsx(inputs))
+}