Container-by-container reads on a large imported doc (LoroMap.get / LoroList.get / LoroText.toJSON) retain ~4 KB wasm memory per container, scale superlinearly, and end in a bare RuntimeError: unreachable at 4 GiB #1092

/**
 * Standalone repro: `new Mirror({ doc, schema })` over a large imported doc.
 * Only loro-crdt + loro-mirror. Shape mirrors Lody's session doc: a root
 * `history` LoroList of turn LoroMaps; each turn has scalar fields and an
 * `items` LoroList of item LoroMaps; text items carry a `text` LoroText,
 * tool_call items carry nested maps/lists inferred under an `Any` catchall
 * with `defaultLoroText: true`.
 *
 *   tsx _repro-mirror-unreachable.ts --turns=570 --items=100 --mode=single
 *   modes: single | tojson | deep | repeat[:N] | bisect
 */
import { readFileSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { dirname, join } from 'node:path';
import { LoroDoc, LoroList, LoroMap, LoroText } from 'loro-crdt';
import { Mirror, schema } from 'loro-mirror';

const require = createRequire(import.meta.url);
const versionOf = (name: string): string => {
  let dir = dirname(require.resolve(name));
  for (let i = 0; i < 6; i += 1) {
    try {
      const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) as { name?: string; version?: string };
      if (pkg.name === name) return pkg.version ?? '?';
    } catch {
      // keep walking up
    }
    dir = dirname(dir);
  }
  return '?';
};

const flag = (name: string, fallback: string): string =>
  process.argv.find((a) => a.startsWith(`--${name}=`))?.slice(name.length + 3) ?? fallback;

const itemSchema = schema
  .LoroMap({ type: schema.String(), text: schema.LoroText({ required: false }) }, { required: true })
  .catchall(schema.Any({ defaultLoroText: true }));

const turnSchema = schema.LoroMap({
  id: schema.String(),
  role: schema.String(),
  timestamp: schema.String(),
  finished: schema.Boolean({ required: false }),
  endedAt: schema.Number({ required: false }),
  items: schema.LoroList(itemSchema, undefined, { required: false }),
  fileDiff: schema.Any(),
  inputConfig: schema.LoroMap({}, { required: false }).catchall(schema.Any()),
  modelInfo: schema.Any({ required: false }),
});

const docSchema = schema({
  session: schema.LoroMap({ id: schema.String() }),
  history: schema.LoroList(turnSchema, (t: { id: string }) => t.id),
});

const CHARS = Number(process.argv.find((a) => a.startsWith('--chars='))?.slice(8) ?? '180');
const TEXT = 'The quick brown fox jumps over the lazy dog. '.repeat(Math.max(1, Math.round(CHARS / 45)));

function makeTurn(index: number, itemsPerTurn: number) {
  const isUser = index % 2 === 0;
  const items: Record<string, unknown>[] = [];
  const count = isUser ? 1 : itemsPerTurn;
  for (let i = 0; i < count; i += 1) {
    if (isUser || i % 4 === 3) {
      items.push({ type: 'text', text: `${TEXT} turn ${index} item ${i}` });
    } else if (i % 4 === 0) {
      items.push({ type: 'thought', text: `thinking ${index}/${i}` });
    } else {
      items.push({
        type: 'tool_call',
        toolCallId: `tc-${index}-${i}`,
        title: 'Run command',
        kind: 'execute',
        status: 'completed',
        rawInput: { command: 'pnpm test', cwd: '/repo', args: ['--run'] },
        content: [
          { type: 'terminal_command', command: 'pnpm test', cwd: '/repo' },
          { type: 'terminal_output', output: TEXT.repeat(2) },
        ],
        locations: [{ path: `src/file-${i}.ts`, line: i }],
      });
    }
  }
  return {
    id: `turn-${index}`,
    role: isUser ? 'user' : 'assistant',
    timestamp: '2026-01-01T00:00:00.000Z',
    finished: true,
    endedAt: 1_700_000_000_000 + index,
    items,
    fileDiff: isUser ? [] : [{ filePath: `src/file-${index}.ts`, add: 3, del: 1 }],
    inputConfig: isUser ? { prompt: `prompt ${index}`, agentType: 'claude' } : undefined,
    modelInfo: isUser ? undefined : { modelId: 'model-x', name: 'Model X' },
  };
}

/**
 * Same shape written with loro-crdt container APIs only (what Lody's history
 * writer does): no Mirror in the builder, so the builder's memory stays small.
 */
function buildSnapshotDirect(turns: number, itemsPerTurn: number): Uint8Array {
  const doc = new LoroDoc();
  doc.getMap('session').set('id', 's');
  const history = doc.getList('history');
  const writeValue = (parent: LoroMap, key: string, value: unknown): void => {
    if (typeof value === 'string' && (key === 'text' || key === 'output' || key === 'command')) {
      const text = parent.setContainer(key, new LoroText());
      text.insert(0, value);
    } else if (Array.isArray(value)) {
      const list = parent.setContainer(key, new LoroList());
      for (const entry of value) {
        if (entry && typeof entry === 'object') {
          writeMap(list.pushContainer(new LoroMap()), entry as Record<string, unknown>);
        } else {
          list.push(entry as never);
        }
      }
    } else if (value && typeof value === 'object') {
      writeMap(parent.setContainer(key, new LoroMap()), value as Record<string, unknown>);
    } else if (value !== undefined) {
      parent.set(key, value as never);
    }
  };
  const writeMap = (map: LoroMap, value: Record<string, unknown>): void => {
    for (const [key, entry] of Object.entries(value)) writeValue(map, key, entry);
  };
  for (let i = 0; i < turns; i += 1) {
    writeMap(history.pushContainer(new LoroMap()), makeTurn(i, itemsPerTurn) as Record<string, unknown>);
    if (i % 50 === 49) doc.commit();
  }
  doc.commit();
  const snapshot = doc.export({ mode: 'snapshot' });
  doc.free();
  return snapshot;
}

function buildSnapshot(turns: number, itemsPerTurn: number): Uint8Array {
  const doc = new LoroDoc();
  const mirror = new Mirror({ doc, schema: docSchema, initialState: { session: { id: 's' }, history: [] } });
  const history = Array.from({ length: turns }, (_, i) => makeTurn(i, itemsPerTurn));
  mirror.setState((prev) => ({ ...prev, history: history as never }));
  doc.commit();
  const snapshot = doc.export({ mode: 'snapshot' });
  mirror.dispose();
  doc.free();
  return snapshot;
}

const countContainers = (doc: LoroDoc): number => {
  // 1 root list + per turn: map, items list, per item map + nested (approx via deep value walk)
  const walk = (v: unknown): number => {
    if (v && typeof v === 'object' && 'cid' in (v as object) && 'value' in (v as object)) {
      return 1 + walk((v as { value: unknown }).value);
    }
    if (Array.isArray(v)) return v.reduce((n: number, x) => n + walk(x), 0);
    if (v && typeof v === 'object') return Object.values(v as object).reduce((n: number, x) => n + walk(x), 0);
    return 0;
  };
  return walk(doc.getDeepValueWithID());
};

const mem = () => {
  const m = process.memoryUsage();
  return `rss=${(m.rss / 2 ** 20).toFixed(0)}MiB heap=${(m.heapUsed / 2 ** 20).toFixed(0)}MiB external=${(m.external / 2 ** 20).toFixed(0)}MiB arrayBuffers=${(m.arrayBuffers / 2 ** 20).toFixed(0)}MiB`;
};

const captured: string[] = [];
for (const level of ['error', 'warn'] as const) {
  const original = console[level].bind(console);
  console[level] = (...args: unknown[]) => {
    captured.push(args.map(String).join(' '));
    original(...args);
  };
}

function attempt(label: string, fn: () => unknown): boolean {
  const t = performance.now();
  try {
    fn();
    console.log(`  ${label}: ok in ${(performance.now() - t).toFixed(0)} ms  [${mem()}]`);
    return true;
  } catch (error) {
    const e = error as Error;
    console.log(`  ${label}: FAILED after ${(performance.now() - t).toFixed(0)} ms  [${mem()}]`);
    console.log(`    name=${e?.name} message=${e?.message}`);
    console.log(`    stack=\n${String(e?.stack).split('\n').slice(0, 14).join('\n')}`);
    if (captured.length) console.log(`    console before failure:\n${captured.slice(-5).join('\n')}`);
    return false;
  }
}

async function main() {
  const turns = Number(flag('turns', '570'));
  const items = Number(flag('items', '100'));
  const mode = flag('mode', 'single');
  console.log(`loro-crdt ${versionOf('loro-crdt')}, loro-mirror ${versionOf('loro-mirror')}, node ${process.version}`);
  console.log(`shape: ${turns} turns x ${items} items/assistant turn, mode=${mode}`);
  const file = flag('file', '');
  if (mode === 'gen' || mode === 'gen-direct') {
    const t0 = performance.now();
    const snapshot = mode === 'gen-direct' ? buildSnapshotDirect(turns, items) : buildSnapshot(turns, items);
    writeFileSync(file, snapshot);
    console.log(`snapshot: ${(snapshot.byteLength / 2 ** 20).toFixed(1)} MiB, built in ${(performance.now() - t0).toFixed(0)} ms, written to ${file}  [${mem()}]`);
    return;
  }
  const t0 = performance.now();
  const snapshot = file ? new Uint8Array(readFileSync(file)) : buildSnapshot(turns, items);
  console.log(`snapshot: ${(snapshot.byteLength / 2 ** 20).toFixed(1)} MiB, ${file ? 'read from ' + file : 'built'} in ${(performance.now() - t0).toFixed(0)} ms  [${mem()}]`);

  if (mode === 'single' || mode === 'tojson' || mode === 'deep') {
    const doc = new LoroDoc();
    attempt('import', () => doc.import(snapshot));
    if (mode === 'deep') {
      attempt('countContainers via getDeepValueWithID', () => console.log('    containers:', countContainers(doc)));
    }
    if (mode === 'tojson') attempt('doc.toJSON()', () => doc.toJSON());
    if (mode === 'deep') attempt('doc.getDeepValueWithID()', () => doc.getDeepValueWithID());
    if (mode === 'single') {
      attempt('new Mirror (full schema)', () => new Mirror({ doc, schema: docSchema, initialState: { session: { id: 's' }, history: [] } }));
    }
    return;
  }
  if (mode.startsWith('walk')) {
    // loro-mirror's init traversal without loro-mirror: keys()/get() per map,
    // get(i) per list, toJSON() per text. `walk-free` frees every container
    // handle right after use; `walk-gc` forces a V8 GC afterwards (needs
    // --expose-gc) to see whether wasm memory is only waiting on finalizers;
    // `walk-repeat:N` imports and walks N docs kept alive until the trap.
    const freeHandles = mode === 'walk-free';
    const repeats = mode.startsWith('walk-repeat') ? Number(mode.split(':')[1] ?? '6') : 1;
    const keep: LoroDoc[] = [];
    let handles = 0;
    const isContainer = (v: unknown): v is { kind(): string; free(): void } =>
      !!v && typeof v === 'object' && typeof (v as { kind?: unknown }).kind === 'function';
    const walk = (c: { kind(): string; free(): void }): unknown => {
      handles += 1;
      const kind = c.kind();
      let out: unknown;
      if (kind === 'Map') {
        const m = c as unknown as { keys(): string[]; get(k: string): unknown };
        const obj: Record<string, unknown> = {};
        for (const k of m.keys()) {
          const v = m.get(k);
          obj[k] = isContainer(v) ? walk(v) : v;
        }
        out = obj;
      } else if (kind === 'List' || kind === 'MovableList') {
        const l = c as unknown as { length: number; get(i: number): unknown };
        const arr: unknown[] = [];
        for (let i = 0; i < l.length; i += 1) {
          const v = l.get(i);
          arr.push(isContainer(v) ? walk(v) : v);
        }
        out = arr;
      } else {
        out = (c as unknown as { toJSON(): unknown }).toJSON();
      }
      if (freeHandles) c.free();
      return out;
    };
    for (let k = 1; k <= repeats; k += 1) {
      const doc = new LoroDoc();
      keep.push(doc);
      const ok = attempt(`#${k} import + walk like loro-mirror (${freeHandles ? 'freeing handles' : 'handles left to GC'})`, () => {
        doc.import(snapshot);
        walk(doc.getList('history') as never);
        console.log(`    handles created so far: ${handles}`);
      });
      if (!ok) return;
    }
    if (mode === 'walk-gc' && typeof globalThis.gc === 'function') {
      globalThis.gc();
      await new Promise((resolve) => setTimeout(resolve, 200));
      globalThis.gc();
      console.log(`  after global.gc(): [${mem()}]`);
    }
    return;
  }
  if (mode.startsWith('materialize')) {
    // Import + fully read (toJSON) N docs kept alive: wasm memory per
    // materialized doc, and where the wasm32 4 GiB limit trips.
    const n = Number(mode.split(':')[1] ?? '8');
    const keep: LoroDoc[] = [];
    let previous = process.memoryUsage().external;
    for (let k = 1; k <= n; k += 1) {
      const doc = new LoroDoc();
      const ok = attempt(`#${k} import + toJSON (docs kept alive)`, () => {
        doc.import(snapshot);
        doc.toJSON();
        keep.push(doc);
      });
      const now = process.memoryUsage().external;
      console.log(`    external delta for this doc: ${((now - previous) / 2 ** 20).toFixed(0)} MiB`);
      previous = now;
      if (!ok) return;
    }
    return;
  }
  if (mode.startsWith('free-reuse')) {
    // Same, but free each doc before the next: does freed wasm memory get reused?
    const n = Number(mode.split(':')[1] ?? '8');
    let previous = process.memoryUsage().external;
    for (let k = 1; k <= n; k += 1) {
      const doc = new LoroDoc();
      const ok = attempt(`#${k} import + toJSON + free()`, () => {
        doc.import(snapshot);
        doc.toJSON();
        doc.free();
      });
      const now = process.memoryUsage().external;
      console.log(`    external delta: ${((now - previous) / 2 ** 20).toFixed(0)} MiB`);
      previous = now;
      if (!ok) return;
    }
    return;
  }
  if (mode.startsWith('repeat-import')) {
    // Import the same snapshot into N docs kept alive: how much wasm linear
    // memory one doc costs and where the 4 GiB wasm32 limit trips.
    const n = Number(mode.split(':')[1] ?? '20');
    const keep: LoroDoc[] = [];
    let previous = process.memoryUsage().external;
    for (let k = 1; k <= n; k += 1) {
      const doc = new LoroDoc();
      const ok = attempt(`#${k} import (docs kept alive)`, () => {
        doc.import(snapshot);
        keep.push(doc);
      });
      const now = process.memoryUsage().external;
      console.log(`    external delta for this doc: ${((now - previous) / 2 ** 20).toFixed(0)} MiB`);
      previous = now;
      if (!ok) return;
    }
    return;
  }
  if (mode.startsWith('repeat')) {
    const n = Number(mode.split(':')[1] ?? '8');
    const keep: unknown[] = [];
    for (let k = 1; k <= n; k += 1) {
      const doc = new LoroDoc();
      const ok = attempt(`#${k} import + new Mirror (docs kept alive)`, () => {
        doc.import(snapshot);
        keep.push(new Mirror({ doc, schema: docSchema, initialState: { session: { id: 's' }, history: [] } }));
        keep.push(doc);
      });
      if (!ok) return;
    }
    return;
  }
  if (mode === 'bisect') {
    // grow turns until new Mirror on a single fresh doc fails
    let lo = 0; let hi = turns;
    const test = (t: number) => {
      const snap = buildSnapshot(t, items);
      const doc = new LoroDoc();
      doc.import(snap);
      const ok = attempt(`turns=${t} new Mirror`, () => new Mirror({ doc, schema: docSchema, initialState: { session: { id: 's' }, history: [] } }));
      doc.free();
      return ok;
    };
    if (test(hi)) { console.log('no failure at max'); return; }
    while (hi - lo > Math.max(1, Math.floor(hi / 20))) {
      const mid = Math.floor((lo + hi) / 2);
      if (test(mid)) lo = mid; else hi = mid;
    }
    console.log(`threshold between ${lo} (ok) and ${hi} (fails) turns x ${items} items`);
  }
}

void main();