open-brainy/tests/integration/history-repacking.test.ts

/**
 * @module tests/integration/history-repacking
 * @description The D1+D3 two-tier history lifecycle end-to-end on a real
 * brain. Laws: (1) repacking is RE-REPRESENTATION — after folding, every
 * asOf() read below the fold boundary answers exactly as before, across a
 * cold reopen; (2) folded per-generation directories are physically gone
 * (the file-count cure is real, not cosmetic); (3) repack + reclaim compose:
 * bounded retention after repacking drops whole segments and asOf below the
 * horizon throws GenerationCompactedError; (4) repackHistory is explicit
 * API and time-bounded (spent budget = consistent no-op).
 *
 * Uses a tiny REPACK_LIVE_WINDOW override so a small history has a cold
 * tier at all (the production window is 1024).
 */
import { describe, it, expect, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as path from 'node:path'
import * as os from 'node:os'
import * as zlib from 'node:zlib'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import { GenerationStore } from '../../src/db/generationStore.js'
import { GenerationCompactedError } from '../../src/db/errors.js'
import { SEGMENTS_PREFIX } from '../../src/db/generationSegments.js'

const stub = async (text: string): Promise<number[]> => {
  const h = text.split('').reduce((a, c) => a + c.charCodeAt(0), 0)
  return new Array(384).fill(0).map((_, i) => Math.sin(h + i))
}

const openBrain = async (dir: string): Promise<Brainy> => {
  const brain = new Brainy({
    requireSubtype: false,
    storage: { type: 'filesystem', path: dir },
    embeddingFunction: stub
  })
  await brain.init()
  return brain
}

describe('history repacking — the two-tier lifecycle', () => {
  const dirs: string[] = []
  const tempDir = (): string => {
    const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-repack-'))
    dirs.push(d)
    return d
  }
  const originalWindow = GenerationStore.REPACK_LIVE_WINDOW

  afterEach(() => {
    ;(GenerationStore as any).REPACK_LIVE_WINDOW = originalWindow
    for (const d of dirs.splice(0)) {
      try {
        fs.rmSync(d, { recursive: true, force: true })
      } catch {
        /* best effort */
      }
    }
  })

  /**
   * THE HOLE, END TO END — the shape a real store carries.
   *
   * A forensic fixture was measured with generation directories 1..2503
   * present except for exactly one: 1416. Its fact-log segment already showed
   * the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration
   * 1940 (531 generations) while recording only 530 facts.
   *
   * Before the fix, repacking such a store folded ACROSS that hole: the batch
   * skipped 1416 (no readable delta) and the sealed segment declared a range
   * spanning it anyway. The next open merged that declared range back into
   * committedRanges, re-admitting 1416 as committed history, and every
   * subsequent auto-compaction pass then asked the packed tier for a frame
   * that was never written — producing, on EVERY run, the non-fatal narration
   *
   *   Auto-compaction of generational history failed (non-fatal): generation
   *   N is inside sealed segment seg-....bgs's declared range but has no frame
   *   — packed history is damaged
   *
   * This pin removes a generation directory to make the same hole, then
   * requires repack + reopen + compaction to complete cleanly.
   */
  it('a missing generation directory does not poison the packed tier', async () => {
    const dir = tempDir()
    // `retention: 'all'` throughout: close() otherwise auto-compacts the
    // history away, and this pin needs the cold generations still on disk so
    // there is something to punch a hole in. The live window stays at its
    // production default for the build phase, so nothing folds yet.
    const archival = async (): Promise<Brainy> => {
      const b = new Brainy({
        requireSubtype: false,
        storage: { type: 'filesystem', path: dir },
        embeddingFunction: stub,
        retention: 'all'
      })
      await b.init()
      return b
    }
    const brain = await archival()

    const id = await brain.add({
      data: 'holed-entity',
      type: NounType.Document,
      metadata: { v: 0 }
    })
    // One flush per update: single-op writes coalesce inside a flush window,
    // so a history deep enough to have a middle needs the windows separated.
    for (let v = 1; v <= 12; v++) {
      await brain.update({ id, metadata: { v } })
      await brain.flush()
    }
    await brain.close()

    // Punch the hole: delete ONE generation directory in the middle of the
    // cold range, exactly as the real store presents it.
    const genRoot = path.join(dir, '_generations')
    const numeric = fs
      .readdirSync(genRoot, { withFileTypes: true })
      .filter((e) => e.isDirectory() && /^\d+$/.test(e.name))
      .map((e) => Number(e.name))
      .sort((a, b) => a - b)
    expect(numeric.length).toBeGreaterThan(6)
    const victim = numeric[Math.floor(numeric.length / 2)]
    fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true })

    // Now shrink the live window and reopen. close() repacks automatically
    // (brainy.ts phase 0b), so this is the production sequence exactly: a
    // store with a hole in its history gets folded by ordinary housekeeping,
    // with nobody asking for it.
    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
    const reopened = await archival()
    const result = await reopened.repackHistory()
    expect(result.foldedGenerations).toBeGreaterThan(0)

    const segDir = path.join(dir, SEGMENTS_PREFIX)
    const manifestPath = ['manifest.json', 'manifest.json.gz']
      .map((f) => path.join(segDir, f))
      .find((p) => fs.existsSync(p))!
    const raw = manifestPath.endsWith('.gz')
      ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8')
      : fs.readFileSync(manifestPath, 'utf8')
    const manifest = JSON.parse(raw) as {
      segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }>
    }

    // THE LAW: every sealed segment declares exactly as many generations as it
    // holds frames, and none of them spans the victim.
    for (const s of manifest.segments) {
      expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames)
      expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false)
    }

    await reopened.close()

    // And the pass that used to fail on every run now completes: reopen (which
    // re-seeds committedRanges from the packed tier) then compact history.
    const third = await openBrain(dir)
    await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined()
    await third.close()
  })

  it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => {
    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
    const dir = tempDir()
    const brain = await openBrain(dir)

    const id = await brain.add({
      data: 'versioned-entity',
      type: NounType.Document,
      metadata: { v: 0 }
    })
    for (let v = 1; v <= 10; v++) await brain.update({ id, metadata: { v } })
    await brain.flush()

    // Ground truth BEFORE repacking: capture asOf views for early generations.
    const before: Record<number, number> = {}
    for (const g of [2, 4, 6]) {
      const db = await brain.asOf(g)
      before[g] = (await db.get(id))?.metadata?.v as number
      await db.release()
    }

    const result = await brain.repackHistory()
    expect(result.foldedGenerations).toBeGreaterThan(0)
    expect(result.segmentsCreated).toBeGreaterThan(0)

    // The folded per-generation directories are PHYSICALLY gone…
    const genDirs = fs
      .readdirSync(path.join(dir, '_generations'), { withFileTypes: true })
      .filter((e) => e.isDirectory() && /^\d+$/.test(e.name)).length
    expect(genDirs).toBeLessThanOrEqual(4) // live window (3) + at most the newest
    // …and the segment tier exists (the filesystem adapter stores objects
    // gzipped, so the manifest may live at either spelling).
    const segDir = path.join(dir, SEGMENTS_PREFIX)
    expect(
      fs.existsSync(path.join(segDir, 'manifest.json')) ||
        fs.existsSync(path.join(segDir, 'manifest.json.gz'))
    ).toBe(true)
    expect(fs.readdirSync(segDir).some((f) => f.endsWith('.bgs'))).toBe(true)

    // Same asOf answers from the packed tier, same process…
    for (const g of [2, 4, 6]) {
      const db = await brain.asOf(g)
      expect((await db.get(id))?.metadata?.v).toBe(before[g])
      await db.release()
    }
    await brain.close()

    // …and across a COLD REOPEN (manifest discovery, no live dirs to list).
    const reopened = await openBrain(dir)
    for (const g of [2, 4, 6]) {
      const db = await reopened.asOf(g)
      expect((await db.get(id))?.metadata?.v).toBe(before[g])
      await db.release()
    }
    expect((await reopened.get(id))?.metadata?.v).toBe(10) // live state untouched
    await reopened.close()
  })

  it('repack + bounded reclaim compose: whole segments drop, horizon is loud', async () => {
    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
    const dir = tempDir()
    const brain = await openBrain(dir)
    const id = await brain.add({ data: 'reclaim-probe', type: NounType.Document, metadata: { v: 0 } })
    for (let v = 1; v <= 8; v++) await brain.update({ id, metadata: { v } })
    await brain.flush()
    await brain.repackHistory()

    // Reclaim down to the 3 newest generations — packed segments below the
    // horizon drop whole; asOf below throws loudly.
    const res = await brain.compactHistory({ maxGenerations: 3 })
    expect(res.removedGenerations).toBeGreaterThan(0)
    await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError)
    expect((await brain.get(id))?.metadata?.v).toBe(8)
    await brain.close()
  })

  it('generationDigest: reopen-stable, divergence-sensitive, loud below the horizon', async () => {
    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
    const dir = tempDir()
    const brain = await openBrain(dir)
    const id = await brain.add({ data: 'digest-probe', type: NounType.Document, metadata: { v: 0 } })
    for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } })
    await brain.flush()
    await brain.repackHistory()

    const gen = brain.generation()
    const atHead = await brain.generationDigest(gen)
    const atMid = await brain.generationDigest(3)
    expect(atHead).toMatch(/^[0-9a-f]{8}$/)
    expect(atMid).not.toBe(atHead) // more history ⇒ different digest
    await brain.close()

    // Reopen-stable: same history, same digests (packed prefix stability).
    const reopened = await openBrain(dir)
    expect(await reopened.generationDigest(gen)).toBe(atHead)
    expect(await reopened.generationDigest(3)).toBe(atMid)

    // New history diverges the head digest.
    await reopened.update({ id, metadata: { v: 7 } })
    await reopened.flush()
    expect(await reopened.generationDigest(reopened.generation())).not.toBe(atHead)

    // Below the horizon: LOUD, never a silent pin of reclaimed history.
    await reopened.compactHistory({ maxGenerations: 2 })
    await expect(reopened.generationDigest(1)).rejects.toBeInstanceOf(GenerationCompactedError)
    await reopened.close()
  })

  it('a spent time budget is a consistent no-op; the next pass resumes', async () => {
    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 2
    const dir = tempDir()
    const brain = await openBrain(dir)
    const id = await brain.add({ data: 'budget-probe', type: NounType.Document, metadata: { v: 0 } })
    for (let v = 1; v <= 6; v++) await brain.update({ id, metadata: { v } })
    await brain.flush()

    const bounded = await brain.repackHistory({ timeBudgetMs: 0 })
    expect(bounded).toEqual({ foldedGenerations: 0, segmentsCreated: 0 })

    const resumed = await brain.repackHistory()
    expect(resumed.foldedGenerations).toBeGreaterThan(0)
    const db = await brain.asOf(3)
    expect((await db.get(id))?.metadata?.v).toBeDefined()
    await db.release()
    await brain.close()
  })
})