Merge remote-tracking branch 'origin/reclaim/packed-history-density'

tests/integration/history-repacking.test.ts

@@ -16,6 +16,7 @@ import { describe, it, expect, afterEach } from 'vitest'
 import * as fs from 'node:fs'
 import * as path from 'node:path'
 import * as os from 'node:os'
+import * as zlib from 'node:zlib'
 import { Brainy } from '../../src/brainy.js'
 import { NounType } from '../../src/types/graphTypes.js'
 import { GenerationStore } from '../../src/db/generationStore.js'
@@ -57,6 +58,107 @@ describe('history repacking — the two-tier lifecycle', () => {
     }
   })
 
+  /**
+   * THE HOLE, END TO END — the shape a real store carries.
+   *
+   * A forensic fixture was measured with generation directories 1..2503
+   * present except for exactly one: 1416. Its fact-log segment already showed
+   * the tell — `seg-...1410.bfl` declaring firstGeneration 1410, lastGeneration
+   * 1940 (531 generations) while recording only 530 facts.
+   *
+   * Before the fix, repacking such a store folded ACROSS that hole: the batch
+   * skipped 1416 (no readable delta) and the sealed segment declared a range
+   * spanning it anyway. The next open merged that declared range back into
+   * committedRanges, re-admitting 1416 as committed history, and every
+   * subsequent auto-compaction pass then asked the packed tier for a frame
+   * that was never written — producing, on EVERY run, the non-fatal narration
+   *
+   *   Auto-compaction of generational history failed (non-fatal): generation
+   *   N is inside sealed segment seg-....bgs's declared range but has no frame
+   *   — packed history is damaged
+   *
+   * This pin removes a generation directory to make the same hole, then
+   * requires repack + reopen + compaction to complete cleanly.
+   */
+  it('a missing generation directory does not poison the packed tier', async () => {
+    const dir = tempDir()
+    // `retention: 'all'` throughout: close() otherwise auto-compacts the
+    // history away, and this pin needs the cold generations still on disk so
+    // there is something to punch a hole in. The live window stays at its
+    // production default for the build phase, so nothing folds yet.
+    const archival = async (): Promise<Brainy> => {
+      const b = new Brainy({
+        requireSubtype: false,
+        storage: { type: 'filesystem', path: dir },
+        embeddingFunction: stub,
+        retention: 'all'
+      })
+      await b.init()
+      return b
+    }
+    const brain = await archival()
+
+    const id = await brain.add({
+      data: 'holed-entity',
+      type: NounType.Document,
+      metadata: { v: 0 }
+    })
+    // One flush per update: single-op writes coalesce inside a flush window,
+    // so a history deep enough to have a middle needs the windows separated.
+    for (let v = 1; v <= 12; v++) {
+      await brain.update({ id, metadata: { v } })
+      await brain.flush()
+    }
+    await brain.close()
+
+    // Punch the hole: delete ONE generation directory in the middle of the
+    // cold range, exactly as the real store presents it.
+    const genRoot = path.join(dir, '_generations')
+    const numeric = fs
+      .readdirSync(genRoot, { withFileTypes: true })
+      .filter((e) => e.isDirectory() && /^\d+$/.test(e.name))
+      .map((e) => Number(e.name))
+      .sort((a, b) => a - b)
+    expect(numeric.length).toBeGreaterThan(6)
+    const victim = numeric[Math.floor(numeric.length / 2)]
+    fs.rmSync(path.join(genRoot, String(victim)), { recursive: true, force: true })
+
+    // Now shrink the live window and reopen. close() repacks automatically
+    // (brainy.ts phase 0b), so this is the production sequence exactly: a
+    // store with a hole in its history gets folded by ordinary housekeeping,
+    // with nobody asking for it.
+    ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
+    const reopened = await archival()
+    const result = await reopened.repackHistory()
+    expect(result.foldedGenerations).toBeGreaterThan(0)
+
+    const segDir = path.join(dir, SEGMENTS_PREFIX)
+    const manifestPath = ['manifest.json', 'manifest.json.gz']
+      .map((f) => path.join(segDir, f))
+      .find((p) => fs.existsSync(p))!
+    const raw = manifestPath.endsWith('.gz')
+      ? zlib.gunzipSync(fs.readFileSync(manifestPath)).toString('utf8')
+      : fs.readFileSync(manifestPath, 'utf8')
+    const manifest = JSON.parse(raw) as {
+      segments: Array<{ firstGeneration: number; lastGeneration: number; frames: number }>
+    }
+
+    // THE LAW: every sealed segment declares exactly as many generations as it
+    // holds frames, and none of them spans the victim.
+    for (const s of manifest.segments) {
+      expect(s.lastGeneration - s.firstGeneration + 1).toBe(s.frames)
+      expect(victim >= s.firstGeneration && victim <= s.lastGeneration).toBe(false)
+    }
+
+    await reopened.close()
+
+    // And the pass that used to fail on every run now completes: reopen (which
+    // re-seeds committedRanges from the packed tier) then compact history.
+    const third = await openBrain(dir)
+    await expect(third.compactHistory({ maxGenerations: 2 })).resolves.toBeDefined()
+    await third.close()
+  })
+
   it('repack preserves every historical read across cold reopen; folded dirs are gone', async () => {
     ;(GenerationStore as any).REPACK_LIVE_WINDOW = 3
     const dir = tempDir()