/** * @module tests/integration/db-mvcc * @description Credibility-bar proof suite for Brainy 8.0's generational MVCC * storage core + Datomic-style Db API. Each test PROVES a stated guarantee * (not merely exercises the API): * * 1. Isolation — a pinned Db reads its pinned state exactly, across 200 * subsequent transact mutations (updates + removes). * 2. Atomicity — a batch with a failing later operation applies NOTHING * (generation unchanged, no partial records), for both plan-time and * execution-phase failures (the latter via real fault injection into the * storage adapter, exercising the production rollback path). * 3. CAS — `ifAtGeneration` commits when fresh, throws * `GenerationConflictError` with correct expected/actual when stale. * 4. Snapshot integrity — persist → mutate source heavily → `Brainy.load()` * is byte-identical to the pre-mutation state (hard-link safety), and an * in-memory brain persists to a loadable directory. * 5. Compaction safety — pinned reads stay correct across `compactHistory()`; * after release + compact the superseded record-sets are reclaimed. * 6. with() — speculative overlays are visible on the overlay Db only; the * underlying brain and the original Db are untouched. * 7. Generation monotonicity across close/reopen. * 8. Crash consistency — a simulated crash between record staging and the * manifest rename (the commit point) recovers to the exact pre-transaction * state on reopen, through the REAL production recovery path. * 9. Versioned provider wiring — a provider implementing the 4-method * `VersionedIndexProvider` contract receives balanced pin/release (and a * visibility consult per pin) as Db values are created and released. * 10. Historical full query surface — vector search, graph traversal, and * aggregation at a past pinned generation return at-generation results * via ephemeral index materialization (for `now()` and `asOf()` pins * alike, built once per Db and cached); `release()` frees the * materialization by closing the ephemeral reader; speculative overlays * keep the documented `SpeculativeOverlayError` boundary. * * All entities carry explicit vectors so the suite never invokes the * embedding engine (vector queries use find({ vector }); semantic string * search on overlays is exercised only for its documented error). */ import { describe, it, expect, afterEach, vi } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' import { Brainy } from '../../src/brainy.js' import { Db, type HistoricalQueryHandle } from '../../src/db/db.js' import { GenerationCompactedError, GenerationConflictError, SpeculativeOverlayError } from '../../src/db/errors.js' import type { GenerationStore } from '../../src/db/generationStore.js' import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js' import { RevisionConflictError } from '../../src/transaction/RevisionConflictError.js' import type { StorageAdapter } from '../../src/coreTypes.js' import { NounType, VerbType } from '../../src/types/graphTypes.js' /** Deterministic 384-dim vector so no test ever invokes the embedder. */ function vec(seed: number): number[] { return Array.from({ length: 384 }, (_, i) => ((seed * 31 + i * 7) % 100) / 100) } /** * Map a readable label to a deterministic UUID-shaped id (entity ids must be * UUIDs — the sharded storage layout derives the shard from the UUID hex). * Two independent FNV-style mixes keep 7+ distinct labels collision-free. */ function uid(label: string): string { let h1 = 0x811c9dc5 for (let i = 0; i < label.length; i++) { h1 = Math.imul(h1 ^ label.charCodeAt(i), 0x01000193) >>> 0 } let h2 = 0xdeadbeef for (let i = label.length - 1; i >= 0; i--) { h2 = Math.imul(h2 ^ label.charCodeAt(i), 0x85ebca6b) >>> 0 } const hex = h1.toString(16).padStart(8, '0') + h2.toString(16).padStart(8, '0') return `00000000-0000-4000-8000-${hex.slice(0, 12)}` } /** Typed access to the brain's private generation store (test injection point). */ function generationStoreOf(brain: Brainy): GenerationStore { return (brain as unknown as { generationStore: GenerationStore }).generationStore } describe('8.0 Db API — generational MVCC', () => { const brains: Brainy[] = [] const tempDirs: string[] = [] /** Track a brain for afterEach teardown. */ async function openMemoryBrain(): Promise { const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) await brain.init() brains.push(brain) return brain } /** Open (and track) a filesystem brain rooted at a fresh temp directory. */ async function openFsBrain(dir?: string): Promise<{ brain: Brainy; dir: string }> { const rootDirectory = dir ?? makeTempDir() const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: rootDirectory } }) await brain.init() brains.push(brain) return { brain, dir: rootDirectory } } function makeTempDir(): string { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-db-mvcc-')) tempDirs.push(dir) return dir } afterEach(async () => { for (const brain of brains.splice(0)) { try { await brain.close() } catch { // already closed by the test } } for (const dir of tempDirs.splice(0)) { await fs.promises.rm(dir, { recursive: true, force: true }) } }) // ========================================================================== // 1. Isolation // ========================================================================== it('proof 1 — a pinned Db reads its pinned state exactly across 200 transact mutations', async () => { const brain = await openMemoryBrain() const ids = Array.from({ length: 20 }, (_, i) => uid(`iso-e${i}`)) await brain.transact( ids.map((id, i) => ({ op: 'add' as const, id, type: NounType.Document, data: `doc-${i}`, vector: vec(i), metadata: { v: 0, slot: i } })) ) const db = brain.now() const pinned = new Map() for (const id of ids) { pinned.set(id, await db.get(id)) expect(pinned.get(id)).not.toBeNull() } const pinnedFind = await db.find({ type: NounType.Document, limit: 100 }) expect(pinnedFind.length).toBe(20) // 200 mutations: 198 updates over the first 18 ids + 2 removes. for (let i = 0; i < 200; i++) { if (i === 100) { await brain.transact([{ op: 'remove', id: uid('iso-e18') }]) } else if (i === 150) { await brain.transact([{ op: 'remove', id: uid('iso-e19') }]) } else { await brain.transact([ { op: 'update', id: ids[i % 18], metadata: { v: i + 1, touchedAt: i } } ]) } } // Live state moved… const liveUpdated = await brain.get(ids[0]) expect((liveUpdated?.metadata as { v: number }).v).toBeGreaterThan(0) expect(await brain.get(uid('iso-e18'))).toBeNull() expect(await brain.get(uid('iso-e19'))).toBeNull() // …but EVERY read through the pinned Db is exactly the pinned state, // including the two entities removed after the pin. for (const id of ids) { expect(await db.get(id)).toEqual(pinned.get(id)) } // Set-level isolation: metadata find() at the pin still sees all 20. const histFind = await db.find({ type: NounType.Document, limit: 100 }) expect(histFind.length).toBe(20) for (const row of histFind) { expect((row.metadata as { v: number }).v).toBe(0) } await db.release() }) // ========================================================================== // 2. Atomicity // ========================================================================== it('proof 2a — a plan-time failure in a later op applies nothing', async () => { const brain = await openMemoryBrain() const g0 = brain.generation() await expect( brain.transact([ { op: 'add', id: uid('atomic-a'), type: NounType.Document, data: 'a', vector: vec(1), metadata: { ok: true } }, { op: 'update', id: uid('never-created'), metadata: { boom: true } } ]) ).rejects.toThrow(`Entity ${uid('never-created')} not found`) expect(brain.generation()).toBe(g0) expect(await brain.get(uid('atomic-a'))).toBeNull() expect((await brain.find({ type: NounType.Document, limit: 10 })).length).toBe(0) }) it('proof 2b — an execution-phase failure rolls back every applied op (generation unchanged, zero partial records)', async () => { const brain = await openMemoryBrain() await brain.transact([ { op: 'add', id: uid('atomic-base'), type: NounType.Document, data: 'base', vector: vec(0), metadata: { base: true } } ]) const g0 = brain.generation() const storage = brain.storageAdapter // Real fault injection: the verb-metadata write of the LAST operation // fails once, after the earlier add operations have already executed — // exercising the production TransactionManager rollback path. const spy = vi .spyOn(storage, 'saveVerbMetadata') .mockImplementationOnce(async () => { throw new Error('disk full (injected)') }) await expect( brain.transact([ { op: 'add', id: uid('atomic-b'), type: NounType.Person, data: 'b', vector: vec(2), metadata: { name: 'b' } }, { op: 'relate', from: uid('atomic-b'), to: uid('atomic-base'), type: VerbType.RelatedTo } ]) ).rejects.toThrow('disk full (injected)') spy.mockRestore() // ZERO ops applied: generation unchanged, the add rolled back, no edge. expect(brain.generation()).toBe(g0) expect(await brain.get(uid('atomic-b'))).toBeNull() expect((await brain.related({ from: uid('atomic-b') })).length).toBe(0) expect((await brain.find({ type: NounType.Person, limit: 10 })).length).toBe(0) // No partial generation records: the staging directory was removed. const records = await storage.listRawObjects(`_generations/${g0 + 1}`) expect(records).toEqual([]) // The store remains fully writable and consistent after the rollback. const db = await brain.transact([ { op: 'add', id: uid('atomic-c'), type: NounType.Person, data: 'c', vector: vec(3), metadata: { name: 'c' } } ]) expect(db.generation).toBe(g0 + 1) await db.release() }) it('proof 2c — an ifRev conflict on a later update rejects the whole batch', async () => { const brain = await openMemoryBrain() await brain.transact([ { op: 'add', id: uid('rev-e'), type: NounType.Document, data: 'rev', vector: vec(4), metadata: { v: 1 } } ]) await brain.update({ id: uid('rev-e'), metadata: { v: 2 } }) // _rev is now 2 const g0 = brain.generation() await expect( brain.transact([ { op: 'add', id: uid('rev-new'), type: NounType.Document, data: 'new', vector: vec(5), metadata: {} }, { op: 'update', id: uid('rev-e'), metadata: { v: 3 }, ifRev: 1 } // stale ]) ).rejects.toThrow(RevisionConflictError) expect(brain.generation()).toBe(g0) expect(await brain.get(uid('rev-new'))).toBeNull() expect(((await brain.get(uid('rev-e')))?.metadata as { v: number }).v).toBe(2) }) // ========================================================================== // 3. CAS (ifAtGeneration) // ========================================================================== it('proof 3 — ifAtGeneration commits when fresh and conflicts when stale', async () => { const brain = await openMemoryBrain() const db = brain.now() const committed = await brain.transact( [ { op: 'add', id: uid('cas-e'), type: NounType.Document, data: 'cas', vector: vec(6), metadata: { n: 1 } } ], { ifAtGeneration: db.generation } ) expect(committed.generation).toBe(db.generation + 1) try { await brain.transact( [{ op: 'update', id: uid('cas-e'), metadata: { n: 2 } }], { ifAtGeneration: db.generation } // stale — committed moved past it ) expect.unreachable('should have thrown GenerationConflictError') } catch (err) { expect(err).toBeInstanceOf(GenerationConflictError) expect((err as GenerationConflictError).expected).toBe(db.generation) expect((err as GenerationConflictError).actual).toBe(committed.generation) } // Nothing committed by the conflicting attempt. expect(((await brain.get(uid('cas-e')))?.metadata as { n: number }).n).toBe(1) expect(brain.generation()).toBe(committed.generation) await db.release() await committed.release() }) // ========================================================================== // 4. Snapshot integrity (persist → mutate → load) // ========================================================================== it('proof 4a — a persisted snapshot is immune to heavy source mutation (hard-link safety)', async () => { const { brain } = await openFsBrain() const ids = Array.from({ length: 10 }, (_, i) => uid(`snap-e${i}`)) await brain.transact( ids.map((id, i) => ({ op: 'add' as const, id, type: NounType.Document, data: `snapshot-doc-${i}`, vector: vec(i + 10), metadata: { generation: 'pre', slot: i } })) ) const relDb = await brain.transact([ { op: 'relate', from: ids[0], to: ids[1], type: VerbType.References }, { op: 'relate', from: ids[1], to: ids[2], type: VerbType.References } ]) await relDb.release() // Capture pre-mutation state, then snapshot. const preState = new Map() for (const id of ids) { preState.set(id, await brain.get(id)) } const preRelations = await brain.related({ from: ids[0] }) expect(preRelations.length).toBe(1) const snapDir = path.join(makeTempDir(), 'snapshot') const db = brain.now() await db.persist(snapDir) await db.release() // Mutate the source heavily: rewrite every entity, delete some, add new. for (const [i, id] of ids.entries()) { await brain.transact([ { op: 'update', id, data: `mutated-${i}`, metadata: { generation: 'post' } } ]) } await brain.transact([{ op: 'remove', id: ids[0] }]) await brain.transact([ { op: 'add', id: uid('snap-after'), type: NounType.Document, data: 'added after snapshot', vector: vec(99), metadata: { generation: 'post' } } ]) await brain.flush() // The snapshot opens as a self-contained store with the PRE state. const loaded = await Brainy.load(snapDir) for (const id of ids) { const entity = await loaded.get(id) expect(entity).toEqual(preState.get(id)) } expect(await loaded.get(uid('snap-after'))).toBeNull() const loadedFind = await loaded.find({ type: NounType.Document, limit: 100 }) expect(loadedFind.length).toBe(10) for (const row of loadedFind) { expect((row.metadata as { generation: string }).generation).toBe('pre') } const loadedRelations = await loaded.related({ from: ids[0] }) expect(loadedRelations.length).toBe(1) expect(loadedRelations[0].to).toBe(ids[1]) await loaded.release() // And asOf(path) is the instance-method route to the same snapshot. const viaAsOf = await brain.asOf(snapDir) expect(((await viaAsOf.get(ids[3])) as { metadata: { generation: string } }).metadata.generation).toBe('pre') await viaAsOf.release() }) it('proof 4b — an in-memory brain persists to a directory loadable as a real store', async () => { const brain = await openMemoryBrain() await brain.transact([ { op: 'add', id: uid('mem-e1'), type: NounType.Document, data: 'memory-persisted', vector: vec(21), metadata: { from: 'memory' } } ]) const snapDir = path.join(makeTempDir(), 'memory-snapshot') const db = brain.now() await db.persist(snapDir) await db.release() const loaded = await Brainy.load(snapDir) const entity = await loaded.get(uid('mem-e1')) expect(entity).not.toBeNull() expect((entity?.metadata as { from: string }).from).toBe('memory') await loaded.release() }) it('proof 4c — persist() refuses a view that history has moved past (honest boundary)', async () => { const brain = await openMemoryBrain() const db = brain.now() await brain.transact([ { op: 'add', id: uid('stale-persist'), type: NounType.Document, data: 'x', vector: vec(30), metadata: {} } ]) const snapDir = path.join(makeTempDir(), 'stale-snapshot') await expect(db.persist(snapDir)).rejects.toThrow(GenerationConflictError) await db.release() }) // ========================================================================== // 5. Compaction safety // ========================================================================== it('proof 5 — compaction never breaks pinned reads; after release the records are reclaimed', async () => { const { brain } = await openFsBrain() const storage = brain.storageAdapter // Every transact() returns a PINNED Db — release the ones this test // does not keep, so compaction eligibility is determined solely by `db`. await ( await brain.transact([ { op: 'add', id: uid('compact-e'), type: NounType.Document, data: 'v1', vector: vec(40), metadata: { v: 1 } } ]) ).release() await ( await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 2 } }]) ).release() const db = brain.now() // pinned at v2 const pinnedEntity = await db.get(uid('compact-e')) await ( await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 3 } }]) ).release() await ( await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 4 } }]) ).release() const recordsBefore = (await storage.listRawObjects('_generations')).length expect(recordsBefore).toBeGreaterThan(0) // Compact while pinned: record-sets above the pin survive, pinned reads stay correct. const first = await brain.compactHistory() expect(await db.get(uid('compact-e'))).toEqual(pinnedEntity) expect(((await db.get(uid('compact-e')))?.metadata as { v: number }).v).toBe(2) // After release, everything is reclaimable. await db.release() const second = await brain.compactHistory() expect(first.removedGenerations + second.removedGenerations).toBeGreaterThan(0) const recordsAfter = (await storage.listRawObjects('_generations')).length expect(recordsAfter).toBeLessThan(recordsBefore) expect(recordsAfter).toBe(0) // Compacted generations are honestly unreachable. await expect(brain.asOf(1)).rejects.toThrow(GenerationCompactedError) }) // ========================================================================== // 6. with() — speculative overlays // ========================================================================== it('proof 6 — with() overlays are visible only on the overlay Db; brain and base Db untouched', async () => { const brain = await openMemoryBrain() await brain.transact([ { op: 'add', id: uid('with-a'), type: NounType.Person, data: 'Ada', vector: vec(50), metadata: { v: 1 } }, { op: 'add', id: uid('with-b'), type: NounType.Person, data: 'Grace', vector: vec(51), metadata: { v: 1 } } ]) const g0 = brain.generation() const db = brain.now() const spec = await db.with([ { op: 'update', id: uid('with-a'), metadata: { v: 2 } }, { op: 'add', id: uid('with-new'), type: NounType.Project, data: 'Apollo', metadata: { speculative: true } }, { op: 'remove', id: uid('with-b') }, { op: 'relate', from: uid('with-a'), to: uid('with-new'), type: VerbType.WorksOn } ]) // The overlay view sees everything… expect(spec.speculative).toBe(true) expect(((await spec.get(uid('with-a')))?.metadata as { v: number }).v).toBe(2) expect(await spec.get(uid('with-new'))).not.toBeNull() expect(await spec.get(uid('with-b'))).toBeNull() const specEdges = await spec.related({ from: uid('with-a') }) expect(specEdges.length).toBe(1) expect(specEdges[0].to).toBe(uid('with-new')) const specFind = await spec.find({ type: NounType.Project, limit: 10 }) expect(specFind.length).toBe(1) // …the base Db sees none of it… expect(db.speculative).toBe(false) expect(((await db.get(uid('with-a')))?.metadata as { v: number }).v).toBe(1) expect(await db.get(uid('with-new'))).toBeNull() expect(await db.get(uid('with-b'))).not.toBeNull() // …and the brain (disk, indexes, generation counter) is untouched. expect(brain.generation()).toBe(g0) expect(((await brain.get(uid('with-a')))?.metadata as { v: number }).v).toBe(1) expect(await brain.get(uid('with-new'))).toBeNull() expect((await brain.related({ from: uid('with-a') })).length).toBe(0) await spec.release() await db.release() }) // ========================================================================== // 7. Generation monotonicity across close/reopen // ========================================================================== it('proof 7 — the generation counter never moves backwards across close/reopen', async () => { const dir = makeTempDir() const { brain: first } = await openFsBrain(dir) await first.transact([ { op: 'add', id: uid('mono-e'), type: NounType.Document, data: 'v1', vector: vec(60), metadata: { v: 1 } } ]) await first.transact([{ op: 'update', id: uid('mono-e'), metadata: { v: 2 } }]) // Single-operation write — bumps the counter outside transact. await first.update({ id: uid('mono-e'), metadata: { v: 3 } }) const beforeClose = first.generation() expect(beforeClose).toBeGreaterThanOrEqual(3) await first.close() const { brain: second } = await openFsBrain(dir) expect(second.generation()).toBeGreaterThanOrEqual(beforeClose) const db = await second.transact([{ op: 'update', id: uid('mono-e'), metadata: { v: 4 } }]) expect(db.generation).toBeGreaterThan(beforeClose) await db.release() }) // ========================================================================== // 8. Crash consistency // ========================================================================== it('proof 8 — a crash before the manifest rename recovers to the exact pre-transaction state', async () => { const dir = makeTempDir() const { brain: first } = await openFsBrain(dir) await first.transact([ { op: 'add', id: uid('crash-e'), type: NounType.Document, data: 'stable', vector: vec(70), metadata: { v: 1 } } ]) const committedGen = first.generation() // Simulated crash at the worst point of the REAL commit protocol: every // record is staged and the batch has executed, but the atomic manifest // rename — the commit point — never happens. The throwing injector skips // the abort cleanup exactly as a dead process would. generationStoreOf(first).setCommitFaultInjector((phase) => { if (phase === 'before-manifest-rename') { throw new Error('simulated process crash') } }) await expect( first.transact([ { op: 'update', id: uid('crash-e'), metadata: { v: 2 } }, { op: 'add', id: uid('crash-new'), type: NounType.Document, data: 'uncommitted', vector: vec(71), metadata: {} } ]) ).rejects.toThrow('simulated process crash') generationStoreOf(first).setCommitFaultInjector(undefined) // The in-process state is dirty (batch executed) — close flushes it all, // the realistic worst case for the recovery path. await first.close() // Reopen: recovery rolls the uncommitted generation back and rebuilds // the indexes from the repaired records. const { brain: second } = await openFsBrain(dir) const recovered = await second.get(uid('crash-e')) expect((recovered?.metadata as { v: number }).v).toBe(1) expect(await second.get(uid('crash-new'))).toBeNull() const found = await second.find({ type: NounType.Document, limit: 10 }) expect(found.length).toBe(1) expect((found[0].metadata as { v: number }).v).toBe(1) // The crashed generation number is never reissued. expect(second.generation()).toBeGreaterThanOrEqual(committedGen + 1) const db = await second.transact([{ op: 'update', id: uid('crash-e'), metadata: { v: 5 } }]) expect(db.generation).toBeGreaterThan(committedGen + 1) await db.release() }) // ========================================================================== // 9. Versioned provider wiring // ========================================================================== it('proof 9 — a VersionedIndexProvider receives balanced pin/release as Db values live and die', async () => { const calls = { pins: [] as bigint[], releases: [] as bigint[], visibility: [] as bigint[] } const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } }) brain.use({ name: 'test-versioned-graph-index', activate: async (ctx) => { ctx.registerProvider('graphIndex', (storage: StorageAdapter) => Object.assign(new GraphAdjacencyIndex(storage), { generation: () => 0n, isGenerationVisible: (g: bigint) => { calls.visibility.push(g) return true }, pin: (g: bigint) => { calls.pins.push(g) }, release: (g: bigint) => { calls.releases.push(g) } }) ) return true } }) await brain.init() brains.push(brain) const base = brain.generation() const db1 = brain.now() expect(calls.pins).toEqual([BigInt(base)]) expect(calls.visibility).toEqual([BigInt(base)]) // consulted at pin time const db2 = await brain.transact([ { op: 'add', id: uid('vp-e'), type: NounType.Document, data: 'pinned', vector: vec(80), metadata: {} } ]) expect(calls.pins).toEqual([BigInt(base), BigInt(base + 1)]) const db3 = await brain.asOf(base) expect(calls.pins).toEqual([BigInt(base), BigInt(base + 1), BigInt(base)]) // A with() overlay takes its own pin on the same generation. const spec = await db2.with([{ op: 'update', id: uid('vp-e'), metadata: { spec: true } }]) expect(calls.pins.length).toBe(4) await db1.release() await db2.release() await db3.release() await spec.release() await spec.release() // idempotent — must NOT double-release the pin expect(calls.releases.length).toBe(calls.pins.length) expect([...calls.releases].sort()).toEqual([...calls.pins].sort()) }) // ========================================================================== // Supporting guarantees: asOf, since, receipts, tx-log, honest errors // ========================================================================== it('asOf(generation) and asOf(Date) resolve historical state; future/negative are rejected', async () => { const brain = await openMemoryBrain() const tx1 = await brain.transact( [ { op: 'add', id: uid('asof-e'), type: NounType.Document, data: 'v1', vector: vec(90), metadata: { v: 1 } } ], { meta: { reason: 'first commit' } } ) await new Promise((resolve) => setTimeout(resolve, 5)) const betweenCommits = new Date() await new Promise((resolve) => setTimeout(resolve, 5)) await brain.transact([{ op: 'update', id: uid('asof-e'), metadata: { v: 2 } }]) // By generation. const atFirst = await brain.asOf(tx1.generation) expect(((await atFirst.get(uid('asof-e')))?.metadata as { v: number }).v).toBe(1) expect(atFirst.timestamp).toBe(tx1.timestamp) await atFirst.release() // By timestamp: resolves to the newest commit at-or-before the instant. const atInstant = await brain.asOf(betweenCommits) expect(atInstant.generation).toBe(tx1.generation) expect(((await atInstant.get(uid('asof-e')))?.metadata as { v: number }).v).toBe(1) await atInstant.release() // Honest range errors. await expect(brain.asOf(brain.generation() + 100)).rejects.toThrow(RangeError) await expect(brain.asOf(-1)).rejects.toThrow(RangeError) await expect(brain.asOf('/no/such/snapshot/dir')).rejects.toThrow('not a snapshot directory') // The transact meta was reified into the tx-log. const lines = await brain.storageAdapter.readTxLogLines() const entries = lines.map((line) => JSON.parse(line)) const first = entries.find((entry) => entry.generation === tx1.generation) expect(first?.meta).toEqual({ reason: 'first commit' }) await tx1.release() }) it('receipts resolve ids in input order, including intra-batch references and relate dedupe', async () => { const brain = await openMemoryBrain() const db = await brain.transact([ { op: 'add', id: uid('rcpt-aa'), type: NounType.Person, data: 'a', vector: vec(91), metadata: {} }, { op: 'add', id: uid('rcpt-bb'), type: NounType.Person, data: 'b', vector: vec(92), metadata: {} }, { op: 'relate', from: uid('rcpt-aa'), to: uid('rcpt-bb'), type: VerbType.Knows }, { op: 'update', id: uid('rcpt-bb'), metadata: { seen: true } } ]) expect(db.receipt).toBeDefined() expect(db.receipt!.generation).toBe(db.generation) expect(db.receipt!.ids.length).toBe(4) expect(db.receipt!.ids[0]).toBe(uid('rcpt-aa')) expect(db.receipt!.ids[1]).toBe(uid('rcpt-bb')) expect(db.receipt!.ids[3]).toBe(uid('rcpt-bb')) const relationId = db.receipt!.ids[2] expect((await brain.related({ from: uid('rcpt-aa') }))[0].id).toBe(relationId) // Duplicate relate (same from/to/type) dedupes to the SAME id — both // within a batch and against committed state, mirroring relate(). const dup = await brain.transact([ { op: 'relate', from: uid('rcpt-aa'), to: uid('rcpt-bb'), type: VerbType.Knows } ]) expect(dup.receipt!.ids[0]).toBe(relationId) expect((await brain.related({ from: uid('rcpt-aa') })).length).toBe(1) await db.release() await dup.release() }) it('since() reports the ids changed between two pinned views', async () => { const brain = await openMemoryBrain() await brain.transact([ { op: 'add', id: uid('since-a'), type: NounType.Document, data: 'a', vector: vec(93), metadata: {} } ]) const before = brain.now() await brain.transact([ { op: 'add', id: uid('since-b'), type: NounType.Document, data: 'b', vector: vec(94), metadata: {} }, { op: 'relate', from: uid('since-a'), to: uid('since-b'), type: VerbType.References } ]) await brain.transact([{ op: 'update', id: uid('since-a'), metadata: { touched: true } }]) const after = brain.now() const changed = await after.since(before) expect([...changed.nouns].sort()).toEqual([uid('since-a'), uid('since-b')].sort()) expect(changed.verbs.length).toBe(1) expect(changed.fromGeneration).toBe(before.generation) expect(changed.toGeneration).toBe(after.generation) // Direction is enforced. await expect(before.since(after)).rejects.toThrow(RangeError) await before.release() await after.release() }) it('historical vector search is served at the pinned generation via index materialization', async () => { const brain = await openMemoryBrain() // Two one-hot vectors: orthogonal under cosine, so nearest-neighbor // results are fully deterministic. const axisA = Array.from({ length: 384 }, (_, i) => (i === 0 ? 1 : 0)) const axisB = Array.from({ length: 384 }, (_, i) => (i === 1 ? 1 : 0)) const axisC = Array.from({ length: 384 }, (_, i) => (i === 2 ? 1 : 0)) await brain.transact([ { op: 'add', id: uid('mat-a'), type: NounType.Document, data: 'a', vector: axisA, metadata: { who: 'a' } }, { op: 'add', id: uid('mat-b'), type: NounType.Document, data: 'b', vector: axisB, metadata: { who: 'b' } }, { op: 'add', id: uid('mat-del'), type: NounType.Document, data: 'del', vector: axisC, metadata: { who: 'del' } } ]) const db = brain.now() // Mutate distinctively after the pin: swap a/b's vectors, delete 'del'. await brain.transact([ { op: 'update', id: uid('mat-a'), vector: axisB }, { op: 'update', id: uid('mat-b'), vector: axisA }, { op: 'remove', id: uid('mat-del') } ]) // Live index reflects the swap… const liveNearA = await brain.find({ vector: axisA, limit: 1 }) expect(liveNearA[0].id).toBe(uid('mat-b')) // …the pinned view finds the OLD vector placement, including the // since-deleted entity, via the at-generation materialization. const pinnedNearA = await db.find({ vector: axisA, limit: 1 }) expect(pinnedNearA[0].id).toBe(uid('mat-a')) const pinnedNearC = await db.find({ vector: axisC, limit: 1 }) expect(pinnedNearC[0].id).toBe(uid('mat-del')) // Record-path reads on the same Db stay consistent with the materialization. expect(((await db.get(uid('mat-del')))?.metadata as { who: string }).who).toBe('del') // Released views refuse all reads (and free the materialization). await db.release() await expect(db.find({ vector: axisA, limit: 1 })).rejects.toThrow('released Db') }) it('historical graph traversal is served at the pinned generation via index materialization', async () => { const brain = await openMemoryBrain() await brain.transact([ { op: 'add', id: uid('trav-a'), type: NounType.Person, data: 'a', vector: vec(110), metadata: {} }, { op: 'add', id: uid('trav-b'), type: NounType.Document, data: 'b', vector: vec(111), metadata: {} }, { op: 'add', id: uid('trav-c'), type: NounType.Document, data: 'c', vector: vec(112), metadata: {} }, { op: 'relate', from: uid('trav-a'), to: uid('trav-b'), type: VerbType.References } ]) const db = brain.now() // Rewire the graph after the pin: a→b becomes a→c. const oldEdge = (await brain.related({ from: uid('trav-a') }))[0] await brain.transact([ { op: 'unrelate', id: oldEdge.id }, { op: 'relate', from: uid('trav-a'), to: uid('trav-c'), type: VerbType.References } ]) // Live traversal sees the new wiring… const liveConnected = await brain.find({ connected: { from: uid('trav-a') }, limit: 10 }) expect(liveConnected.map((row) => row.id)).toContain(uid('trav-c')) expect(liveConnected.map((row) => row.id)).not.toContain(uid('trav-b')) // …the pinned view traverses the OLD graph. const pinnedConnected = await db.find({ connected: { from: uid('trav-a') }, limit: 10 }) expect(pinnedConnected.map((row) => row.id)).toContain(uid('trav-b')) expect(pinnedConnected.map((row) => row.id)).not.toContain(uid('trav-c')) // The record-path related() agrees with the materialized traversal. const pinnedEdges = await db.related({ from: uid('trav-a') }) expect(pinnedEdges.length).toBe(1) expect(pinnedEdges[0].to).toBe(uid('trav-b')) await db.release() }) it('historical aggregation computes at-generation values via index materialization', async () => { const brain = await openMemoryBrain() brain.defineAggregate({ name: 'order_totals', source: { type: NounType.Document }, groupBy: ['category'], metrics: { total: { op: 'sum', field: 'amount' }, orders: { op: 'count' } } }) await brain.transact([ { op: 'add', id: uid('agg-1'), type: NounType.Document, data: 'o1', vector: vec(120), metadata: { category: 'invoice', amount: 10 } }, { op: 'add', id: uid('agg-2'), type: NounType.Document, data: 'o2', vector: vec(121), metadata: { category: 'invoice', amount: 20 } } ]) const db = brain.now() // Mutate after the pin: add an order and inflate an existing one. await brain.transact([ { op: 'add', id: uid('agg-3'), type: NounType.Document, data: 'o3', vector: vec(122), metadata: { category: 'invoice', amount: 70 } }, { op: 'update', id: uid('agg-1'), metadata: { amount: 100 } } ]) // Live aggregate reflects the mutations… const liveRows = await brain.find({ aggregate: 'order_totals' }) const liveInvoice = liveRows.find((row) => (row.metadata as { category: string }).category === 'invoice') expect((liveInvoice?.metadata as { total: number }).total).toBe(190) expect((liveInvoice?.metadata as { orders: number }).orders).toBe(3) // …the pinned view computes the aggregate over the at-G record set. const pinnedRows = await db.find({ aggregate: 'order_totals' }) const pinnedInvoice = pinnedRows.find((row) => (row.metadata as { category: string }).category === 'invoice') expect((pinnedInvoice?.metadata as { total: number }).total).toBe(30) expect((pinnedInvoice?.metadata as { orders: number }).orders).toBe(2) await db.release() }) it('asOf() pins serve the materialized surface; the build is cached per Db; release() closes the ephemeral reader', async () => { const brain = await openMemoryBrain() // Two one-hot vectors — orthogonal, fully deterministic neighbors. const axisA = Array.from({ length: 384 }, (_, i) => (i === 0 ? 1 : 0)) const axisB = Array.from({ length: 384 }, (_, i) => (i === 1 ? 1 : 0)) const tx = await brain.transact([ { op: 'add', id: uid('asof-mat-a'), type: NounType.Document, data: 'a', vector: axisA, metadata: {} }, { op: 'add', id: uid('asof-mat-b'), type: NounType.Document, data: 'b', vector: axisB, metadata: {} } ]) const pinnedGeneration = tx.generation await tx.release() // Swap the vectors after the pin point, then open the past via asOf(). await brain.transact([ { op: 'update', id: uid('asof-mat-a'), vector: axisB }, { op: 'update', id: uid('asof-mat-b'), vector: axisA } ]) const db = await brain.asOf(pinnedGeneration) // The asOf() view serves index-accelerated queries at its pinned // generation — same materialized surface as a now() pin. expect((await brain.find({ vector: axisA, limit: 1 }))[0].id).toBe(uid('asof-mat-b')) expect((await db.find({ vector: axisA, limit: 1 }))[0].id).toBe(uid('asof-mat-a')) // The O(n at G) build runs ONCE per Db: the second index query reuses // the cached handle (typed access to the private cache, mirroring // generationStoreOf above). const internals = db as unknown as { materializedHandle?: Promise } expect(internals.materializedHandle).toBeDefined() const handle = await internals.materializedHandle! expect((await db.find({ vector: axisB, limit: 1 }))[0].id).toBe(uid('asof-mat-b')) expect(await internals.materializedHandle).toBe(handle) // release() frees the materialization by CLOSING the ephemeral reader — // the reader itself refuses reads afterwards, proving its indexes and // timers were torn down (not merely hidden behind the released-Db guard). await db.release() await db.release() // idempotent await expect(db.find({ vector: axisA, limit: 1 })).rejects.toThrow('released Db') await expect(handle.find({ vector: axisA, limit: 1 })).rejects.toThrow('not initialized') }) it('speculative overlays keep the one honest boundary: index queries and persist throw SpeculativeOverlayError', async () => { const brain = await openMemoryBrain() await brain.transact([ { op: 'add', id: uid('spec-e'), type: NounType.Document, data: 'x', vector: vec(95), metadata: { v: 1 } } ]) const db = brain.now() const spec = await db.with([{ op: 'update', id: uid('spec-e'), metadata: { v: 2 } }]) // Metadata reads on the overlay are fully supported… expect(((await spec.get(uid('spec-e')))?.metadata as { v: number }).v).toBe(2) expect((await spec.find({ type: NounType.Document, where: { v: 2 } })).length).toBe(1) // …index-accelerated dimensions and persist throw the named error. await expect(spec.find({ query: 'anything' })).rejects.toThrow(SpeculativeOverlayError) await expect(spec.find({ vector: vec(95) })).rejects.toThrow(SpeculativeOverlayError) await expect(spec.find({ connected: { from: uid('spec-e') } })).rejects.toThrow( SpeculativeOverlayError ) await expect(spec.persist(path.join(makeTempDir(), 'overlay-snap'))).rejects.toThrow( SpeculativeOverlayError ) // The non-speculative base view is unaffected by the overlay boundary. expect(((await db.get(uid('spec-e')))?.metadata as { v: number }).v).toBe(1) await spec.release() await db.release() }) it('restore() requires confirmation and replaces state from a snapshot (generation floor preserved)', async () => { const { brain } = await openFsBrain() await brain.transact([ { op: 'add', id: uid('restore-e'), type: NounType.Document, data: 'keep', vector: vec(96), metadata: { v: 1 } } ]) const snapDir = path.join(makeTempDir(), 'restore-snapshot') const db = brain.now() await db.persist(snapDir) await db.release() await brain.transact([{ op: 'update', id: uid('restore-e'), metadata: { v: 2 } }]) await brain.transact([ { op: 'add', id: uid('restore-extra'), type: NounType.Document, data: 'extra', vector: vec(97), metadata: {} } ]) const beforeRestore = brain.generation() await expect( brain.restore(snapDir, { confirm: false }) ).rejects.toThrow('confirm') await brain.restore(snapDir, { confirm: true }) expect(((await brain.get(uid('restore-e')))?.metadata as { v: number }).v).toBe(1) expect(await brain.get(uid('restore-extra'))).toBeNull() expect((await brain.find({ type: NounType.Document, limit: 10 })).length).toBe(1) // The counter never moves backwards — pre-restore generations are not reissued. expect(brain.generation()).toBeGreaterThanOrEqual(beforeRestore) const after = await brain.transact([{ op: 'update', id: uid('restore-e'), metadata: { v: 9 } }]) expect(after.generation).toBeGreaterThan(beforeRestore) await after.release() }) it('restore() rebuilds graph adjacency — relationships survive a snapshot round-trip (id-mapper reloaded before graph)', async () => { const { brain } = await openFsBrain() const a = uid('rr-a') const b = uid('rr-b') const c = uid('rr-c') await brain.transact([ { op: 'add', id: a, type: NounType.Person, data: 'a', vector: vec(1), metadata: {} }, { op: 'add', id: b, type: NounType.Person, data: 'b', vector: vec(2), metadata: {} }, { op: 'add', id: c, type: NounType.Document, data: 'c', vector: vec(3), metadata: {} }, { op: 'relate', from: a, to: b, type: VerbType.RelatedTo }, { op: 'relate', from: a, to: c, type: VerbType.References } ]) expect((await brain.related(a)).map((r) => r.to).sort()).toEqual([b, c].sort()) const snapDir = path.join(makeTempDir(), 'rel-snapshot') const db = brain.now() await db.persist(snapDir) await db.release() // Drop a + its edges live so the restore must genuinely RECONSTRUCT them. await brain.remove(a) expect(await brain.related(a)).toEqual([]) await brain.restore(snapDir, { confirm: true }) // a's relationships are back: the graph adjacency rebuilt by resolving each // verb's endpoints (sourceId/targetId → ints) through the id-mapper that // restore() reloaded from the snapshot BEFORE graphIndex.rebuild(). Without // that reload, a native adjacency keyed on those ints loses the edges. expect((await brain.related(a)).map((r) => r.to).sort()).toEqual([b, c].sort()) }) it('Db values are first-class: Brainy.open() + chained with() overlays compose', async () => { const brain = await Brainy.open({ requireSubtype: false, storage: { type: 'memory' } }) brains.push(brain) expect(brain.isInitialized).toBe(true) await brain.transact([ { op: 'add', id: uid('chain-e'), type: NounType.Document, data: 'x', vector: vec(98), metadata: { v: 1 } } ]) const db = brain.now() const spec1 = await db.with([{ op: 'update', id: uid('chain-e'), metadata: { v: 2 } }]) const spec2 = await spec1.with([{ op: 'update', id: uid('chain-e'), metadata: { v: 3 } }]) expect(((await db.get(uid('chain-e')))?.metadata as { v: number }).v).toBe(1) expect(((await spec1.get(uid('chain-e')))?.metadata as { v: number }).v).toBe(2) expect(((await spec2.get(uid('chain-e')))?.metadata as { v: number }).v).toBe(3) expect(spec2).toBeInstanceOf(Db) await spec2.release() await spec1.release() await db.release() }) it('transactionLog() includes single-op AND transact generations, newest first, with meta and limit', async () => { const brain = await openMemoryBrain() // Model-B: a single-op write is its OWN generation and IS logged (no meta — // tx metadata is a transact()-only concept). It is generation 1 on a fresh // brain (init-time infrastructure writes are the un-versioned gen-0 baseline). await brain.add({ id: uid('txlog-solo'), type: NounType.Document, data: 'solo', vector: vec(99), subtype: 'note' }) const soloLog = await brain.transactionLog() expect(soloLog.map((entry) => entry.generation)).toEqual([1]) expect(soloLog[0].meta).toBeUndefined() const soloGen = 1 const first = await brain.transact( [{ op: 'add', id: uid('txlog-a'), type: NounType.Document, data: 'a', vector: vec(100), metadata: {} }], { meta: { author: 'job-1' } } ) const second = await brain.transact( [{ op: 'update', id: uid('txlog-a'), metadata: { v: 2 } }], { meta: { author: 'job-2' } } ) const third = await brain.transact([{ op: 'update', id: uid('txlog-a'), metadata: { v: 3 } }]) const entries = await brain.transactionLog() // Newest first: the three transacts, then the single-op solo write (gen 1). expect(entries.map((entry) => entry.generation)).toEqual([ third.generation, second.generation, first.generation, soloGen ]) expect(entries[1].meta).toEqual({ author: 'job-2' }) expect(entries[2].meta).toEqual({ author: 'job-1' }) expect(entries[0].meta).toBeUndefined() // third() had no meta expect(entries[3].meta).toBeUndefined() // the single-op solo write carries no meta for (const entry of entries) { expect(entry.timestamp).toBeGreaterThan(0) } const limited = await brain.transactionLog({ limit: 2 }) expect(limited.map((entry) => entry.generation)).toEqual([third.generation, second.generation]) await first.release() await second.release() await third.release() }) // ========================================================================== // 8b. Model-B single-op generation-stamping (every write versioned) // ========================================================================== it('Model-B — a now() pin freezes against single-op add/update/remove (the Model-A hole is closed)', async () => { const brain = await openMemoryBrain() const a = uid('mb-a') const b = uid('mb-b') // Seed `a` with a single-op add, then pin the current generation. await brain.add({ id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }) const pin = brain.now() // Single-op mutations AFTER the pin: update `a`, add `b`. await brain.update({ id: a, metadata: { v: 2 } }) await brain.add({ id: b, type: NounType.Document, data: 'b', vector: vec(2), metadata: { v: 1 } }) // The pin is frozen at its generation — single-op writes do not leak through // it (pre-Model-B, the pin tracked the live single-op mutation). expect((await pin.get(a))?.metadata?.v).toBe(1) expect(await pin.get(b)).toBeNull() expect((await brain.get(a))?.metadata?.v).toBe(2) expect((await brain.get(b))?.metadata?.v).toBe(1) expect(pin.isHistorical()).toBe(true) // A single-op REMOVE also leaves the pin untouched. await brain.remove(a) expect((await pin.get(a))?.metadata?.v).toBe(1) expect(await brain.get(a)).toBeNull() await pin.release() }) it('Model-B — historical find() overlays an un-flushed single-op write (overlay bound is generation, not committed)', async () => { const brain = await openMemoryBrain() const a = uid('ov-a') const b = uid('ov-b') await ( await brain.transact([ { op: 'add', id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }, { op: 'add', id: b, type: NounType.Document, data: 'b', vector: vec(2), metadata: { v: 1 } } ]) ).release() const at1 = await brain.asOf(1) // A single-op REMOVE of `b` lands AFTER the pin and is NOT flushed (pending). await brain.remove(b) const liveIds = (await brain.find({})).map((r) => r.id) const pastIds = (await at1.find({})).map((r) => r.id) // Live: `b` is gone. Historical (pinned at gen 1): the un-flushed removal is // overlaid out, so `b` is still present at its pinned state. expect(liveIds).toContain(a) expect(liveIds).not.toContain(b) expect(pastIds).toContain(a) expect(pastIds).toContain(b) await at1.release() }) it('Model-B retention — explicit caps reclaim single-op history; committed history survives reopen', async () => { const { brain, dir } = await openFsBrain() const a = uid('ret-a') await brain.add({ id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } }) for (let v = 2; v <= 6; v++) await brain.update({ id: a, metadata: { v } }) await brain.flush() // persist the per-write generations to disk expect(brain.generation()).toBe(6) // Cap to the 2 most recent generations — older single-op history is reclaimed. const res = await brain.compactHistory({ maxGenerations: 2 }) expect(res.removedGenerations).toBeGreaterThan(0) expect(res.horizon).toBeGreaterThan(0) await brain.close() // Reopen: the survivors + live value are intact; below-horizon asOf throws. const { brain: reopened } = await openFsBrain(dir) expect((await reopened.get(a))?.metadata?.v).toBe(6) await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) }) it('Model-B retention — setRetentionBudget drives adaptive reclaim on flush; live data intact', async () => { // Default brain → ADAPTIVE retention. A coordinator (e.g. cor's ResourceManager) // pushes a byte budget via setRetentionBudget(); auto-compaction on flush() reclaims // oldest history down toward it. Each update's before-image carries the full prior // 384-dim vector (~KBs), so ~13 generations far exceed a few-KB budget. const { brain } = await openFsBrain() const a = uid('ret-budget') await brain.add({ id: a, type: NounType.Document, data: 'v0', vector: vec(1), metadata: { v: 0 } }) for (let v = 1; v <= 12; v++) await brain.update({ id: a, metadata: { v } }) brain.setRetentionBudget(6000) // ~6 KB — well below the accumulated history await brain.flush() // group-commit + adaptive auto-compaction under the budget // History was reclaimed (the horizon advanced past the oldest generations)… expect(generationStoreOf(brain).horizon()).toBeGreaterThan(0) await expect(brain.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError) // …but the budget reclaims HISTORY only — the live record is untouched. expect((await brain.get(a))?.metadata?.v).toBe(12) }) // ========================================================================== // 9. asOf() / released-Db error paths (Y.15 spot-checks) // ========================================================================== it('proof 9 — asOf() rejects out-of-range targets and a released Db throws on every read', async () => { const brain = await openMemoryBrain() await ( await brain.transact([ { op: 'add', id: uid('asof-err'), type: NounType.Document, data: 'x', metadata: { v: 1 } } ]) ).release() const current = brain.generation() // Negative, non-integer, and future generations are honest RangeErrors — // never a silent clamp to a neighbouring generation. await expect(brain.asOf(-1)).rejects.toThrow(RangeError) await expect(brain.asOf(1.5)).rejects.toThrow(RangeError) await expect(brain.asOf(current + 999)).rejects.toThrow(RangeError) // A string that is not a snapshot directory is a descriptive error, not a crash. await expect(brain.asOf('/no/such/snapshot/dir')).rejects.toThrow(/not a snapshot directory/) // Use-after-release is rejected on every read method, naming the released generation. const db = brain.now() await db.release() await expect(db.get(uid('asof-err'))).rejects.toThrow(/released Db/) await expect(db.find({ where: { v: 1 } })).rejects.toThrow(/released Db/) await expect(db.related(uid('asof-err'))).rejects.toThrow(/released Db/) }) })