David Snelling
8f93add705
The COW version-control surface (fork, branches, checkout, commit,
getHistory/streamHistory, asOfCommit, brain.versions) is gone, along with
its subsystems: src/versioning/, the COW object store (CommitLog,
CommitObject, RefManager, TreeObject), HistoricalStorageAdapter, and the
TypeAwareHNSWIndex path it kept alive. The Db API (now/transact/asOf/
with/persist/restore) is the one versioning model in 8.0.
Survivors and replacements:
- BlobStorage survives (the VFS stores file content through it), relocated
to src/storage/blobStorage.ts with binaryDataCodec.ts; its adapter
interface is now BlobStoreAdapter, slimmed to the consumed surface
(write/read/has/delete/getMetadata + MIME-aware compression policy).
- brain.migrate() backup branches are replaced by persist-before-migrate:
MigrateOptions.backupTo persists a hard-link snapshot of the current
generation before any transform runs; MigrationResult.backupPath reports
it, and brain.restore(path) brings it back wholesale.
- CLI: cow.ts (fork/branch/checkout/history/migrate) is replaced by
snapshot.ts — snapshot <path>, restore <path>, history (tx-log),
generation.
- New public read API: brain.transactionLog({limit}) exposes the reified
tx-log (generation/timestamp/meta, newest first) that backs the CLI
history command; TxLogEntry is exported.
Tests: superseded suites deleted; fork/commit blocks excised from shared
suites; BlobStorage tests relocated + reworked against the slimmed store;
migration tests now prove the backupTo snapshot/restore round trip; new
transactionLog coverage in db-mvcc.
993 lines
37 KiB
TypeScript
993 lines
37 KiB
TypeScript
/**
|
|
* @module tests/integration/db-mvcc
|
|
* @description Credibility-bar proof suite for Brainy 8.0's generational MVCC
|
|
* storage core + Datomic-style Db API. Each test PROVES a stated guarantee
|
|
* (not merely exercises the API):
|
|
*
|
|
* 1. Isolation — a pinned Db reads its pinned state exactly, across 200
|
|
* subsequent transact mutations (updates + removes).
|
|
* 2. Atomicity — a batch with a failing later operation applies NOTHING
|
|
* (generation unchanged, no partial records), for both plan-time and
|
|
* execution-phase failures (the latter via real fault injection into the
|
|
* storage adapter, exercising the production rollback path).
|
|
* 3. CAS — `ifAtGeneration` commits when fresh, throws
|
|
* `GenerationConflictError` with correct expected/actual when stale.
|
|
* 4. Snapshot integrity — persist → mutate source heavily → `Brainy.load()`
|
|
* is byte-identical to the pre-mutation state (hard-link safety), and an
|
|
* in-memory brain persists to a loadable directory.
|
|
* 5. Compaction safety — pinned reads stay correct across `compactHistory()`;
|
|
* after release + compact the superseded record-sets are reclaimed.
|
|
* 6. with() — speculative overlays are visible on the overlay Db only; the
|
|
* underlying brain and the original Db are untouched.
|
|
* 7. Generation monotonicity across close/reopen.
|
|
* 8. Crash consistency — a simulated crash between record staging and the
|
|
* manifest rename (the commit point) recovers to the exact pre-transaction
|
|
* state on reopen, through the REAL production recovery path.
|
|
* 9. Versioned provider wiring — a provider implementing the 4-method
|
|
* `VersionedIndexProvider` contract receives balanced pin/release (and a
|
|
* visibility consult per pin) as Db values are created and released.
|
|
*
|
|
* All entities carry explicit vectors so the suite never invokes the
|
|
* embedding engine (semantic search is exercised only for its documented
|
|
* historical-generation error).
|
|
*/
|
|
|
|
import { describe, it, expect, afterEach, vi } from 'vitest'
|
|
import * as fs from 'node:fs'
|
|
import * as os from 'node:os'
|
|
import * as path from 'node:path'
|
|
import { Brainy } from '../../src/brainy.js'
|
|
import { Db } from '../../src/db/db.js'
|
|
import {
|
|
GenerationCompactedError,
|
|
GenerationConflictError,
|
|
NotYetSupportedAtHistoricalGenerationError
|
|
} from '../../src/db/errors.js'
|
|
import type { GenerationStore } from '../../src/db/generationStore.js'
|
|
import { GraphAdjacencyIndex } from '../../src/graph/graphAdjacencyIndex.js'
|
|
import { RevisionConflictError } from '../../src/transaction/RevisionConflictError.js'
|
|
import type { StorageAdapter } from '../../src/coreTypes.js'
|
|
import { NounType, VerbType } from '../../src/types/graphTypes.js'
|
|
|
|
/** Deterministic 384-dim vector so no test ever invokes the embedder. */
|
|
function vec(seed: number): number[] {
|
|
return Array.from({ length: 384 }, (_, i) => ((seed * 31 + i * 7) % 100) / 100)
|
|
}
|
|
|
|
/**
|
|
* Map a readable label to a deterministic UUID-shaped id (entity ids must be
|
|
* UUIDs — the sharded storage layout derives the shard from the UUID hex).
|
|
* Two independent FNV-style mixes keep 7+ distinct labels collision-free.
|
|
*/
|
|
function uid(label: string): string {
|
|
let h1 = 0x811c9dc5
|
|
for (let i = 0; i < label.length; i++) {
|
|
h1 = Math.imul(h1 ^ label.charCodeAt(i), 0x01000193) >>> 0
|
|
}
|
|
let h2 = 0xdeadbeef
|
|
for (let i = label.length - 1; i >= 0; i--) {
|
|
h2 = Math.imul(h2 ^ label.charCodeAt(i), 0x85ebca6b) >>> 0
|
|
}
|
|
const hex = h1.toString(16).padStart(8, '0') + h2.toString(16).padStart(8, '0')
|
|
return `00000000-0000-4000-8000-${hex.slice(0, 12)}`
|
|
}
|
|
|
|
/** Typed access to the brain's private generation store (test injection point). */
|
|
function generationStoreOf(brain: Brainy): GenerationStore {
|
|
return (brain as unknown as { generationStore: GenerationStore }).generationStore
|
|
}
|
|
|
|
describe('8.0 Db API — generational MVCC', () => {
|
|
const brains: Brainy[] = []
|
|
const tempDirs: string[] = []
|
|
|
|
/** Track a brain for afterEach teardown. */
|
|
async function openMemoryBrain(): Promise<Brainy> {
|
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
await brain.init()
|
|
brains.push(brain)
|
|
return brain
|
|
}
|
|
|
|
/** Open (and track) a filesystem brain rooted at a fresh temp directory. */
|
|
async function openFsBrain(dir?: string): Promise<{ brain: Brainy; dir: string }> {
|
|
const rootDirectory = dir ?? makeTempDir()
|
|
const brain = new Brainy({
|
|
requireSubtype: false,
|
|
storage: { type: 'filesystem', rootDirectory }
|
|
})
|
|
await brain.init()
|
|
brains.push(brain)
|
|
return { brain, dir: rootDirectory }
|
|
}
|
|
|
|
function makeTempDir(): string {
|
|
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-db-mvcc-'))
|
|
tempDirs.push(dir)
|
|
return dir
|
|
}
|
|
|
|
afterEach(async () => {
|
|
for (const brain of brains.splice(0)) {
|
|
try {
|
|
await brain.close()
|
|
} catch {
|
|
// already closed by the test
|
|
}
|
|
}
|
|
for (const dir of tempDirs.splice(0)) {
|
|
await fs.promises.rm(dir, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 1. Isolation
|
|
// ==========================================================================
|
|
it('proof 1 — a pinned Db reads its pinned state exactly across 200 transact mutations', async () => {
|
|
const brain = await openMemoryBrain()
|
|
|
|
const ids = Array.from({ length: 20 }, (_, i) => uid(`iso-e${i}`))
|
|
await brain.transact(
|
|
ids.map((id, i) => ({
|
|
op: 'add' as const,
|
|
id,
|
|
type: NounType.Document,
|
|
data: `doc-${i}`,
|
|
vector: vec(i),
|
|
metadata: { v: 0, slot: i }
|
|
}))
|
|
)
|
|
|
|
const db = brain.now()
|
|
const pinned = new Map<string, unknown>()
|
|
for (const id of ids) {
|
|
pinned.set(id, await db.get(id))
|
|
expect(pinned.get(id)).not.toBeNull()
|
|
}
|
|
const pinnedFind = await db.find({ type: NounType.Document, limit: 100 })
|
|
expect(pinnedFind.length).toBe(20)
|
|
|
|
// 200 mutations: 198 updates over the first 18 ids + 2 removes.
|
|
for (let i = 0; i < 200; i++) {
|
|
if (i === 100) {
|
|
await brain.transact([{ op: 'remove', id: uid('iso-e18') }])
|
|
} else if (i === 150) {
|
|
await brain.transact([{ op: 'remove', id: uid('iso-e19') }])
|
|
} else {
|
|
await brain.transact([
|
|
{ op: 'update', id: ids[i % 18], metadata: { v: i + 1, touchedAt: i } }
|
|
])
|
|
}
|
|
}
|
|
|
|
// Live state moved…
|
|
const liveUpdated = await brain.get(ids[0])
|
|
expect((liveUpdated?.metadata as { v: number }).v).toBeGreaterThan(0)
|
|
expect(await brain.get(uid('iso-e18'))).toBeNull()
|
|
expect(await brain.get(uid('iso-e19'))).toBeNull()
|
|
|
|
// …but EVERY read through the pinned Db is exactly the pinned state,
|
|
// including the two entities removed after the pin.
|
|
for (const id of ids) {
|
|
expect(await db.get(id)).toEqual(pinned.get(id))
|
|
}
|
|
|
|
// Set-level isolation: metadata find() at the pin still sees all 20.
|
|
const histFind = await db.find({ type: NounType.Document, limit: 100 })
|
|
expect(histFind.length).toBe(20)
|
|
for (const row of histFind) {
|
|
expect((row.metadata as { v: number }).v).toBe(0)
|
|
}
|
|
|
|
await db.release()
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 2. Atomicity
|
|
// ==========================================================================
|
|
it('proof 2a — a plan-time failure in a later op applies nothing', async () => {
|
|
const brain = await openMemoryBrain()
|
|
const g0 = brain.generation()
|
|
|
|
await expect(
|
|
brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('atomic-a'),
|
|
type: NounType.Document,
|
|
data: 'a',
|
|
vector: vec(1),
|
|
metadata: { ok: true }
|
|
},
|
|
{ op: 'update', id: uid('never-created'), metadata: { boom: true } }
|
|
])
|
|
).rejects.toThrow(`Entity ${uid('never-created')} not found`)
|
|
|
|
expect(brain.generation()).toBe(g0)
|
|
expect(await brain.get(uid('atomic-a'))).toBeNull()
|
|
expect((await brain.find({ type: NounType.Document, limit: 10 })).length).toBe(0)
|
|
})
|
|
|
|
it('proof 2b — an execution-phase failure rolls back every applied op (generation unchanged, zero partial records)', async () => {
|
|
const brain = await openMemoryBrain()
|
|
|
|
await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('atomic-base'),
|
|
type: NounType.Document,
|
|
data: 'base',
|
|
vector: vec(0),
|
|
metadata: { base: true }
|
|
}
|
|
])
|
|
const g0 = brain.generation()
|
|
const storage = brain.storageAdapter
|
|
|
|
// Real fault injection: the verb-metadata write of the LAST operation
|
|
// fails once, after the earlier add operations have already executed —
|
|
// exercising the production TransactionManager rollback path.
|
|
const spy = vi
|
|
.spyOn(storage, 'saveVerbMetadata')
|
|
.mockImplementationOnce(async () => {
|
|
throw new Error('disk full (injected)')
|
|
})
|
|
|
|
await expect(
|
|
brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('atomic-b'),
|
|
type: NounType.Person,
|
|
data: 'b',
|
|
vector: vec(2),
|
|
metadata: { name: 'b' }
|
|
},
|
|
{
|
|
op: 'relate',
|
|
from: uid('atomic-b'),
|
|
to: uid('atomic-base'),
|
|
type: VerbType.RelatedTo
|
|
}
|
|
])
|
|
).rejects.toThrow('disk full (injected)')
|
|
spy.mockRestore()
|
|
|
|
// ZERO ops applied: generation unchanged, the add rolled back, no edge.
|
|
expect(brain.generation()).toBe(g0)
|
|
expect(await brain.get(uid('atomic-b'))).toBeNull()
|
|
expect((await brain.getRelations({ from: uid('atomic-b') })).length).toBe(0)
|
|
expect((await brain.find({ type: NounType.Person, limit: 10 })).length).toBe(0)
|
|
|
|
// No partial generation records: the staging directory was removed.
|
|
const records = await storage.listRawObjects(`_generations/${g0 + 1}`)
|
|
expect(records).toEqual([])
|
|
|
|
// The store remains fully writable and consistent after the rollback.
|
|
const db = await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('atomic-c'),
|
|
type: NounType.Person,
|
|
data: 'c',
|
|
vector: vec(3),
|
|
metadata: { name: 'c' }
|
|
}
|
|
])
|
|
expect(db.generation).toBe(g0 + 1)
|
|
await db.release()
|
|
})
|
|
|
|
it('proof 2c — an ifRev conflict on a later update rejects the whole batch', async () => {
|
|
const brain = await openMemoryBrain()
|
|
await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('rev-e'),
|
|
type: NounType.Document,
|
|
data: 'rev',
|
|
vector: vec(4),
|
|
metadata: { v: 1 }
|
|
}
|
|
])
|
|
await brain.update({ id: uid('rev-e'), metadata: { v: 2 } }) // _rev is now 2
|
|
const g0 = brain.generation()
|
|
|
|
await expect(
|
|
brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('rev-new'),
|
|
type: NounType.Document,
|
|
data: 'new',
|
|
vector: vec(5),
|
|
metadata: {}
|
|
},
|
|
{ op: 'update', id: uid('rev-e'), metadata: { v: 3 }, ifRev: 1 } // stale
|
|
])
|
|
).rejects.toThrow(RevisionConflictError)
|
|
|
|
expect(brain.generation()).toBe(g0)
|
|
expect(await brain.get(uid('rev-new'))).toBeNull()
|
|
expect(((await brain.get(uid('rev-e')))?.metadata as { v: number }).v).toBe(2)
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 3. CAS (ifAtGeneration)
|
|
// ==========================================================================
|
|
it('proof 3 — ifAtGeneration commits when fresh and conflicts when stale', async () => {
|
|
const brain = await openMemoryBrain()
|
|
const db = brain.now()
|
|
|
|
const committed = await brain.transact(
|
|
[
|
|
{
|
|
op: 'add',
|
|
id: uid('cas-e'),
|
|
type: NounType.Document,
|
|
data: 'cas',
|
|
vector: vec(6),
|
|
metadata: { n: 1 }
|
|
}
|
|
],
|
|
{ ifAtGeneration: db.generation }
|
|
)
|
|
expect(committed.generation).toBe(db.generation + 1)
|
|
|
|
try {
|
|
await brain.transact(
|
|
[{ op: 'update', id: uid('cas-e'), metadata: { n: 2 } }],
|
|
{ ifAtGeneration: db.generation } // stale — committed moved past it
|
|
)
|
|
expect.unreachable('should have thrown GenerationConflictError')
|
|
} catch (err) {
|
|
expect(err).toBeInstanceOf(GenerationConflictError)
|
|
expect((err as GenerationConflictError).expected).toBe(db.generation)
|
|
expect((err as GenerationConflictError).actual).toBe(committed.generation)
|
|
}
|
|
|
|
// Nothing committed by the conflicting attempt.
|
|
expect(((await brain.get(uid('cas-e')))?.metadata as { n: number }).n).toBe(1)
|
|
expect(brain.generation()).toBe(committed.generation)
|
|
|
|
await db.release()
|
|
await committed.release()
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 4. Snapshot integrity (persist → mutate → load)
|
|
// ==========================================================================
|
|
it('proof 4a — a persisted snapshot is immune to heavy source mutation (hard-link safety)', async () => {
|
|
const { brain } = await openFsBrain()
|
|
|
|
const ids = Array.from({ length: 10 }, (_, i) => uid(`snap-e${i}`))
|
|
await brain.transact(
|
|
ids.map((id, i) => ({
|
|
op: 'add' as const,
|
|
id,
|
|
type: NounType.Document,
|
|
data: `snapshot-doc-${i}`,
|
|
vector: vec(i + 10),
|
|
metadata: { generation: 'pre', slot: i }
|
|
}))
|
|
)
|
|
const relDb = await brain.transact([
|
|
{ op: 'relate', from: ids[0], to: ids[1], type: VerbType.References },
|
|
{ op: 'relate', from: ids[1], to: ids[2], type: VerbType.References }
|
|
])
|
|
await relDb.release()
|
|
|
|
// Capture pre-mutation state, then snapshot.
|
|
const preState = new Map<string, unknown>()
|
|
for (const id of ids) {
|
|
preState.set(id, await brain.get(id))
|
|
}
|
|
const preRelations = await brain.getRelations({ from: ids[0] })
|
|
expect(preRelations.length).toBe(1)
|
|
|
|
const snapDir = path.join(makeTempDir(), 'snapshot')
|
|
const db = brain.now()
|
|
await db.persist(snapDir)
|
|
await db.release()
|
|
|
|
// Mutate the source heavily: rewrite every entity, delete some, add new.
|
|
for (const [i, id] of ids.entries()) {
|
|
await brain.transact([
|
|
{ op: 'update', id, data: `mutated-${i}`, metadata: { generation: 'post' } }
|
|
])
|
|
}
|
|
await brain.transact([{ op: 'remove', id: ids[0] }])
|
|
await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('snap-after'),
|
|
type: NounType.Document,
|
|
data: 'added after snapshot',
|
|
vector: vec(99),
|
|
metadata: { generation: 'post' }
|
|
}
|
|
])
|
|
await brain.flush()
|
|
|
|
// The snapshot opens as a self-contained store with the PRE state.
|
|
const loaded = await Brainy.load(snapDir)
|
|
for (const id of ids) {
|
|
const entity = await loaded.get(id)
|
|
expect(entity).toEqual(preState.get(id))
|
|
}
|
|
expect(await loaded.get(uid('snap-after'))).toBeNull()
|
|
const loadedFind = await loaded.find({ type: NounType.Document, limit: 100 })
|
|
expect(loadedFind.length).toBe(10)
|
|
for (const row of loadedFind) {
|
|
expect((row.metadata as { generation: string }).generation).toBe('pre')
|
|
}
|
|
const loadedRelations = await loaded.related({ from: ids[0] })
|
|
expect(loadedRelations.length).toBe(1)
|
|
expect(loadedRelations[0].to).toBe(ids[1])
|
|
await loaded.release()
|
|
|
|
// And asOf(path) is the instance-method route to the same snapshot.
|
|
const viaAsOf = await brain.asOf(snapDir)
|
|
expect(((await viaAsOf.get(ids[3])) as { metadata: { generation: string } }).metadata.generation).toBe('pre')
|
|
await viaAsOf.release()
|
|
})
|
|
|
|
it('proof 4b — an in-memory brain persists to a directory loadable as a real store', async () => {
|
|
const brain = await openMemoryBrain()
|
|
await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('mem-e1'),
|
|
type: NounType.Document,
|
|
data: 'memory-persisted',
|
|
vector: vec(21),
|
|
metadata: { from: 'memory' }
|
|
}
|
|
])
|
|
|
|
const snapDir = path.join(makeTempDir(), 'memory-snapshot')
|
|
const db = brain.now()
|
|
await db.persist(snapDir)
|
|
await db.release()
|
|
|
|
const loaded = await Brainy.load(snapDir)
|
|
const entity = await loaded.get(uid('mem-e1'))
|
|
expect(entity).not.toBeNull()
|
|
expect((entity?.metadata as { from: string }).from).toBe('memory')
|
|
await loaded.release()
|
|
})
|
|
|
|
it('proof 4c — persist() refuses a view that history has moved past (honest boundary)', async () => {
|
|
const brain = await openMemoryBrain()
|
|
const db = brain.now()
|
|
await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('stale-persist'),
|
|
type: NounType.Document,
|
|
data: 'x',
|
|
vector: vec(30),
|
|
metadata: {}
|
|
}
|
|
])
|
|
|
|
const snapDir = path.join(makeTempDir(), 'stale-snapshot')
|
|
await expect(db.persist(snapDir)).rejects.toThrow(
|
|
NotYetSupportedAtHistoricalGenerationError
|
|
)
|
|
await db.release()
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 5. Compaction safety
|
|
// ==========================================================================
|
|
it('proof 5 — compaction never breaks pinned reads; after release the records are reclaimed', async () => {
|
|
const { brain } = await openFsBrain()
|
|
const storage = brain.storageAdapter
|
|
|
|
// Every transact() returns a PINNED Db — release the ones this test
|
|
// does not keep, so compaction eligibility is determined solely by `db`.
|
|
await (
|
|
await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('compact-e'),
|
|
type: NounType.Document,
|
|
data: 'v1',
|
|
vector: vec(40),
|
|
metadata: { v: 1 }
|
|
}
|
|
])
|
|
).release()
|
|
await (
|
|
await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 2 } }])
|
|
).release()
|
|
const db = brain.now() // pinned at v2
|
|
const pinnedEntity = await db.get(uid('compact-e'))
|
|
await (
|
|
await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 3 } }])
|
|
).release()
|
|
await (
|
|
await brain.transact([{ op: 'update', id: uid('compact-e'), metadata: { v: 4 } }])
|
|
).release()
|
|
|
|
const recordsBefore = (await storage.listRawObjects('_generations')).length
|
|
expect(recordsBefore).toBeGreaterThan(0)
|
|
|
|
// Compact while pinned: record-sets above the pin survive, pinned reads stay correct.
|
|
const first = await brain.compactHistory()
|
|
expect(await db.get(uid('compact-e'))).toEqual(pinnedEntity)
|
|
expect(((await db.get(uid('compact-e')))?.metadata as { v: number }).v).toBe(2)
|
|
|
|
// After release, everything is reclaimable.
|
|
await db.release()
|
|
const second = await brain.compactHistory()
|
|
expect(first.removedGenerations + second.removedGenerations).toBeGreaterThan(0)
|
|
|
|
const recordsAfter = (await storage.listRawObjects('_generations')).length
|
|
expect(recordsAfter).toBeLessThan(recordsBefore)
|
|
expect(recordsAfter).toBe(0)
|
|
|
|
// Compacted generations are honestly unreachable.
|
|
await expect(brain.asOf(1)).rejects.toThrow(GenerationCompactedError)
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 6. with() — speculative overlays
|
|
// ==========================================================================
|
|
it('proof 6 — with() overlays are visible only on the overlay Db; brain and base Db untouched', async () => {
|
|
const brain = await openMemoryBrain()
|
|
await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('with-a'),
|
|
type: NounType.Person,
|
|
data: 'Ada',
|
|
vector: vec(50),
|
|
metadata: { v: 1 }
|
|
},
|
|
{
|
|
op: 'add',
|
|
id: uid('with-b'),
|
|
type: NounType.Person,
|
|
data: 'Grace',
|
|
vector: vec(51),
|
|
metadata: { v: 1 }
|
|
}
|
|
])
|
|
const g0 = brain.generation()
|
|
const db = brain.now()
|
|
|
|
const spec = await db.with([
|
|
{ op: 'update', id: uid('with-a'), metadata: { v: 2 } },
|
|
{
|
|
op: 'add',
|
|
id: uid('with-new'),
|
|
type: NounType.Project,
|
|
data: 'Apollo',
|
|
metadata: { speculative: true }
|
|
},
|
|
{ op: 'remove', id: uid('with-b') },
|
|
{ op: 'relate', from: uid('with-a'), to: uid('with-new'), type: VerbType.WorksOn }
|
|
])
|
|
|
|
// The overlay view sees everything…
|
|
expect(spec.speculative).toBe(true)
|
|
expect(((await spec.get(uid('with-a')))?.metadata as { v: number }).v).toBe(2)
|
|
expect(await spec.get(uid('with-new'))).not.toBeNull()
|
|
expect(await spec.get(uid('with-b'))).toBeNull()
|
|
const specEdges = await spec.related({ from: uid('with-a') })
|
|
expect(specEdges.length).toBe(1)
|
|
expect(specEdges[0].to).toBe(uid('with-new'))
|
|
const specFind = await spec.find({ type: NounType.Project, limit: 10 })
|
|
expect(specFind.length).toBe(1)
|
|
|
|
// …the base Db sees none of it…
|
|
expect(db.speculative).toBe(false)
|
|
expect(((await db.get(uid('with-a')))?.metadata as { v: number }).v).toBe(1)
|
|
expect(await db.get(uid('with-new'))).toBeNull()
|
|
expect(await db.get(uid('with-b'))).not.toBeNull()
|
|
|
|
// …and the brain (disk, indexes, generation counter) is untouched.
|
|
expect(brain.generation()).toBe(g0)
|
|
expect(((await brain.get(uid('with-a')))?.metadata as { v: number }).v).toBe(1)
|
|
expect(await brain.get(uid('with-new'))).toBeNull()
|
|
expect((await brain.getRelations({ from: uid('with-a') })).length).toBe(0)
|
|
|
|
await spec.release()
|
|
await db.release()
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 7. Generation monotonicity across close/reopen
|
|
// ==========================================================================
|
|
it('proof 7 — the generation counter never moves backwards across close/reopen', async () => {
|
|
const dir = makeTempDir()
|
|
const { brain: first } = await openFsBrain(dir)
|
|
|
|
await first.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('mono-e'),
|
|
type: NounType.Document,
|
|
data: 'v1',
|
|
vector: vec(60),
|
|
metadata: { v: 1 }
|
|
}
|
|
])
|
|
await first.transact([{ op: 'update', id: uid('mono-e'), metadata: { v: 2 } }])
|
|
// Single-operation write — bumps the counter outside transact.
|
|
await first.update({ id: uid('mono-e'), metadata: { v: 3 } })
|
|
const beforeClose = first.generation()
|
|
expect(beforeClose).toBeGreaterThanOrEqual(3)
|
|
await first.close()
|
|
|
|
const { brain: second } = await openFsBrain(dir)
|
|
expect(second.generation()).toBeGreaterThanOrEqual(beforeClose)
|
|
|
|
const db = await second.transact([{ op: 'update', id: uid('mono-e'), metadata: { v: 4 } }])
|
|
expect(db.generation).toBeGreaterThan(beforeClose)
|
|
await db.release()
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 8. Crash consistency
|
|
// ==========================================================================
|
|
it('proof 8 — a crash before the manifest rename recovers to the exact pre-transaction state', async () => {
|
|
const dir = makeTempDir()
|
|
const { brain: first } = await openFsBrain(dir)
|
|
|
|
await first.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('crash-e'),
|
|
type: NounType.Document,
|
|
data: 'stable',
|
|
vector: vec(70),
|
|
metadata: { v: 1 }
|
|
}
|
|
])
|
|
const committedGen = first.generation()
|
|
|
|
// Simulated crash at the worst point of the REAL commit protocol: every
|
|
// record is staged and the batch has executed, but the atomic manifest
|
|
// rename — the commit point — never happens. The throwing injector skips
|
|
// the abort cleanup exactly as a dead process would.
|
|
generationStoreOf(first).setCommitFaultInjector((phase) => {
|
|
if (phase === 'before-manifest-rename') {
|
|
throw new Error('simulated process crash')
|
|
}
|
|
})
|
|
await expect(
|
|
first.transact([
|
|
{ op: 'update', id: uid('crash-e'), metadata: { v: 2 } },
|
|
{
|
|
op: 'add',
|
|
id: uid('crash-new'),
|
|
type: NounType.Document,
|
|
data: 'uncommitted',
|
|
vector: vec(71),
|
|
metadata: {}
|
|
}
|
|
])
|
|
).rejects.toThrow('simulated process crash')
|
|
generationStoreOf(first).setCommitFaultInjector(undefined)
|
|
|
|
// The in-process state is dirty (batch executed) — close flushes it all,
|
|
// the realistic worst case for the recovery path.
|
|
await first.close()
|
|
|
|
// Reopen: recovery rolls the uncommitted generation back and rebuilds
|
|
// the indexes from the repaired records.
|
|
const { brain: second } = await openFsBrain(dir)
|
|
const recovered = await second.get(uid('crash-e'))
|
|
expect((recovered?.metadata as { v: number }).v).toBe(1)
|
|
expect(await second.get(uid('crash-new'))).toBeNull()
|
|
const found = await second.find({ type: NounType.Document, limit: 10 })
|
|
expect(found.length).toBe(1)
|
|
expect((found[0].metadata as { v: number }).v).toBe(1)
|
|
|
|
// The crashed generation number is never reissued.
|
|
expect(second.generation()).toBeGreaterThanOrEqual(committedGen + 1)
|
|
const db = await second.transact([{ op: 'update', id: uid('crash-e'), metadata: { v: 5 } }])
|
|
expect(db.generation).toBeGreaterThan(committedGen + 1)
|
|
await db.release()
|
|
})
|
|
|
|
// ==========================================================================
|
|
// 9. Versioned provider wiring
|
|
// ==========================================================================
|
|
it('proof 9 — a VersionedIndexProvider receives balanced pin/release as Db values live and die', async () => {
|
|
const calls = {
|
|
pins: [] as bigint[],
|
|
releases: [] as bigint[],
|
|
visibility: [] as bigint[]
|
|
}
|
|
|
|
const brain = new Brainy({ requireSubtype: false, storage: { type: 'memory' } })
|
|
brain.use({
|
|
name: 'test-versioned-graph-index',
|
|
activate: async (ctx) => {
|
|
ctx.registerProvider('graphIndex', (storage: StorageAdapter) =>
|
|
Object.assign(new GraphAdjacencyIndex(storage), {
|
|
generation: () => 0n,
|
|
isGenerationVisible: (g: bigint) => {
|
|
calls.visibility.push(g)
|
|
return true
|
|
},
|
|
pin: (g: bigint) => {
|
|
calls.pins.push(g)
|
|
},
|
|
release: (g: bigint) => {
|
|
calls.releases.push(g)
|
|
}
|
|
})
|
|
)
|
|
return true
|
|
}
|
|
})
|
|
await brain.init()
|
|
brains.push(brain)
|
|
|
|
const base = brain.generation()
|
|
const db1 = brain.now()
|
|
expect(calls.pins).toEqual([BigInt(base)])
|
|
expect(calls.visibility).toEqual([BigInt(base)]) // consulted at pin time
|
|
|
|
const db2 = await brain.transact([
|
|
{
|
|
op: 'add',
|
|
id: uid('vp-e'),
|
|
type: NounType.Document,
|
|
data: 'pinned',
|
|
vector: vec(80),
|
|
metadata: {}
|
|
}
|
|
])
|
|
expect(calls.pins).toEqual([BigInt(base), BigInt(base + 1)])
|
|
|
|
const db3 = await brain.asOf(base)
|
|
expect(calls.pins).toEqual([BigInt(base), BigInt(base + 1), BigInt(base)])
|
|
|
|
// A with() overlay takes its own pin on the same generation.
|
|
const spec = await db2.with([{ op: 'update', id: uid('vp-e'), metadata: { spec: true } }])
|
|
expect(calls.pins.length).toBe(4)
|
|
|
|
await db1.release()
|
|
await db2.release()
|
|
await db3.release()
|
|
await spec.release()
|
|
await spec.release() // idempotent — must NOT double-release the pin
|
|
|
|
expect(calls.releases.length).toBe(calls.pins.length)
|
|
expect([...calls.releases].sort()).toEqual([...calls.pins].sort())
|
|
})
|
|
|
|
// ==========================================================================
|
|
// Supporting guarantees: asOf, since, receipts, tx-log, honest errors
|
|
// ==========================================================================
|
|
it('asOf(generation) and asOf(Date) resolve historical state; future/negative are rejected', async () => {
|
|
const brain = await openMemoryBrain()
|
|
|
|
const tx1 = await brain.transact(
|
|
[
|
|
{
|
|
op: 'add',
|
|
id: uid('asof-e'),
|
|
type: NounType.Document,
|
|
data: 'v1',
|
|
vector: vec(90),
|
|
metadata: { v: 1 }
|
|
}
|
|
],
|
|
{ meta: { reason: 'first commit' } }
|
|
)
|
|
await new Promise((resolve) => setTimeout(resolve, 5))
|
|
const betweenCommits = new Date()
|
|
await new Promise((resolve) => setTimeout(resolve, 5))
|
|
await brain.transact([{ op: 'update', id: uid('asof-e'), metadata: { v: 2 } }])
|
|
|
|
// By generation.
|
|
const atFirst = await brain.asOf(tx1.generation)
|
|
expect(((await atFirst.get(uid('asof-e')))?.metadata as { v: number }).v).toBe(1)
|
|
expect(atFirst.timestamp).toBe(tx1.timestamp)
|
|
await atFirst.release()
|
|
|
|
// By timestamp: resolves to the newest commit at-or-before the instant.
|
|
const atInstant = await brain.asOf(betweenCommits)
|
|
expect(atInstant.generation).toBe(tx1.generation)
|
|
expect(((await atInstant.get(uid('asof-e')))?.metadata as { v: number }).v).toBe(1)
|
|
await atInstant.release()
|
|
|
|
// Honest range errors.
|
|
await expect(brain.asOf(brain.generation() + 100)).rejects.toThrow(RangeError)
|
|
await expect(brain.asOf(-1)).rejects.toThrow(RangeError)
|
|
await expect(brain.asOf('/no/such/snapshot/dir')).rejects.toThrow('not a snapshot directory')
|
|
|
|
// The transact meta was reified into the tx-log.
|
|
const lines = await brain.storageAdapter.readTxLogLines()
|
|
const entries = lines.map((line) => JSON.parse(line))
|
|
const first = entries.find((entry) => entry.generation === tx1.generation)
|
|
expect(first?.meta).toEqual({ reason: 'first commit' })
|
|
|
|
await tx1.release()
|
|
})
|
|
|
|
it('receipts resolve ids in input order, including intra-batch references and relate dedupe', async () => {
|
|
const brain = await openMemoryBrain()
|
|
|
|
const db = await brain.transact([
|
|
{ op: 'add', id: uid('rcpt-aa'), type: NounType.Person, data: 'a', vector: vec(91), metadata: {} },
|
|
{ op: 'add', id: uid('rcpt-bb'), type: NounType.Person, data: 'b', vector: vec(92), metadata: {} },
|
|
{ op: 'relate', from: uid('rcpt-aa'), to: uid('rcpt-bb'), type: VerbType.Knows },
|
|
{ op: 'update', id: uid('rcpt-bb'), metadata: { seen: true } }
|
|
])
|
|
expect(db.receipt).toBeDefined()
|
|
expect(db.receipt!.generation).toBe(db.generation)
|
|
expect(db.receipt!.ids.length).toBe(4)
|
|
expect(db.receipt!.ids[0]).toBe(uid('rcpt-aa'))
|
|
expect(db.receipt!.ids[1]).toBe(uid('rcpt-bb'))
|
|
expect(db.receipt!.ids[3]).toBe(uid('rcpt-bb'))
|
|
const relationId = db.receipt!.ids[2]
|
|
expect((await brain.getRelations({ from: uid('rcpt-aa') }))[0].id).toBe(relationId)
|
|
|
|
// Duplicate relate (same from/to/type) dedupes to the SAME id — both
|
|
// within a batch and against committed state, mirroring relate().
|
|
const dup = await brain.transact([
|
|
{ op: 'relate', from: uid('rcpt-aa'), to: uid('rcpt-bb'), type: VerbType.Knows }
|
|
])
|
|
expect(dup.receipt!.ids[0]).toBe(relationId)
|
|
expect((await brain.getRelations({ from: uid('rcpt-aa') })).length).toBe(1)
|
|
|
|
await db.release()
|
|
await dup.release()
|
|
})
|
|
|
|
it('since() reports the ids changed between two pinned views', async () => {
|
|
const brain = await openMemoryBrain()
|
|
await brain.transact([
|
|
{ op: 'add', id: uid('since-a'), type: NounType.Document, data: 'a', vector: vec(93), metadata: {} }
|
|
])
|
|
const before = brain.now()
|
|
|
|
await brain.transact([
|
|
{ op: 'add', id: uid('since-b'), type: NounType.Document, data: 'b', vector: vec(94), metadata: {} },
|
|
{ op: 'relate', from: uid('since-a'), to: uid('since-b'), type: VerbType.References }
|
|
])
|
|
await brain.transact([{ op: 'update', id: uid('since-a'), metadata: { touched: true } }])
|
|
const after = brain.now()
|
|
|
|
const changed = await after.since(before)
|
|
expect([...changed.nouns].sort()).toEqual([uid('since-a'), uid('since-b')].sort())
|
|
expect(changed.verbs.length).toBe(1)
|
|
expect(changed.fromGeneration).toBe(before.generation)
|
|
expect(changed.toGeneration).toBe(after.generation)
|
|
|
|
// Direction is enforced.
|
|
await expect(before.since(after)).rejects.toThrow(RangeError)
|
|
|
|
await before.release()
|
|
await after.release()
|
|
})
|
|
|
|
it('index-accelerated queries at a historical generation throw the documented error — never silently-wrong results', async () => {
|
|
const brain = await openMemoryBrain()
|
|
await brain.transact([
|
|
{ op: 'add', id: uid('hist-e'), type: NounType.Document, data: 'x', vector: vec(95), metadata: { v: 1 } }
|
|
])
|
|
const db = brain.now()
|
|
await brain.transact([{ op: 'update', id: uid('hist-e'), metadata: { v: 2 } }])
|
|
|
|
// Storage-record reads remain fully supported…
|
|
expect(((await db.get(uid('hist-e')))?.metadata as { v: number }).v).toBe(1)
|
|
expect((await db.find({ type: NounType.Document, where: { v: 1 } })).length).toBe(1)
|
|
|
|
// …index-accelerated dimensions throw the documented, named error.
|
|
await expect(db.search('anything')).rejects.toThrow(
|
|
NotYetSupportedAtHistoricalGenerationError
|
|
)
|
|
await expect(db.find({ vector: vec(95) })).rejects.toThrow(
|
|
NotYetSupportedAtHistoricalGenerationError
|
|
)
|
|
await expect(db.find({ connected: { from: uid('hist-e') } })).rejects.toThrow(
|
|
NotYetSupportedAtHistoricalGenerationError
|
|
)
|
|
|
|
// Released views refuse all reads.
|
|
await db.release()
|
|
await expect(db.get(uid('hist-e'))).rejects.toThrow('released Db')
|
|
})
|
|
|
|
it('restore() requires confirmation and replaces state from a snapshot (generation floor preserved)', async () => {
|
|
const { brain } = await openFsBrain()
|
|
await brain.transact([
|
|
{ op: 'add', id: uid('restore-e'), type: NounType.Document, data: 'keep', vector: vec(96), metadata: { v: 1 } }
|
|
])
|
|
|
|
const snapDir = path.join(makeTempDir(), 'restore-snapshot')
|
|
const db = brain.now()
|
|
await db.persist(snapDir)
|
|
await db.release()
|
|
|
|
await brain.transact([{ op: 'update', id: uid('restore-e'), metadata: { v: 2 } }])
|
|
await brain.transact([
|
|
{ op: 'add', id: uid('restore-extra'), type: NounType.Document, data: 'extra', vector: vec(97), metadata: {} }
|
|
])
|
|
const beforeRestore = brain.generation()
|
|
|
|
await expect(
|
|
brain.restore(snapDir, { confirm: false })
|
|
).rejects.toThrow('confirm')
|
|
|
|
await brain.restore(snapDir, { confirm: true })
|
|
expect(((await brain.get(uid('restore-e')))?.metadata as { v: number }).v).toBe(1)
|
|
expect(await brain.get(uid('restore-extra'))).toBeNull()
|
|
expect((await brain.find({ type: NounType.Document, limit: 10 })).length).toBe(1)
|
|
|
|
// The counter never moves backwards — pre-restore generations are not reissued.
|
|
expect(brain.generation()).toBeGreaterThanOrEqual(beforeRestore)
|
|
const after = await brain.transact([{ op: 'update', id: uid('restore-e'), metadata: { v: 9 } }])
|
|
expect(after.generation).toBeGreaterThan(beforeRestore)
|
|
await after.release()
|
|
})
|
|
|
|
it('Db values are first-class: Brainy.open() + chained with() overlays compose', async () => {
|
|
const brain = await Brainy.open({ requireSubtype: false, storage: { type: 'memory' } })
|
|
brains.push(brain)
|
|
expect(brain.isInitialized).toBe(true)
|
|
|
|
await brain.transact([
|
|
{ op: 'add', id: uid('chain-e'), type: NounType.Document, data: 'x', vector: vec(98), metadata: { v: 1 } }
|
|
])
|
|
const db = brain.now()
|
|
const spec1 = await db.with([{ op: 'update', id: uid('chain-e'), metadata: { v: 2 } }])
|
|
const spec2 = await spec1.with([{ op: 'update', id: uid('chain-e'), metadata: { v: 3 } }])
|
|
|
|
expect(((await db.get(uid('chain-e')))?.metadata as { v: number }).v).toBe(1)
|
|
expect(((await spec1.get(uid('chain-e')))?.metadata as { v: number }).v).toBe(2)
|
|
expect(((await spec2.get(uid('chain-e')))?.metadata as { v: number }).v).toBe(3)
|
|
expect(spec2).toBeInstanceOf(Db)
|
|
|
|
await spec2.release()
|
|
await spec1.release()
|
|
await db.release()
|
|
})
|
|
|
|
it('transactionLog() returns committed entries newest first, with meta and limit', async () => {
|
|
const brain = await openMemoryBrain()
|
|
|
|
// Nothing committed yet — the log is empty (single-op writes do not log).
|
|
await brain.add({ id: uid('txlog-solo'), type: NounType.Document, data: 'solo', vector: vec(99), subtype: 'note' })
|
|
expect(await brain.transactionLog()).toEqual([])
|
|
|
|
const first = await brain.transact(
|
|
[{ op: 'add', id: uid('txlog-a'), type: NounType.Document, data: 'a', vector: vec(100), metadata: {} }],
|
|
{ meta: { author: 'job-1' } }
|
|
)
|
|
const second = await brain.transact(
|
|
[{ op: 'update', id: uid('txlog-a'), metadata: { v: 2 } }],
|
|
{ meta: { author: 'job-2' } }
|
|
)
|
|
const third = await brain.transact([{ op: 'update', id: uid('txlog-a'), metadata: { v: 3 } }])
|
|
|
|
const entries = await brain.transactionLog()
|
|
expect(entries.map((entry) => entry.generation)).toEqual([
|
|
third.generation,
|
|
second.generation,
|
|
first.generation
|
|
])
|
|
expect(entries[1].meta).toEqual({ author: 'job-2' })
|
|
expect(entries[2].meta).toEqual({ author: 'job-1' })
|
|
expect(entries[0].meta).toBeUndefined()
|
|
for (const entry of entries) {
|
|
expect(entry.timestamp).toBeGreaterThan(0)
|
|
}
|
|
|
|
const limited = await brain.transactionLog({ limit: 2 })
|
|
expect(limited.map((entry) => entry.generation)).toEqual([third.generation, second.generation])
|
|
|
|
await first.release()
|
|
await second.release()
|
|
await third.release()
|
|
})
|
|
})
|