brainy/tests/unit/db/db-portable-graph.test.ts

550 lines
23 KiB
TypeScript

/**
* Unit tests for the 8.0 portable graph backup/restore surface:
* db.export() / brain.export() (read, at a pinned generation)
* brain.import(backup) (write, one atomic transaction; polymorphic)
*
* Real round-trips against in-memory storage (no mocks of the API under test),
* including the 8.0-specific compositions (asOf/with export) and the polymorphic
* import() dispatch. Entities carry subtypes so the suite passes under 8.0's
* subtype-required default.
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { randomUUID } from 'node:crypto'
import * as fs from 'node:fs/promises'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../../src/brainy'
import { createTestConfig } from '../../helpers/test-factory'
import { NounType, VerbType } from '../../../src/types/graphTypes'
import { validatePortableGraph } from '../../../src/db/portableGraph'
import type { PortableGraph } from '../../../src/db/portableGraph'
import { CanonicalEnumerationUnavailableError } from '../../../src/db/errors'
describe('8.0 portable graph export/import (PortableGraph v1)', () => {
let brain: Brainy
beforeEach(async () => {
brain = new Brainy(createTestConfig())
await brain.init()
})
afterEach(async () => {
await brain.close()
})
describe('format + round-trip', () => {
it('brain.export() produces a versioned PortableGraph document', async () => {
const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' })
const b = await brain.add({ data: 'Acme', type: NounType.Organization, subtype: 'vendor' })
await brain.relate({ from: a, to: b, type: VerbType.WorksWith, subtype: 'full-time' })
const backup = await brain.export()
expect(backup.format).toBe('brainy-portable-graph')
expect(backup.formatVersion).toBe(1)
expect(backup.entities.map((e) => e.id).sort()).toEqual([a, b].sort())
expect(backup.relations).toHaveLength(1)
expect(backup.entities.find((e) => e.id === a)?.subtype).toBe('employee')
expect(backup.relations[0].subtype).toBe('full-time')
})
it('round-trips into a second brain via polymorphic brain.import()', async () => {
const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' })
const b = await brain.add({ data: 'Bob', type: NounType.Person, subtype: 'employee' })
await brain.relate({ from: a, to: b, type: VerbType.FriendOf, subtype: 'close' })
const backup = await brain.export({}, { includeVectors: true })
const target = new Brainy(createTestConfig())
await target.init()
try {
const result = await target.import(backup)
expect(result.imported).toBe(2)
expect(result.errors).toHaveLength(0)
expect((await target.get(a))?.id).toBe(a)
const rels = await target.related({ from: a })
expect(rels.some((r) => r.to === b && r.type === VerbType.FriendOf)).toBe(true)
} finally {
await target.close()
}
})
})
describe('selectors', () => {
it('ids — exactly the requested entities', async () => {
const a = await brain.add({ data: 'A', type: NounType.Thing, subtype: 'x' })
const b = await brain.add({ data: 'B', type: NounType.Thing, subtype: 'x' })
await brain.add({ data: 'C', type: NounType.Thing, subtype: 'x' })
const backup = await brain.export({ ids: [a, b] })
expect(backup.entities.map((e) => e.id).sort()).toEqual([a, b].sort())
})
it('collection — collection + transitive Contains members', async () => {
const root = await brain.add({ data: 'Folder', type: NounType.Collection, subtype: 'dir' })
const c1 = await brain.add({ data: 'C1', type: NounType.Document, subtype: 'doc' })
const c2 = await brain.add({ data: 'C2', type: NounType.Document, subtype: 'doc' })
await brain.add({ data: 'Outside', type: NounType.Document, subtype: 'doc' })
await brain.relate({ from: root, to: c1, type: VerbType.Contains, subtype: 'has' })
await brain.relate({ from: c1, to: c2, type: VerbType.Contains, subtype: 'has' })
const backup = await brain.export({ collection: root })
expect(backup.entities.map((e) => e.id).sort()).toEqual([root, c1, c2].sort())
})
it('induced edges only by default', async () => {
const a = await brain.add({ data: 'A', type: NounType.Thing, subtype: 'x' })
const b = await brain.add({ data: 'B', type: NounType.Thing, subtype: 'x' })
const c = await brain.add({ data: 'C', type: NounType.Thing, subtype: 'x' })
await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'r' })
await brain.relate({ from: b, to: c, type: VerbType.RelatedTo, subtype: 'r' })
const backup = await brain.export({ ids: [a, b] })
expect(backup.relations).toHaveLength(1)
expect(backup.relations[0].from).toBe(a)
})
})
describe('vectors + conflict', () => {
it('omits vectors by default and re-embeds on import', async () => {
const a = await brain.add({ data: 'Re-embed me', type: NounType.Thing, subtype: 'x' })
const backup = await brain.export({ ids: [a] })
expect(backup.entities[0].vector).toBeUndefined()
const target = new Brainy(createTestConfig())
await target.init()
try {
const result = await target.import(backup)
expect(result.reembedded).toBe(1)
expect((await target.get(a, { includeVectors: true }))?.vector?.length).toBeGreaterThan(0)
} finally {
await target.close()
}
})
it('onConflict:merge updates an existing entity in place', async () => {
const a = await brain.add({ data: 'V1', type: NounType.Thing, subtype: 'x', metadata: { n: 1 } })
const backup = await brain.export({ ids: [a] }, { includeVectors: true })
await brain.update({ id: a, metadata: { n: 99 }, merge: false })
const result = await brain.import(backup, { onConflict: 'merge' })
expect(result.merged).toBe(1)
expect((await brain.get(a))?.metadata?.n).toBe(1)
})
})
describe('8.0 composition — export on the immutable Db', () => {
it('asOf(generation).export() is a time-travel export', async () => {
// Only transact() advances the committed generation (add() does not), so
// distinct generations are created via transact for a true time-travel pin.
const a = '22222222-2222-4222-8222-222222222222'
const b = '33333333-3333-4333-8333-333333333333'
await brain.transact([{ op: 'add', id: a, data: 'First', type: NounType.Thing, subtype: 'x' }])
const g1 = brain.generation()
await brain.transact([{ op: 'add', id: b, data: 'Second', type: NounType.Thing, subtype: 'x' }])
const past = await brain.asOf(g1)
try {
// Whole-brain export at g1 enumerates via the generation-correct find() path.
const backup = await past.export()
const ids = backup.entities.map((e) => e.id)
expect(ids).toContain(a)
expect(ids).not.toContain(b) // b was committed in a later generation
} finally {
await past.release()
}
})
it('now().with(ops).export() is a what-if export', async () => {
const a = await brain.add({ data: 'Real', type: NounType.Thing, subtype: 'x' })
const speculativeId = '11111111-1111-4111-8111-111111111111'
const view = await brain.now().with([
{ op: 'add', id: speculativeId, data: 'Speculative', type: NounType.Thing, subtype: 'x' }
])
try {
const backup = await view.export({ ids: [a, speculativeId] })
expect(backup.entities.map((e) => e.id).sort()).toEqual([a, speculativeId].sort())
} finally {
await view.release()
}
// The speculative entity never touched the durable brain.
expect(await brain.get(speculativeId)).toBeNull()
})
})
describe('import validation', () => {
it('rejects a newer formatVersion', async () => {
const future: PortableGraph = {
format: 'brainy-portable-graph',
formatVersion: 999,
brainyVersion: 'x',
createdAt: new Date().toISOString(),
embedding: { model: 'm', dimensions: 384 },
entities: [],
relations: [],
stats: { entityCount: 0, relationCount: 0, blobCount: 0 }
}
await expect(brain.import(future)).rejects.toThrow(/formatVersion/)
})
})
describe('id remapping (clone)', () => {
it('imports a subgraph under fresh ids (a copy, not a move)', async () => {
const a = await brain.add({ data: 'A', type: NounType.Thing, subtype: 'x' })
const b = await brain.add({ data: 'B', type: NounType.Thing, subtype: 'x' })
await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'r' })
const backup = await brain.export({ ids: [a, b] }, { includeVectors: true })
const remap = new Map<string, string>([
[a, randomUUID()],
[b, randomUUID()]
])
const result = await brain.import(backup, { remapIds: (id) => remap.get(id) ?? id })
expect(result.imported).toBe(2)
expect((await brain.get(remap.get(a)!))?.id).toBe(remap.get(a))
expect((await brain.get(a))?.id).toBe(a) // original still present
const rels = await brain.related({ from: remap.get(a)! })
expect(rels.some((r) => r.to === remap.get(b))).toBe(true)
})
})
describe('validatePortableGraph() — dry-run check', () => {
const valid = (): PortableGraph => ({
format: 'brainy-portable-graph',
formatVersion: 1,
brainyVersion: 'x',
createdAt: new Date().toISOString(),
embedding: { model: 'all-MiniLM-L6-v2', dimensions: 384 },
entities: [{ id: 'a', type: NounType.Thing }],
relations: [],
stats: { entityCount: 1, relationCount: 0, blobCount: 0, vectorDimensions: 384 }
})
it('accepts a well-formed backup', () => {
const r = validatePortableGraph(valid())
expect(r.valid).toBe(true)
expect(r.errors).toHaveLength(0)
})
it('rejects a non-PortableGraph value', () => {
const r = validatePortableGraph({ entities: [] })
expect(r.valid).toBe(false)
expect(r.errors[0]).toMatch(/PortableGraph/)
})
it('rejects a newer formatVersion', () => {
const r = validatePortableGraph({ ...valid(), formatVersion: 999 })
expect(r.valid).toBe(false)
expect(r.errors.some((e) => /formatVersion/.test(e))).toBe(true)
})
it('flags duplicate entity ids as an error', () => {
const b = valid()
b.entities = [
{ id: 'dup', type: NounType.Thing },
{ id: 'dup', type: NounType.Thing }
]
const r = validatePortableGraph(b)
expect(r.valid).toBe(false)
expect(r.errors.some((e) => /duplicate/.test(e))).toBe(true)
})
it('warns (not errors) on a relation endpoint missing from entities', () => {
const b = valid()
b.relations = [{ id: 'r1', from: 'a', to: 'missing', type: VerbType.RelatedTo }]
const r = validatePortableGraph(b)
expect(r.valid).toBe(true) // a dangling endpoint is a warning, not a hard error
expect(r.warnings.some((w) => /missing/.test(w))).toBe(true)
})
})
})
describe('8.0 export includeContent (VFS blobs, filesystem)', () => {
let dir: string
let brain: Brainy
beforeEach(async () => {
dir = await fs.mkdtemp(path.join(os.tmpdir(), 'brainy-blob-'))
brain = new Brainy({ storage: { type: 'filesystem', path: dir } })
await brain.init()
})
afterEach(async () => {
await brain.close()
await fs.rm(dir, { recursive: true, force: true })
})
it('captures VFS file bytes in blobs and writes them on import', async () => {
await brain.vfs.writeFile('/hello.txt', 'Hello blobs')
const backup = await brain.export(
{ vfsPath: '/hello.txt' },
{ includeVectors: true, includeContent: true }
)
expect(backup.stats.blobCount).toBeGreaterThanOrEqual(1)
const b64 = Object.values(backup.blobs ?? {})[0]
expect(Buffer.from(b64, 'base64').toString()).toBe('Hello blobs')
const dir2 = await fs.mkdtemp(path.join(os.tmpdir(), 'brainy-blob2-'))
const target = new Brainy({ storage: { type: 'filesystem', path: dir2 } })
await target.init()
try {
const result = await target.import(backup)
expect(result.blobsWritten).toBeGreaterThanOrEqual(1)
expect(result.imported).toBeGreaterThanOrEqual(1)
} finally {
await target.close()
await fs.rm(dir2, { recursive: true, force: true })
}
})
})
describe('8.0 export enumeration:"canonical" — canon-complete against index blindness', () => {
let brain: Brainy
beforeEach(async () => {
brain = new Brainy(createTestConfig())
await brain.init()
})
afterEach(async () => {
await brain.close()
})
it('(i) equals the index-based export when the index is healthy — same entity ids, relations, vectors', async () => {
const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' })
const b = await brain.add({ data: 'Bob', type: NounType.Person, subtype: 'employee' })
const c = await brain.add({ data: 'Acme', type: NounType.Organization, subtype: 'vendor' })
await brain.relate({ from: a, to: b, type: VerbType.FriendOf, subtype: 'close' })
await brain.relate({ from: a, to: c, type: VerbType.WorksWith, subtype: 'full-time' })
const indexExport = await brain.export({}, { includeVectors: true, enumeration: 'index' })
const canonicalExport = await brain.export({}, { includeVectors: true, enumeration: 'canonical' })
expect(canonicalExport.entities.map((e) => e.id).sort()).toEqual(
indexExport.entities.map((e) => e.id).sort()
)
expect(canonicalExport.relations.map((r) => r.id).sort()).toEqual(
indexExport.relations.map((r) => r.id).sort()
)
expect(canonicalExport.entities.map((e) => e.id).sort()).toEqual([a, b, c].sort())
for (const e of canonicalExport.entities) {
expect(e.vector?.length).toBeGreaterThan(0)
}
expect(canonicalExport.drift).toBeUndefined() // reportIndexDrift not requested
})
it('(ii) survives simulated metadata-index blindness; the index export misses the record; drift names it canonicalOnly', async () => {
const staff = await brain.add({
data: 'Staff',
type: NounType.Person,
subtype: 'employee',
metadata: { role: 'staff' }
})
const other = await brain.add({
data: 'Other',
type: NounType.Person,
subtype: 'employee',
metadata: { role: 'staff' }
})
// Surgically poison the metadata index (the lowest-level seam the existing
// find() phantom-row guard tests use — see find-index-integrity-guard.test.ts,
// which does the mirror-image ADD case) so the predicate query
// enumeration:'index' issues (find({ type: Person })) never returns `staff` —
// a real canonical record the index has lost track of, the exact
// canon-present/index-missing state canonical mode exists to survive.
const mi = (brain as any).metadataIndex
const original = mi.getIdsForFilter.bind(mi)
mi.getIdsForFilter = async (filter: any, opts?: any): Promise<string[]> => {
const ids: string[] = await original(filter, opts)
return ids.filter((id: string) => id !== staff)
}
try {
const indexExport = await brain.export({ type: NounType.Person }, { enumeration: 'index' })
expect(indexExport.entities.map((e) => e.id)).not.toContain(staff)
expect(indexExport.entities.map((e) => e.id)).toContain(other)
const canonicalExport = await brain.export(
{ type: NounType.Person },
{ enumeration: 'canonical', reportIndexDrift: true }
)
expect(canonicalExport.entities.map((e) => e.id)).toContain(staff)
expect(canonicalExport.entities.map((e) => e.id)).toContain(other)
expect(canonicalExport.drift?.canonicalOnly).toEqual([staff])
expect(canonicalExport.drift?.indexOnly).toEqual([])
} finally {
mi.getIdsForFilter = original
}
})
it('(iii) drift report shape + loud console.warn only when nonzero', async () => {
const staff = await brain.add({ data: 'Staff', type: NounType.Person, subtype: 'employee' })
await brain.add({ data: 'Other', type: NounType.Person, subtype: 'employee' })
const mi = (brain as any).metadataIndex
const original = mi.getIdsForFilter.bind(mi)
mi.getIdsForFilter = async (filter: any, opts?: any): Promise<string[]> => {
const ids: string[] = await original(filter, opts)
return ids.filter((id: string) => id !== staff)
}
const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
try {
const drifted = await brain.export(
{ type: NounType.Person },
{ enumeration: 'canonical', reportIndexDrift: true }
)
expect(drifted.drift).toEqual({ canonicalOnly: [staff], indexOnly: [] })
expect(warnSpy).toHaveBeenCalledTimes(1)
expect(warnSpy.mock.calls[0].join(' ')).toMatch(/drift/i)
} finally {
mi.getIdsForFilter = original
warnSpy.mockClear()
}
// Healthy index: drift is reported (both lists present) but never warned about.
try {
const healthy = await brain.export(
{ type: NounType.Person },
{ enumeration: 'canonical', reportIndexDrift: true }
)
expect(healthy.drift).toEqual({ canonicalOnly: [], indexOnly: [] })
expect(warnSpy).not.toHaveBeenCalled()
} finally {
warnSpy.mockRestore()
}
})
it('(iv) throws CanonicalEnumerationUnavailableError on a historical asOf() view and a speculative with() overlay', async () => {
const a = '22222222-2222-4222-8222-222222222222'
const b = '33333333-3333-4333-8333-333333333333'
await brain.transact([{ op: 'add', id: a, data: 'First', type: NounType.Thing, subtype: 'x' }])
const g1 = brain.generation()
await brain.transact([{ op: 'add', id: b, data: 'Second', type: NounType.Thing, subtype: 'x' }])
const past = await brain.asOf(g1)
try {
await expect(past.export({}, { enumeration: 'canonical' })).rejects.toThrow(
CanonicalEnumerationUnavailableError
)
// The default (index) mode is unaffected — still a valid time-travel export.
const backup = await past.export()
expect(backup.entities.map((e) => e.id)).toContain(a)
} finally {
await past.release()
}
const speculativeId = '11111111-1111-4111-8111-111111111111'
const view = await brain.now().with([
{ op: 'add', id: speculativeId, data: 'Speculative', type: NounType.Thing, subtype: 'x' }
])
try {
await expect(view.export({}, { enumeration: 'canonical' })).rejects.toThrow(
CanonicalEnumerationUnavailableError
)
} finally {
await view.release()
}
})
it('throws a plain Error when enumeration:"canonical" has no storage adapter to walk', async () => {
const { exportGraph } = await import('../../../src/db/portableGraph')
const readerOnly = { get: async () => null, find: async () => [], related: async () => [] }
await expect(
exportGraph(readerOnly as any, undefined, {}, { enumeration: 'canonical' })
).rejects.toThrow(/enumeration:'canonical' requires a storage adapter/)
})
})
describe('8.0 export includeHidden — every visibility tier for migration-grade canon completeness', () => {
// The fixed-id VFS root Brainy.init() always creates is the one 'system'-visibility
// entity a consumer can rely on existing (visibility:'system' is not settable via the
// public add() API — "intentionally not accepted", per AddParams.visibility's doc).
const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000'
let brain: Brainy
beforeEach(async () => {
brain = new Brainy(createTestConfig())
await brain.init()
})
afterEach(async () => {
await brain.close()
})
it('canonical + includeHidden carries an internal row AND the system row; round-trips through import', async () => {
const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
const internalId = await brain.add({
data: 'Internal',
type: NounType.Thing,
subtype: 'x',
visibility: 'internal'
})
const migrationExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true })
const ids = migrationExport.entities.map((e) => e.id)
expect(ids).toContain(publicId)
expect(ids).toContain(internalId)
expect(ids).toContain(VFS_ROOT_ID)
expect(migrationExport.entities.find((e) => e.id === internalId)?.visibility).toBe('internal')
expect(migrationExport.entities.find((e) => e.id === VFS_ROOT_ID)?.visibility).toBe('system')
const target = new Brainy(createTestConfig())
await target.init()
try {
const result = await target.import(migrationExport)
expect(result.errors).toHaveLength(0)
expect((await target.get(internalId))?.visibility).toBe('internal')
} finally {
await target.close()
}
})
it('default export (includeHidden omitted) still excludes both hidden tiers — pins today\'s behavior', async () => {
const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
const internalId = await brain.add({
data: 'Internal',
type: NounType.Thing,
subtype: 'x',
visibility: 'internal'
})
for (const opts of [{ enumeration: 'index' as const }, { enumeration: 'canonical' as const }]) {
const backup = await brain.export({}, opts)
const ids = backup.entities.map((e) => e.id)
expect(ids).toContain(publicId)
expect(ids).not.toContain(internalId)
expect(ids).not.toContain(VFS_ROOT_ID)
}
})
it('index mode + includeHidden also reaches both tiers — find() takes includeInternal + includeSystem in one pass', async () => {
const publicId = await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
const internalId = await brain.add({
data: 'Internal',
type: NounType.Thing,
subtype: 'x',
visibility: 'internal'
})
const indexExport = await brain.export({}, { enumeration: 'index', includeHidden: true })
const canonicalExport = await brain.export({}, { enumeration: 'canonical', includeHidden: true })
const indexIds = indexExport.entities.map((e) => e.id).sort()
const canonicalIds = canonicalExport.entities.map((e) => e.id).sort()
expect(indexIds).toEqual(canonicalIds)
expect(indexIds).toContain(publicId)
expect(indexIds).toContain(internalId)
expect(indexIds).toContain(VFS_ROOT_ID)
})
it('drift stays pure under includeHidden — no tier-policy noise when the index is healthy', async () => {
await brain.add({ data: 'Public', type: NounType.Thing, subtype: 'x' })
await brain.add({ data: 'Internal', type: NounType.Thing, subtype: 'x', visibility: 'internal' })
const audited = await brain.export(
{},
{ enumeration: 'canonical', includeHidden: true, reportIndexDrift: true }
)
expect(audited.drift).toEqual({ canonicalOnly: [], indexOnly: [] })
})
})