diff --git a/src/brainy.ts b/src/brainy.ts index 63398c99..bd24e4e4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -115,6 +115,15 @@ import type { AggregateDefinition, AggregateQueryParams, AggregateResult } from import { resolveJsHnswConfig, DEFAULT_RECALL } from './utils/recallPreset.js' import * as fs from 'node:fs' import { Db, type DbHost, type HistoricalQueryHandle } from './db/db.js' +import { + importGraph, + isBackupData, + type BackupData, + type ExportSelector, + type ExportOptions, + type ImportOptions, + type ImportResult +} from './db/backup.js' import { GenerationStore } from './db/generationStore.js' import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError } from './db/errors.js' @@ -5060,6 +5069,26 @@ export class Brainy implements BrainyInterface { }) } + /** + * @description Serialize part or all of the brain into a portable `BackupData` + * document — sugar for `brain.now().export(selector, options)` (the current + * generation). For a past generation use `(await brain.asOf(g)).export(...)`; + * for a speculative state use `brain.now().with(ops).export(...)`. + * + * Restore a `BackupData` with {@link Brainy.import} (it dispatches a backup to the + * graph round-trip; a file/buffer to ingestion). Distinct from `db.persist()` + * (native whole-brain snapshot with generation history). + * + * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}. + * @param options - HOW to export (vectors / VFS bytes / edge policy). See {@link ExportOptions}. + * @returns A versioned, portable `BackupData` document. + * @example + * const backup = await brain.export({ ids }, { includeVectors: true }) + */ + async export(selector: ExportSelector = {}, options: ExportOptions = {}): Promise { + return this.now().export(selector, options) + } + /** * @description Execute a declarative operation batch atomically: either * every operation applies and the store advances exactly ONE generation, @@ -5434,6 +5463,7 @@ export class Brainy implements BrainyInterface { if (!this._dbHost) { this._dbHost = { store: this.generationStore, + storage: this.storage, get: (id, options) => this.get(id, options), find: (query) => this.find(query), related: (paramsOrId) => this.related(paramsOrId), @@ -6709,7 +6739,7 @@ export class Brainy implements BrainyInterface { * - Reduced confusion (removed redundant options) */ async import( - source: Buffer | string | object, + source: Buffer | string | object | BackupData, options?: { format?: 'excel' | 'pdf' | 'csv' | 'json' | 'markdown' | 'yaml' | 'docx' | 'image' vfsPath?: string @@ -6734,14 +6764,20 @@ export class Brainy implements BrainyInterface { throughput?: number eta?: number }) => void + } & Partial + ): Promise { + // Portable graph round-trip: a BackupData document (produced by export()) is + // restored via ONE atomic transaction — distinct from foreign-file ingestion below. + if (isBackupData(source)) { + return importGraph(this, this.storage, source, options as ImportOptions) } - ) { - // Lazy load ImportCoordinator + + // Lazy load ImportCoordinator (foreign-file ingestion: CSV/PDF/Excel/JSON/…) const { ImportCoordinator } = await import('./import/ImportCoordinator.js') const coordinator = new ImportCoordinator(this) await coordinator.init() - return await coordinator.import(source, options) + return await coordinator.import(source as Buffer | string | object, options) } /** diff --git a/src/db/backup.ts b/src/db/backup.ts new file mode 100644 index 00000000..07fd3ee8 --- /dev/null +++ b/src/db/backup.ts @@ -0,0 +1,688 @@ +/** + * @module db/backup + * @description Portable graph backup/restore engine for Brainy 8.0 — the + * `BackupData v1` format plus the `export`/`import` engines that power + * `db.export()` / `brain.export()` (read, at a pinned generation) and + * `brain.import(backup)` (write, applied as one atomic transaction). + * + * The wire format is identical to the 7.x line (`formatVersion: 1`), so a + * document written by 7.x imports cleanly here and vice-versa. This is the + * **portable** round-trip (human-readable, partial-or-whole, cross-version), + * distinct from `db.persist()`/`Brainy.load()` (the native whole-brain snapshot, + * which preserves generation history) and from `brain.import(file)` (foreign-file + * ingestion). + * + * **Reserved-field split:** each `BackupEntity` carries Brainy's standard fields + * (`subtype`, `visibility`, `data`, `confidence`, `weight`, `service`, `createdBy`, + * `createdAt`) at the TOP LEVEL and `metadata` holds ONLY custom user fields — + * mirroring the in-memory `Entity`, so import maps each to its dedicated `add`/ + * `relate` parameter rather than the metadata bag. + */ + +import { Entity, Relation, Result } from '../types/brainy.types.js' +import { NounType, VerbType } from '../types/graphTypes.js' +import { StorageAdapter } from '../coreTypes.js' +import { getBrainyVersion } from '../utils/version.js' +import { TxOperation } from './types.js' + +/** Fixed entity id of the VFS root collection (a `system` entity; excluded unless `includeSystem`). */ +const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' +/** Magic string identifying a Brainy portable backup document. */ +export const BACKUP_FORMAT = 'brainy-backup' +/** Current portable-format version (import gates on this for cross-version migration). */ +export const BACKUP_FORMAT_VERSION = 1 +/** Default embedding model label (informational; `dimensions` is the real compat gate). */ +const DEFAULT_EMBED_MODEL = 'all-MiniLM-L6-v2' +/** Page size for find-based enumeration (whole-brain / predicate selectors). */ +const ENUM_PAGE = 1000 +/** Per-node relation fetch ceiling (source/target-filtered, so no unfiltered-scan warning). */ +const RELATION_FETCH_LIMIT = 100_000 + +// ============================================================================ +// Public types — identical wire shape to the 7.x line +// ============================================================================ + +/** + * @description Selects WHICH part of the graph to export. Reuses `find()`'s grammar + * plus export-specific selectors. Omit (or pass `{}`) to export the whole brain. + * Structural selectors (`ids`/`collection`/`connected`/`vfsPath`) and predicate + * selectors (`type`/`subtype`/`where`/`service`/`visibility`) **compose**. + */ +export interface ExportSelector { + /** Exactly these entity ids. */ + ids?: string[] + /** A collection id → the collection + its transitive `Contains` members. */ + collection?: string + /** Alias for `collection`. */ + memberOf?: string + /** An entity + its N-hop neighbourhood. */ + connected?: { + from: string + depth?: number + verbs?: VerbType[] + direction?: 'out' | 'in' | 'both' + } + /** A VFS path → the directory/file + (for a directory) its `Contains` subtree. */ + vfsPath?: string + /** For `vfsPath` directories: include the whole subtree (default: true). */ + recursive?: boolean + /** For `collection`/`vfsPath`: cap traversal depth (default: unbounded). */ + depth?: number + /** Predicate: entity type(s). */ + type?: NounType | NounType[] + /** Predicate: entity subtype(s). */ + subtype?: string | string[] + /** Predicate: exact-match metadata fields. */ + where?: Record + /** Predicate: multi-tenancy service id. */ + service?: string + /** Predicate: visibility (`'public'` matches entities with no explicit visibility). */ + visibility?: string +} + +/** Controls HOW the selected graph is serialized. */ +export interface ExportOptions { + /** Include embedding vectors verbatim (default: false → `import()` re-embeds from `data`). */ + includeVectors?: boolean + /** Include VFS file bytes in `blobs` so files round-trip byte-identically (default: false). */ + includeContent?: boolean + /** Include `visibility:'system'` entities (e.g. the VFS root) (default: false). */ + includeSystem?: boolean + /** Which edges to include (default: `'induced'`). */ + edges?: 'induced' | 'incident' | 'none' +} + +/** Controls how a `BackupData` is applied on `import()`. */ +export interface ImportOptions { + /** Conflict policy for an existing id (default: `'merge'` — dedup-by-id). */ + onConflict?: 'merge' | 'replace' | 'skip' + /** Vector policy (default: `'auto'` — re-embed from `data` when no vector is carried). */ + reembed?: 'auto' | 'never' + /** Rewrite every id on the way in (e.g. to clone a template subgraph under fresh ids). */ + remapIds?: (id: string) => string + /** Transaction metadata, recorded in the tx-log alongside the new generation. */ + meta?: Record +} + +/** One entity in a `BackupData`. Standard fields top-level; `metadata` is custom-only. */ +export interface BackupEntity { + id: string + type: string + subtype?: string + visibility?: string + data?: any + confidence?: number + weight?: number + service?: string + createdBy?: any + createdAt?: number + vector?: number[] + metadata?: any +} + +/** One relation (edge) in a `BackupData`. */ +export interface BackupRelation { + id: string + from: string + to: string + type: string + subtype?: string + visibility?: string + weight?: number + confidence?: number + metadata?: any +} + +/** A self-describing, versioned, portable graph document (identical on 7.x and 8.0). */ +export interface BackupData { + format: typeof BACKUP_FORMAT + formatVersion: number + brainyVersion: string + createdAt: string + embedding: { model: string; dimensions: number } + selector?: ExportSelector + entities: BackupEntity[] + relations: BackupRelation[] + blobs?: Record + danglingIds?: string[] + stats: { entityCount: number; relationCount: number; blobCount: number; vectorDimensions?: number } +} + +/** Outcome of an `import()`. */ +export interface ImportResult { + imported: number + merged: number + skipped: number + reembedded: number + blobsWritten: number + errors: Array<{ id: string; error: string }> +} + +/** The minimal read surface the export engine needs — satisfied by `Db` and `Brainy`. */ +export interface BackupReader { + get(id: string, options?: { includeVectors?: boolean }): Promise | null> + find(query: any): Promise[]> + related(paramsOrId?: any): Promise[]> +} + +/** The minimal write surface the import engine needs — satisfied by `Brainy`. */ +export interface BackupWriter { + get(id: string, options?: { includeVectors?: boolean }): Promise | null> + transact(ops: TxOperation[], options?: { meta?: Record }): Promise +} + +/** Type guard: is this value a Brainy `BackupData` document (vs a file/foreign object)? */ +export function isBackupData(value: unknown): value is BackupData { + return ( + typeof value === 'object' && + value !== null && + (value as any).format === BACKUP_FORMAT && + Array.isArray((value as any).entities) + ) +} + +// ============================================================================ +// EXPORT +// ============================================================================ + +/** + * @description Serialize part or all of a graph (read through `reader` at its pinned + * generation) into a portable `BackupData` document. + * @param reader - Generation-correct read surface (`Db` or `Brainy`). + * @param storage - Storage adapter (used only for VFS blob bytes when `includeContent`). + * @param selector - WHAT to export (omit for the whole brain). + * @param options - HOW to export (vectors / file bytes / edge policy). + * @param dimensions - Embedding dimensionality for the manifest. + */ +export async function exportGraph( + reader: BackupReader, + storage: StorageAdapter | undefined, + selector: ExportSelector, + options: ExportOptions +): Promise { + const { + includeVectors = false, + includeContent = false, + includeSystem = false, + edges = 'induced' + } = options + + // 1. Resolve the node-id set. + const idSet = await resolveSelector(reader, selector, includeSystem) + + // 2. Read canonical entities (reserved fields top-level), applying any predicate filter. + const usePredicate = hasPredicate(selector) + const entityMap = new Map>() + const entities: BackupEntity[] = [] + for (const id of idSet) { + const e = await reader.get(id, { includeVectors }) + if (!e) continue + if (!includeSystem && (e as any).visibility === 'system') continue + if (usePredicate && !matchesPredicate(e, selector)) continue + entityMap.set(id, e) + entities.push(toBackupEntity(e, includeVectors)) + } + const keptIds = new Set(entityMap.keys()) + + // 3. Edges per policy. + const { relations, danglingIds } = await collectEdges(reader, keptIds, edges) + + // 4. VFS blob bytes (only when requested). + let blobs: Record | undefined + if (includeContent) blobs = await collectBlobs(storage, entityMap) + const blobCount = blobs ? Object.keys(blobs).length : 0 + + // Embedding dimensionality for the manifest: detect from a carried vector, + // else the default model's 384 (informational — `dimensions` is the compat gate). + const dimensions = entities.find((e) => e.vector && e.vector.length)?.vector?.length ?? 384 + + return { + format: BACKUP_FORMAT, + formatVersion: BACKUP_FORMAT_VERSION, + brainyVersion: getBrainyVersion(), + createdAt: new Date().toISOString(), + embedding: { model: DEFAULT_EMBED_MODEL, dimensions }, + selector, + entities, + relations, + ...(blobs && blobCount > 0 ? { blobs } : {}), + ...(danglingIds && danglingIds.length > 0 ? { danglingIds } : {}), + stats: { + entityCount: entities.length, + relationCount: relations.length, + blobCount, + vectorDimensions: dimensions + } + } +} + +// ============================================================================ +// IMPORT +// ============================================================================ + +/** + * @description Apply a `BackupData` to the brain as ONE atomic transaction (a single + * new generation). Dedup-by-id merge by default; re-embeds from `data` when no vector + * is carried. VFS blob bytes are written first (content-addressed, idempotent). + * @param writer - The brain (`get` + `transact`). + * @param storage - Storage adapter (for VFS blob bytes). + * @param data - A `BackupData` document. + * @param options - Conflict / vector / id-remap policy. + * @throws If `data` is not a `BackupData`, or its `formatVersion` is newer than supported. + */ +export async function importGraph( + writer: BackupWriter, + storage: StorageAdapter | undefined, + data: BackupData, + options: ImportOptions = {} +): Promise { + if (!isBackupData(data)) { + throw new Error( + `import() expects a BackupData document (format:'${BACKUP_FORMAT}'). ` + + `For file ingestion (CSV/PDF/Excel/JSON), pass a file/buffer instead.` + ) + } + if (typeof data.formatVersion === 'number' && data.formatVersion > BACKUP_FORMAT_VERSION) { + throw new Error( + `Backup formatVersion ${data.formatVersion} is newer than this Brainy supports ` + + `(max ${BACKUP_FORMAT_VERSION}). Upgrade Brainy to import this document.` + ) + } + + const { onConflict = 'merge', reembed = 'auto', remapIds, meta } = options + const result: ImportResult = { + imported: 0, + merged: 0, + skipped: 0, + reembedded: 0, + blobsWritten: 0, + errors: [] + } + const mapId = (id: string) => (remapIds ? remapIds(id) : id) + + // 1. Blob bytes first (content-addressed → idempotent), so file entities resolve content. + if (data.blobs && Object.keys(data.blobs).length > 0) { + const blobStorage = (storage as any)?.blobStorage + if (blobStorage?.write) { + for (const [hash, b64] of Object.entries(data.blobs)) { + try { + await blobStorage.write(Buffer.from(b64, 'base64')) + result.blobsWritten++ + } catch (e) { + result.errors.push({ id: hash, error: `blob: ${(e as Error).message}` }) + } + } + } else { + for (const hash of Object.keys(data.blobs)) { + result.errors.push({ id: hash, error: 'blob: storage does not support binary blobs' }) + } + } + } + + // 2. Build the operation batch (one atomic transaction → one generation). + const ops: TxOperation[] = [] + for (const be of data.entities || []) { + const id = mapId(be.id) + try { + const exists = (await writer.get(id)) !== null + if (exists) { + if (onConflict === 'skip') { + result.skipped++ + continue + } + if (onConflict === 'merge') { + ops.push({ op: 'update', id, ...entityUpdateFields(be), merge: true } as TxOperation) + result.merged++ + continue + } + ops.push({ op: 'remove', id } as TxOperation) // 'replace' + } + const useVector = Array.isArray(be.vector) && be.vector.length > 0 + if (!useVector && reembed === 'never') { + result.errors.push({ id, error: 'no vector carried and reembed:never' }) + continue + } + ops.push({ + op: 'add', + id, + ...entityAddFields(be), + ...(useVector ? { vector: be.vector } : {}) + } as TxOperation) + result.imported++ + if (!useVector) result.reembedded++ + } catch (e) { + result.errors.push({ id, error: (e as Error).message }) + } + } + + for (const br of data.relations || []) { + ops.push({ + op: 'relate', + from: mapId(br.from), + to: mapId(br.to), + type: br.type as VerbType, + ...(br.subtype !== undefined ? { subtype: br.subtype } : {}), + ...(br.weight !== undefined ? { weight: br.weight } : {}), + ...(br.confidence !== undefined ? { confidence: br.confidence } : {}), + ...(br.metadata !== undefined ? { metadata: br.metadata } : {}) + } as TxOperation) + } + + // 3. Apply atomically — one generation for the whole backup, or none. + if (ops.length > 0) { + await writer.transact(ops, meta ? { meta } : undefined) + } + + return result +} + +// ============================================================================ +// Selector resolution (private) +// ============================================================================ + +function hasStructural(s: ExportSelector): boolean { + return !!(s.ids || s.collection || s.memberOf || s.connected || s.vfsPath) +} + +function hasPredicate(s: ExportSelector): boolean { + return ( + s.type !== undefined || + s.subtype !== undefined || + s.where !== undefined || + s.service !== undefined || + s.visibility !== undefined + ) +} + +async function resolveSelector( + reader: BackupReader, + s: ExportSelector, + includeSystem: boolean +): Promise> { + let idSet: Set + if (s.ids && s.ids.length) { + idSet = new Set(s.ids) + } else if (s.collection ?? s.memberOf) { + idSet = await resolveCollectionSubtree(reader, (s.collection ?? s.memberOf)!, s.depth) + } else if (s.connected) { + idSet = await resolveConnected(reader, s.connected) + } else if (s.vfsPath) { + idSet = await resolveVfsPath(reader, s.vfsPath, s.recursive ?? true, s.depth) + } else { + idSet = await enumerateAll(reader, s) + } + if (!includeSystem) idSet.delete(VFS_ROOT_ID) + return idSet +} + +/** Whole-brain / predicate enumeration via generation-correct paginated `find()`. */ +async function enumerateAll(reader: BackupReader, s: ExportSelector): Promise> { + const params: any = {} + if (s.type !== undefined) params.type = s.type + if (s.subtype !== undefined) params.subtype = s.subtype + if (s.where !== undefined) params.where = s.where + if (s.service !== undefined) params.service = s.service + const ids = new Set() + let offset = 0 + // eslint-disable-next-line no-constant-condition + while (true) { + const batch = await reader.find({ ...params, limit: ENUM_PAGE, offset }) + for (const r of batch) ids.add(r.id) + if (batch.length < ENUM_PAGE) break + offset += ENUM_PAGE + } + return ids +} + +async function resolveCollectionSubtree( + reader: BackupReader, + rootId: string, + depth?: number +): Promise> { + const set = new Set([rootId]) + const maxDepth = depth ?? Infinity + let frontier = [rootId] + let d = 0 + while (frontier.length && d < maxDepth) { + const next: string[] = [] + for (const id of frontier) { + const rels = await reader.related({ from: id, type: VerbType.Contains, limit: RELATION_FETCH_LIMIT }) + for (const r of rels) { + if (!set.has(r.to)) { + set.add(r.to) + next.push(r.to) + } + } + } + frontier = next + d++ + } + return set +} + +async function resolveConnected( + reader: BackupReader, + c: NonNullable +): Promise> { + const { from, depth = 1, verbs, direction = 'out' } = c + const set = new Set([from]) + let frontier = [from] + for (let d = 0; d < depth; d++) { + const next: string[] = [] + for (const id of frontier) { + const neighbours = await neighboursOf(reader, id, direction, verbs) + for (const n of neighbours) { + if (!set.has(n)) { + set.add(n) + next.push(n) + } + } + } + frontier = next + if (!next.length) break + } + return set +} + +async function neighboursOf( + reader: BackupReader, + id: string, + direction: 'out' | 'in' | 'both', + verbs?: VerbType[] +): Promise { + const out: string[] = [] + if (direction === 'out' || direction === 'both') { + const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT }) + for (const r of rels) if (!verbs || verbs.includes(r.type)) out.push(r.to) + } + if (direction === 'in' || direction === 'both') { + const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT }) + for (const r of rels) if (!verbs || verbs.includes(r.type)) out.push(r.from) + } + return out +} + +async function resolveVfsPath( + reader: BackupReader, + path: string, + recursive: boolean, + depth?: number +): Promise> { + const dirId = await resolveVfsPathToId(reader, path) + if (!dirId) return new Set() + if (!recursive) { + const set = new Set([dirId]) + const rels = await reader.related({ from: dirId, type: VerbType.Contains, limit: RELATION_FETCH_LIMIT }) + for (const r of rels) set.add(r.to) + return set + } + return resolveCollectionSubtree(reader, dirId, depth) +} + +async function resolveVfsPathToId(reader: BackupReader, path: string): Promise { + const segments = path.split('/').filter(Boolean) + let currentId = VFS_ROOT_ID + for (const seg of segments) { + const rels = await reader.related({ from: currentId, type: VerbType.Contains, limit: RELATION_FETCH_LIMIT }) + let found: string | null = null + for (const r of rels) { + const child = await reader.get(r.to) + if ((child?.metadata as any)?.name === seg) { + found = r.to + break + } + } + if (!found) return null + currentId = found + } + return currentId +} + +function matchesPredicate(e: Entity, s: ExportSelector): boolean { + if (s.type !== undefined) { + const types = Array.isArray(s.type) ? s.type : [s.type] + if (!types.includes(e.type)) return false + } + if (s.subtype !== undefined) { + const subs = Array.isArray(s.subtype) ? s.subtype : [s.subtype] + if (e.subtype === undefined || !subs.includes(e.subtype)) return false + } + if (s.service !== undefined && e.service !== s.service) return false + if (s.visibility !== undefined) { + const vis = (e as any).visibility ?? 'public' + if (vis !== s.visibility) return false + } + if (s.where && !matchesWhere(e.metadata, s.where)) return false + return true +} + +function matchesWhere(metadata: any, where: Record): boolean { + if (!metadata) return false + for (const [key, value] of Object.entries(where)) { + if (metadata[key] !== value) return false + } + return true +} + +// ============================================================================ +// Serialization helpers (private) +// ============================================================================ + +function toBackupEntity(e: Entity, includeVectors: boolean): BackupEntity { + const be: BackupEntity = { id: e.id, type: e.type as string } + if (e.subtype !== undefined) be.subtype = e.subtype + const vis = (e as any).visibility + if (vis !== undefined && vis !== 'public') be.visibility = vis + if (e.data !== undefined) be.data = e.data + if (e.confidence !== undefined) be.confidence = e.confidence + if (e.weight !== undefined) be.weight = e.weight + if (e.service !== undefined) be.service = e.service + if ((e as any).createdBy !== undefined) be.createdBy = (e as any).createdBy + if (e.createdAt !== undefined) be.createdAt = e.createdAt + if (includeVectors && e.vector && e.vector.length) be.vector = e.vector + if (e.metadata && Object.keys(e.metadata as any).length) be.metadata = e.metadata + return be +} + +function toBackupRelation(r: Relation): BackupRelation { + const br: BackupRelation = { id: r.id, from: r.from, to: r.to, type: r.type as string } + if (r.subtype !== undefined) br.subtype = r.subtype + const vis = (r as any).visibility + if (vis !== undefined && vis !== 'public') br.visibility = vis + if (r.weight !== undefined) br.weight = r.weight + if (r.confidence !== undefined) br.confidence = r.confidence + if (r.metadata && Object.keys(r.metadata as any).length) br.metadata = r.metadata + return br +} + +async function collectEdges( + reader: BackupReader, + idSet: Set, + edges: 'induced' | 'incident' | 'none' +): Promise<{ relations: BackupRelation[]; danglingIds?: string[] }> { + if (edges === 'none') return { relations: [] } + + const relations: BackupRelation[] = [] + const dangling = new Set() + const seen = new Set() + + for (const id of idSet) { + const rels = await reader.related({ from: id, limit: RELATION_FETCH_LIMIT }) + for (const r of rels) { + if (seen.has(r.id)) continue + const toIn = idSet.has(r.to) + if (edges === 'induced' && !toIn) continue + if (!toIn) dangling.add(r.to) + seen.add(r.id) + relations.push(toBackupRelation(r)) + } + } + + if (edges === 'incident') { + for (const id of idSet) { + const rels = await reader.related({ to: id, limit: RELATION_FETCH_LIMIT }) + for (const r of rels) { + if (seen.has(r.id)) continue + if (!idSet.has(r.from)) { + dangling.add(r.from) + seen.add(r.id) + relations.push(toBackupRelation(r)) + } + } + } + } + + return dangling.size > 0 ? { relations, danglingIds: Array.from(dangling) } : { relations } +} + +async function collectBlobs( + storage: StorageAdapter | undefined, + entityMap: Map> +): Promise> { + const blobs: Record = {} + const blobStorage = (storage as any)?.blobStorage + if (!blobStorage?.read) return blobs + + for (const e of entityMap.values()) { + const storageMeta = (e.metadata as any)?.storage + const hash = storageMeta?.hash + if (storageMeta?.type === 'blob' && hash && !blobs[hash]) { + try { + const buf = await blobStorage.read(hash) + blobs[hash] = Buffer.from(buf).toString('base64') + } catch { + // Referenced blob bytes unreadable (storage drift): skip — file structure still + // travels, and stats.blobCount reflects what was actually captured. + } + } + } + return blobs +} + +// ============================================================================ +// add()/update() param mapping (private) +// ============================================================================ + +function entityAddFields(be: BackupEntity): Record { + const fields: Record = { data: be.data, type: be.type as NounType } + if (be.subtype !== undefined) fields.subtype = be.subtype + if (be.visibility !== undefined) fields.visibility = be.visibility + if (be.service !== undefined) fields.service = be.service + if (be.confidence !== undefined) fields.confidence = be.confidence + if (be.weight !== undefined) fields.weight = be.weight + if (be.metadata !== undefined) fields.metadata = be.metadata + return fields +} + +function entityUpdateFields(be: BackupEntity): Record { + const fields: Record = {} + if (be.data !== undefined) fields.data = be.data + if (be.type !== undefined) fields.type = be.type as NounType + if (be.subtype !== undefined) fields.subtype = be.subtype + if (be.visibility !== undefined) fields.visibility = be.visibility + if (be.confidence !== undefined) fields.confidence = be.confidence + if (be.weight !== undefined) fields.weight = be.weight + if (be.metadata !== undefined) fields.metadata = be.metadata + if (Array.isArray(be.vector) && be.vector.length) fields.vector = be.vector + return fields +} diff --git a/src/db/db.ts b/src/db/db.ts index 82aff289..40db407e 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -56,6 +56,9 @@ import type { Relation, Result } from '../types/brainy.types.js' +import type { StorageAdapter } from '../coreTypes.js' +import { exportGraph } from './backup.js' +import type { ExportSelector, ExportOptions, BackupData } from './backup.js' import { splitNounMetadataRecord, splitVerbMetadataRecord @@ -105,6 +108,8 @@ export interface SpeculativeOverlay { export interface DbHost { /** The generational record layer of the host brain. */ readonly store: GenerationStore + /** The host brain's storage adapter (portable-export VFS blob bytes). */ + readonly storage: StorageAdapter /** Live `brain.get()` (current-generation fast path). */ get(id: string, options?: GetOptions): Promise | null> /** Live `brain.find()` (current-generation fast path). */ @@ -409,6 +414,28 @@ export class Db { return materialized.find({ ...(options ?? {}), query }) } + /** + * @description Serialize part or all of this database value into a portable + * `BackupData` document, read **as of this view's generation**. Because `export` + * lives on the immutable `Db`, it composes with every way of obtaining one: + * `brain.now().export(sel)` (current), `(await brain.asOf(g)).export(sel)` + * (time-travel export), and `brain.now().with(ops).export(sel)` (what-if export). + * + * The document is portable JSON, versioned (`formatVersion`), and current-state + * (no generation history) — distinct from `persist()` (native whole-brain snapshot + * that preserves history). Restore with `brain.import(backup)`. + * + * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}. + * @param options - HOW to export (vectors / VFS bytes / edge policy). See {@link ExportOptions}. + * @returns A versioned, portable `BackupData` document. + * @example + * const backup = await brain.now().export({ collection: id }, { includeVectors: true }) + */ + async export(selector: ExportSelector = {}, options: ExportOptions = {}): Promise { + this.assertUsable('export') + return exportGraph(this, this.host.storage, selector, options) + } + /** * @description Relationships **as of this view's generation** — the `Db` * counterpart of `brain.related()` (same string-id shorthand and diff --git a/src/index.ts b/src/index.ts index 8abd95a5..5cae505a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -141,6 +141,19 @@ export { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js // Immutable database values: brain.now() / brain.transact() / brain.asOf() / // db.with() / db.persist() / Brainy.load(). See src/db/ for the record layer. export { Db } from './db/db.js' + +// Portable graph backup/restore — db.export() / brain.export() / brain.import() +// (BackupData v1; identical wire format to the 7.x line). +export { isBackupData } from './db/backup.js' +export type { + BackupData, + BackupEntity, + BackupRelation, + ExportSelector, + ExportOptions, + ImportOptions, + ImportResult +} from './db/backup.js' export { GenerationConflictError, SpeculativeOverlayError, diff --git a/tests/unit/db/db-backup.test.ts b/tests/unit/db/db-backup.test.ts new file mode 100644 index 00000000..e5ccd7f0 --- /dev/null +++ b/tests/unit/db/db-backup.test.ts @@ -0,0 +1,180 @@ +/** + * Unit tests for the 8.0 portable graph backup/restore surface: + * db.export() / brain.export() (read, at a pinned generation) + * brain.import(backup) (write, one atomic transaction; polymorphic) + * + * Real round-trips against in-memory storage (no mocks of the API under test), + * including the 8.0-specific compositions (asOf/with export) and the polymorphic + * import() dispatch. Entities carry subtypes so the suite passes under 8.0's + * subtype-required default. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { createTestConfig } from '../../helpers/test-factory' +import { NounType, VerbType } from '../../../src/types/graphTypes' +import type { BackupData } from '../../../src/db/backup' + +describe('8.0 portable graph export/import (BackupData v1)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy(createTestConfig()) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + describe('format + round-trip', () => { + it('brain.export() produces a versioned BackupData document', async () => { + const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' }) + const b = await brain.add({ data: 'Acme', type: NounType.Organization, subtype: 'vendor' }) + await brain.relate({ from: a, to: b, type: VerbType.WorksWith, subtype: 'full-time' }) + + const backup = await brain.export() + expect(backup.format).toBe('brainy-backup') + expect(backup.formatVersion).toBe(1) + expect(backup.entities.map((e) => e.id).sort()).toEqual([a, b].sort()) + expect(backup.relations).toHaveLength(1) + expect(backup.entities.find((e) => e.id === a)?.subtype).toBe('employee') + expect(backup.relations[0].subtype).toBe('full-time') + }) + + it('round-trips into a second brain via polymorphic brain.import()', async () => { + const a = await brain.add({ data: 'Alice', type: NounType.Person, subtype: 'employee' }) + const b = await brain.add({ data: 'Bob', type: NounType.Person, subtype: 'employee' }) + await brain.relate({ from: a, to: b, type: VerbType.FriendOf, subtype: 'close' }) + const backup = await brain.export({}, { includeVectors: true }) + + const target = new Brainy(createTestConfig()) + await target.init() + try { + const result = await target.import(backup) + expect(result.imported).toBe(2) + expect(result.errors).toHaveLength(0) + expect((await target.get(a))?.id).toBe(a) + const rels = await target.related({ from: a }) + expect(rels.some((r) => r.to === b && r.type === VerbType.FriendOf)).toBe(true) + } finally { + await target.close() + } + }) + }) + + describe('selectors', () => { + it('ids — exactly the requested entities', async () => { + const a = await brain.add({ data: 'A', type: NounType.Thing, subtype: 'x' }) + const b = await brain.add({ data: 'B', type: NounType.Thing, subtype: 'x' }) + await brain.add({ data: 'C', type: NounType.Thing, subtype: 'x' }) + const backup = await brain.export({ ids: [a, b] }) + expect(backup.entities.map((e) => e.id).sort()).toEqual([a, b].sort()) + }) + + it('collection — collection + transitive Contains members', async () => { + const root = await brain.add({ data: 'Folder', type: NounType.Collection, subtype: 'dir' }) + const c1 = await brain.add({ data: 'C1', type: NounType.Document, subtype: 'doc' }) + const c2 = await brain.add({ data: 'C2', type: NounType.Document, subtype: 'doc' }) + await brain.add({ data: 'Outside', type: NounType.Document, subtype: 'doc' }) + await brain.relate({ from: root, to: c1, type: VerbType.Contains, subtype: 'has' }) + await brain.relate({ from: c1, to: c2, type: VerbType.Contains, subtype: 'has' }) + const backup = await brain.export({ collection: root }) + expect(backup.entities.map((e) => e.id).sort()).toEqual([root, c1, c2].sort()) + }) + + it('induced edges only by default', async () => { + const a = await brain.add({ data: 'A', type: NounType.Thing, subtype: 'x' }) + const b = await brain.add({ data: 'B', type: NounType.Thing, subtype: 'x' }) + const c = await brain.add({ data: 'C', type: NounType.Thing, subtype: 'x' }) + await brain.relate({ from: a, to: b, type: VerbType.RelatedTo, subtype: 'r' }) + await brain.relate({ from: b, to: c, type: VerbType.RelatedTo, subtype: 'r' }) + const backup = await brain.export({ ids: [a, b] }) + expect(backup.relations).toHaveLength(1) + expect(backup.relations[0].from).toBe(a) + }) + }) + + describe('vectors + conflict', () => { + it('omits vectors by default and re-embeds on import', async () => { + const a = await brain.add({ data: 'Re-embed me', type: NounType.Thing, subtype: 'x' }) + const backup = await brain.export({ ids: [a] }) + expect(backup.entities[0].vector).toBeUndefined() + + const target = new Brainy(createTestConfig()) + await target.init() + try { + const result = await target.import(backup) + expect(result.reembedded).toBe(1) + expect((await target.get(a, { includeVectors: true }))?.vector?.length).toBeGreaterThan(0) + } finally { + await target.close() + } + }) + + it('onConflict:merge updates an existing entity in place', async () => { + const a = await brain.add({ data: 'V1', type: NounType.Thing, subtype: 'x', metadata: { n: 1 } }) + const backup = await brain.export({ ids: [a] }, { includeVectors: true }) + await brain.update({ id: a, metadata: { n: 99 }, merge: false }) + + const result = await brain.import(backup, { onConflict: 'merge' }) + expect(result.merged).toBe(1) + expect((await brain.get(a))?.metadata?.n).toBe(1) + }) + }) + + describe('8.0 composition — export on the immutable Db', () => { + it('asOf(generation).export() is a time-travel export', async () => { + // Only transact() advances the committed generation (add() does not), so + // distinct generations are created via transact for a true time-travel pin. + const a = '22222222-2222-4222-8222-222222222222' + const b = '33333333-3333-4333-8333-333333333333' + await brain.transact([{ op: 'add', id: a, data: 'First', type: NounType.Thing, subtype: 'x' }]) + const g1 = brain.generation() + await brain.transact([{ op: 'add', id: b, data: 'Second', type: NounType.Thing, subtype: 'x' }]) + + const past = await brain.asOf(g1) + try { + // Whole-brain export at g1 enumerates via the generation-correct find() path. + const backup = await past.export() + const ids = backup.entities.map((e) => e.id) + expect(ids).toContain(a) + expect(ids).not.toContain(b) // b was committed in a later generation + } finally { + await past.release() + } + }) + + it('now().with(ops).export() is a what-if export', async () => { + const a = await brain.add({ data: 'Real', type: NounType.Thing, subtype: 'x' }) + const speculativeId = '11111111-1111-4111-8111-111111111111' + const view = await brain.now().with([ + { op: 'add', id: speculativeId, data: 'Speculative', type: NounType.Thing, subtype: 'x' } + ]) + try { + const backup = await view.export({ ids: [a, speculativeId] }) + expect(backup.entities.map((e) => e.id).sort()).toEqual([a, speculativeId].sort()) + } finally { + await view.release() + } + // The speculative entity never touched the durable brain. + expect(await brain.get(speculativeId)).toBeNull() + }) + }) + + describe('import validation', () => { + it('rejects a newer formatVersion', async () => { + const future: BackupData = { + format: 'brainy-backup', + formatVersion: 999, + brainyVersion: 'x', + createdAt: new Date().toISOString(), + embedding: { model: 'm', dimensions: 384 }, + entities: [], + relations: [], + stats: { entityCount: 0, relationCount: 0, blobCount: 0 } + } + await expect(brain.import(future)).rejects.toThrow(/formatVersion/) + }) + }) +})