feat: portable graph export()/import() (BackupData v1) on brain.data()
brain.data() now serializes part or all of a brain to a versioned, portable
JSON document (BackupData) and restores it — the path for partial backups,
cross-environment moves, and 7.x→8.0 migration.
- export(selector?, options?): select by ids, collection (+transitive Contains),
connected neighbourhood, vfsPath subtree, predicate, or whole brain; structural
and predicate selectors compose. Options: includeVectors, includeContent (VFS
blobs), includeSystem, edges ('induced'|'incident'|'none').
- import(backup, options?): onConflict 'merge' (dedup-by-id) | 'replace' | 'skip';
reembed 'auto' (re-embed from data when no vector carried) | 'never'; remapIds
to clone a subgraph under fresh ids.
- BackupData v1: format/formatVersion/brainyVersion/createdAt/embedding/entities/
relations/blobs?/danglingIds?/stats. Reserved fields (subtype, data, confidence,
weight, service) top-level; metadata custom-only. Current-state, no generations.
- Replaces the prior flat-entity export() (dropped relations) with a graph-complete
document. Distinct from brain.import(file) ingestion, which is unchanged.
- Export BackupData/BackupEntity/BackupRelation/ExportSelector/ExportOptions/
ImportOptions/ImportResult/DataAPI from the package root. CLI `brainy export`
now writes a BackupData document.
Guide: docs/guides/backup-and-export.md. Tests: tests/unit/api/data-backup.test.ts.
This commit is contained in:
parent
89c6d043ba
commit
a408d3799d
11 changed files with 1393 additions and 382 deletions
1082
src/api/DataAPI.ts
1082
src/api/DataAPI.ts
File diff suppressed because it is too large
Load diff
|
|
@ -5770,12 +5770,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*/
|
||||
async data() {
|
||||
const { DataAPI } = await import('./api/DataAPI.js')
|
||||
return new DataAPI(
|
||||
this.storage,
|
||||
(id: string) => this.get(id),
|
||||
undefined, // No getRelation method yet
|
||||
this
|
||||
)
|
||||
return new DataAPI(this.storage, this)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -876,60 +876,66 @@ export const coreCommands = {
|
|||
const brain = getBrainy()
|
||||
const format = options.format || 'json'
|
||||
|
||||
// Export all data
|
||||
// Export the whole brain as a portable BackupData document (vectors + VFS file bytes).
|
||||
const dataApi = await brain.data()
|
||||
const data = await dataApi.export({ format: 'json' })
|
||||
const backup = await dataApi.export(undefined, {
|
||||
includeVectors: true,
|
||||
includeContent: true
|
||||
})
|
||||
let output = ''
|
||||
|
||||
|
||||
const csvCell = (v: any): string => {
|
||||
if (v === undefined || v === null) return ''
|
||||
const s = typeof v === 'object' ? JSON.stringify(v) : String(v)
|
||||
return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s
|
||||
}
|
||||
|
||||
switch (format) {
|
||||
case 'json':
|
||||
output = options.pretty
|
||||
? JSON.stringify(data, null, 2)
|
||||
: JSON.stringify(data)
|
||||
output = options.pretty
|
||||
? JSON.stringify(backup, null, 2)
|
||||
: JSON.stringify(backup)
|
||||
break
|
||||
|
||||
case 'jsonl':
|
||||
if (Array.isArray(data)) {
|
||||
output = data.map(item => JSON.stringify(item)).join('\n')
|
||||
} else {
|
||||
output = JSON.stringify(data)
|
||||
}
|
||||
|
||||
case 'jsonl': {
|
||||
// NDJSON: a header line, then one line per entity, then one line per relation.
|
||||
const { entities, relations, ...header } = backup
|
||||
const lines: string[] = [JSON.stringify({ kind: 'header', ...header })]
|
||||
for (const e of entities) lines.push(JSON.stringify({ kind: 'entity', ...e }))
|
||||
for (const r of relations) lines.push(JSON.stringify({ kind: 'relation', ...r }))
|
||||
output = lines.join('\n')
|
||||
break
|
||||
|
||||
case 'csv':
|
||||
if (Array.isArray(data) && data.length > 0) {
|
||||
// Get all unique keys for headers
|
||||
}
|
||||
|
||||
case 'csv': {
|
||||
// CSV represents entities only — a graph's edges and blobs can't be tabular.
|
||||
const entities = backup.entities
|
||||
if (entities.length > 0) {
|
||||
const headers = new Set<string>()
|
||||
data.forEach(item => {
|
||||
Object.keys(item).forEach(key => headers.add(key))
|
||||
})
|
||||
entities.forEach((e: Record<string, any>) => Object.keys(e).forEach(k => headers.add(k)))
|
||||
const headerArray = Array.from(headers)
|
||||
|
||||
// Create CSV
|
||||
output = headerArray.join(',') + '\n'
|
||||
output += data.map(item => {
|
||||
return headerArray.map(h => {
|
||||
const value = item[h]
|
||||
if (typeof value === 'object') {
|
||||
return JSON.stringify(value)
|
||||
}
|
||||
return value || ''
|
||||
}).join(',')
|
||||
}).join('\n')
|
||||
output += entities
|
||||
.map((e: Record<string, any>) => headerArray.map(h => csvCell(e[h])).join(','))
|
||||
.join('\n')
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (file) {
|
||||
writeFileSync(file, output)
|
||||
spinner.succeed(`Exported to ${file}`)
|
||||
|
||||
|
||||
if (!options.json) {
|
||||
console.log(chalk.green(`✓ Successfully exported database to ${file}`))
|
||||
console.log(chalk.dim(` Format: ${format}`))
|
||||
console.log(chalk.dim(` Items: ${Array.isArray(data) ? data.length : 1}`))
|
||||
console.log(chalk.dim(` Entities: ${backup.stats.entityCount} Relations: ${backup.stats.relationCount}`))
|
||||
} else {
|
||||
formatOutput({ file, format, count: Array.isArray(data) ? data.length : 1 }, options)
|
||||
formatOutput(
|
||||
{ file, format, entityCount: backup.stats.entityCount, relationCount: backup.stats.relationCount },
|
||||
options
|
||||
)
|
||||
}
|
||||
} else {
|
||||
spinner.succeed('Export complete')
|
||||
|
|
|
|||
12
src/index.ts
12
src/index.ts
|
|
@ -116,6 +116,18 @@ export {
|
|||
// Export version utilities
|
||||
export { getBrainyVersion } from './utils/version.js'
|
||||
|
||||
// Export portable graph backup/restore API (brain.data())
|
||||
export { DataAPI } from './api/DataAPI.js'
|
||||
export type {
|
||||
BackupData,
|
||||
BackupEntity,
|
||||
BackupRelation,
|
||||
ExportSelector,
|
||||
ExportOptions,
|
||||
ImportOptions,
|
||||
ImportResult
|
||||
} from './api/DataAPI.js'
|
||||
|
||||
// Export plugin system
|
||||
export type { BrainyPlugin, BrainyPluginContext, StorageAdapterFactory } from './plugin.js'
|
||||
export { PluginRegistry } from './plugin.js'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue