refactor(8.0): delete DataAPI — superseded by Db persist/restore + import API + stats
The legacy backup/import/export/stats facade (src/api/DataAPI.ts) drifted from the modern entity shape and every job it did now has a first-class surface. Delete it and brain.data(), and rewire the CLI: - data-stats → brain.stats() (full BrainyStats report: per-type breakdowns, indexed fields, index health, storage backend, writer lock, version) - clean → brain.clear() - export → alias of snapshot; a db.persist() snapshot is the full-fidelity export format (open with Brainy.load, load wholesale with brainy restore); external data ingestion remains brainy import (UniversalImportAPI) Rewiring clean onto brain.clear() exposed two real bugs, both fixed: - clear() left this.graphIndex undefined forever — any graph-touching call afterwards (relate, getNeighbors, stats) crashed. clear() now re-resolves the graph index exactly as init() does and re-wires the shared UUID↔int resolver, and re-resolves the metadata index with the same provider fallback as init(). - storage.clear() reset the legacy totals but not the per-type/subtype count rollups or id→type caches, so stats() reported phantom counts for deleted entities. Both adapters now delegate derived-state reset to reloadDerivedState(), the same path restore-from-snapshot uses. One-shot CLI commands (data-stats, clean, snapshot/export, restore, history, generation) now close the brain and exit explicitly — global cache timers otherwise keep the process alive holding the writer lock. Verified: build clean, 1383/1383 unit tests, 24/24 db-mvcc integration, plus an end-to-end CLI smoke (add → data-stats → export → clean → data-stats).
This commit is contained in:
parent
cc8037db10
commit
478fa176f2
12 changed files with 262 additions and 639 deletions
|
|
@ -1,13 +1,14 @@
|
|||
/**
|
||||
* Core CLI Commands - TypeScript Implementation
|
||||
*
|
||||
* Essential database operations: add, search, get, relate, import, export
|
||||
* @module cli/commands/core
|
||||
* @description Essential database CLI commands: add, search, get, update,
|
||||
* delete, relate, unrelate, and diagnostics. Database export lives under
|
||||
* `brainy snapshot` (aliased as `brainy export`) — a `db.persist()` snapshot
|
||||
* is the full-fidelity export format.
|
||||
*/
|
||||
|
||||
import chalk from 'chalk'
|
||||
import ora from 'ora'
|
||||
import inquirer from 'inquirer'
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { Brainy } from '../../brainy.js'
|
||||
import { BrainyTypes, NounType, VerbType } from '../../index.js'
|
||||
|
||||
|
|
@ -66,10 +67,6 @@ interface RelateOptions extends CoreOptions {
|
|||
subtype?: string
|
||||
}
|
||||
|
||||
interface ExportOptions extends CoreOptions {
|
||||
format?: 'json' | 'csv' | 'jsonl'
|
||||
}
|
||||
|
||||
let brainyInstance: Brainy | null = null
|
||||
|
||||
const getBrainy = (): Brainy => {
|
||||
|
|
@ -866,82 +863,6 @@ export const coreCommands = {
|
|||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Export database
|
||||
*/
|
||||
async export(file: string | undefined, options: ExportOptions) {
|
||||
const spinner = ora('Exporting database...').start()
|
||||
|
||||
try {
|
||||
const brain = getBrainy()
|
||||
const format = options.format || 'json'
|
||||
|
||||
// Export all data
|
||||
const dataApi = await brain.data()
|
||||
const data = await dataApi.export({ format: 'json' })
|
||||
let output = ''
|
||||
|
||||
switch (format) {
|
||||
case 'json':
|
||||
output = options.pretty
|
||||
? JSON.stringify(data, null, 2)
|
||||
: JSON.stringify(data)
|
||||
break
|
||||
|
||||
case 'jsonl':
|
||||
if (Array.isArray(data)) {
|
||||
output = data.map(item => JSON.stringify(item)).join('\n')
|
||||
} else {
|
||||
output = JSON.stringify(data)
|
||||
}
|
||||
break
|
||||
|
||||
case 'csv':
|
||||
if (Array.isArray(data) && data.length > 0) {
|
||||
// Get all unique keys for headers
|
||||
const headers = new Set<string>()
|
||||
data.forEach(item => {
|
||||
Object.keys(item).forEach(key => headers.add(key))
|
||||
})
|
||||
const headerArray = Array.from(headers)
|
||||
|
||||
// Create CSV
|
||||
output = headerArray.join(',') + '\n'
|
||||
output += data.map(item => {
|
||||
return headerArray.map(h => {
|
||||
const value = item[h]
|
||||
if (typeof value === 'object') {
|
||||
return JSON.stringify(value)
|
||||
}
|
||||
return value || ''
|
||||
}).join(',')
|
||||
}).join('\n')
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if (file) {
|
||||
writeFileSync(file, output)
|
||||
spinner.succeed(`Exported to ${file}`)
|
||||
|
||||
if (!options.json) {
|
||||
console.log(chalk.green(`✓ Successfully exported database to ${file}`))
|
||||
console.log(chalk.dim(` Format: ${format}`))
|
||||
console.log(chalk.dim(` Items: ${Array.isArray(data) ? data.length : 1}`))
|
||||
} else {
|
||||
formatOutput({ file, format, count: Array.isArray(data) ? data.length : 1 }, options)
|
||||
}
|
||||
} else {
|
||||
spinner.succeed('Export complete')
|
||||
console.log(output)
|
||||
}
|
||||
} catch (error: any) {
|
||||
spinner.fail('Export failed')
|
||||
console.error(chalk.red(error.message))
|
||||
process.exit(1)
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Show plugin and provider diagnostics
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -1,14 +1,26 @@
|
|||
/**
|
||||
* Data Management Commands
|
||||
* @module cli/commands/data
|
||||
* @description Detailed database statistics for the `data-stats` CLI command.
|
||||
*
|
||||
* Import, export, and statistics operations
|
||||
* Renders `brain.stats()` (the operator-facing `BrainyStats` summary) as a
|
||||
* human-readable report: counts with per-type breakdowns, indexed fields,
|
||||
* per-index health flags, storage backend, writer-lock holder, and library
|
||||
* version. Pass `--json` for machine-readable output.
|
||||
*
|
||||
* The legacy backup/import/export facade that used to live behind this
|
||||
* command was superseded in 8.0: snapshots are `brainy snapshot` / `brainy
|
||||
* restore` (the Db API's `persist()`/`restore()`), and data ingestion is
|
||||
* `brainy import` (UniversalImportAPI).
|
||||
*/
|
||||
|
||||
import chalk from 'chalk'
|
||||
import ora from 'ora'
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { Brainy } from '../../brainy.js'
|
||||
|
||||
/**
|
||||
* @description Shared CLI flags the data commands accept (`--verbose`,
|
||||
* `--json`, `--pretty`), mirroring the other command modules.
|
||||
*/
|
||||
interface DataOptions {
|
||||
verbose?: boolean
|
||||
json?: boolean
|
||||
|
|
@ -17,6 +29,11 @@ interface DataOptions {
|
|||
|
||||
let brainyInstance: Brainy | null = null
|
||||
|
||||
/**
|
||||
* @description Lazily construct the module's shared `Brainy` instance so
|
||||
* repeated handler invocations (and tests) reuse one store handle.
|
||||
* @returns The shared `Brainy` instance.
|
||||
*/
|
||||
const getBrainy = (): Brainy => {
|
||||
if (!brainyInstance) {
|
||||
brainyInstance = new Brainy()
|
||||
|
|
@ -24,60 +41,113 @@ const getBrainy = (): Brainy => {
|
|||
return brainyInstance
|
||||
}
|
||||
|
||||
const formatOutput = (data: any, options: DataOptions): void => {
|
||||
/**
|
||||
* @description Emit machine-readable output when `--json` is set; honors
|
||||
* `--pretty` for indented JSON.
|
||||
* @param data - The value to serialize.
|
||||
* @param options - Shared CLI flags.
|
||||
*/
|
||||
const formatOutput = (data: unknown, options: DataOptions): void => {
|
||||
if (options.json) {
|
||||
console.log(options.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Render a `Record<type, count>` breakdown as indented,
|
||||
* count-sorted lines (largest first).
|
||||
* @param byType - Per-type counts, e.g. `BrainyStats.entitiesByType`.
|
||||
* @returns Formatted lines ready for `console.log`, one per type.
|
||||
*/
|
||||
const formatTypeBreakdown = (byType: Record<string, number>): string[] =>
|
||||
Object.entries(byType)
|
||||
.sort(([, a], [, b]) => b - a)
|
||||
.map(([type, count]) => ` ${type}: ${chalk.green(count)}`)
|
||||
|
||||
/**
|
||||
* @description The `data-stats` command handler registered in
|
||||
* `src/cli/index.ts`.
|
||||
*/
|
||||
export const dataCommands = {
|
||||
/**
|
||||
* Get database statistics
|
||||
* @description Show detailed database statistics via `brain.stats()`:
|
||||
* entity/relationship totals with per-type breakdowns, indexed metadata
|
||||
* fields, index health, storage backend, writer lock, and version.
|
||||
* @param options - Shared CLI flags.
|
||||
* @example
|
||||
* ```bash
|
||||
* brainy data-stats # human-readable report
|
||||
* brainy data-stats --json # raw BrainyStats JSON
|
||||
* ```
|
||||
*/
|
||||
async stats(options: DataOptions) {
|
||||
const spinner = ora('Gathering statistics...').start()
|
||||
|
||||
try {
|
||||
const brain = getBrainy()
|
||||
const dataApi = await brain.data()
|
||||
|
||||
const stats = await dataApi.getStats()
|
||||
await brain.init()
|
||||
const stats = await brain.stats()
|
||||
|
||||
spinner.succeed('Statistics gathered')
|
||||
|
||||
if (!options.json) {
|
||||
console.log(chalk.cyan('\n📊 Database Statistics:\n'))
|
||||
|
||||
console.log(chalk.bold('Entities:'))
|
||||
console.log(` Total: ${chalk.green(stats.entities)}`)
|
||||
|
||||
console.log(chalk.bold('\nRelationships:'))
|
||||
console.log(` Total: ${chalk.green(stats.relations)}`)
|
||||
|
||||
if ((stats as any).storageSize) {
|
||||
console.log(chalk.bold('\nStorage:'))
|
||||
console.log(` Size: ${chalk.green(formatBytes((stats as any).storageSize))}`)
|
||||
}
|
||||
|
||||
if ((stats as any).vectorDimensions) {
|
||||
console.log(chalk.bold('\nVector Index:'))
|
||||
console.log(` Dimensions: ${(stats as any).vectorDimensions}`)
|
||||
}
|
||||
} else {
|
||||
if (options.json) {
|
||||
formatOutput(stats, options)
|
||||
// close() releases the writer lock and indexes, but global timers
|
||||
// (UnifiedCache bookkeeping, PathResolver stats) keep the event loop
|
||||
// alive. CLI commands are one-shot — exit explicitly.
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
}
|
||||
} catch (error: any) {
|
||||
|
||||
console.log(chalk.cyan('\n📊 Database Statistics\n'))
|
||||
|
||||
console.log(chalk.bold('Entities:'))
|
||||
console.log(` Total: ${chalk.green(stats.entityCount)}`)
|
||||
for (const line of formatTypeBreakdown(stats.entitiesByType)) {
|
||||
console.log(` ${line}`)
|
||||
}
|
||||
|
||||
console.log(chalk.bold('\nRelationships:'))
|
||||
console.log(` Total: ${chalk.green(stats.relationCount)}`)
|
||||
for (const line of formatTypeBreakdown(stats.relationsByType)) {
|
||||
console.log(` ${line}`)
|
||||
}
|
||||
|
||||
console.log(chalk.bold('\nIndexes:'))
|
||||
console.log(` Indexed fields: ${chalk.green(stats.fieldRegistry.length)}`)
|
||||
if (options.verbose && stats.fieldRegistry.length > 0) {
|
||||
console.log(chalk.dim(` ${stats.fieldRegistry.join(', ')}`))
|
||||
}
|
||||
const health = (ok: boolean) => (ok ? chalk.green('ok') : chalk.red('empty'))
|
||||
console.log(` Vector: ${health(stats.indexHealth.vector)}`)
|
||||
console.log(` Metadata: ${health(stats.indexHealth.metadata)}`)
|
||||
console.log(` Graph: ${health(stats.indexHealth.graph)}`)
|
||||
|
||||
console.log(chalk.bold('\nStorage:'))
|
||||
console.log(` Backend: ${stats.storage.backend}`)
|
||||
if (stats.storage.rootDir) {
|
||||
console.log(` Root: ${chalk.dim(stats.storage.rootDir)}`)
|
||||
}
|
||||
|
||||
if (stats.writerLock) {
|
||||
console.log(chalk.bold('\nWriter Lock:'))
|
||||
console.log(
|
||||
` PID ${stats.writerLock.pid} on ${stats.writerLock.hostname} ` +
|
||||
chalk.dim(`(since ${stats.writerLock.startedAt})`)
|
||||
)
|
||||
}
|
||||
|
||||
console.log(chalk.dim(`\nMode: ${stats.mode} • Brainy v${stats.version}`))
|
||||
|
||||
// close() releases the writer lock and indexes, but global timers
|
||||
// (UnifiedCache bookkeeping, PathResolver stats) keep the event loop
|
||||
// alive. CLI commands are one-shot — exit explicitly.
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
} catch (error: unknown) {
|
||||
spinner.fail('Failed to get statistics')
|
||||
console.error(chalk.red(error.message))
|
||||
console.error(chalk.red((error as Error).message))
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes === 0) return '0 B'
|
||||
const k = 1024
|
||||
const sizes = ['B', 'KB', 'MB', 'GB']
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k))
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]
|
||||
}
|
||||
|
|
@ -1,12 +1,13 @@
|
|||
/**
|
||||
* Import Commands - Neural Import & Data Import
|
||||
* @module cli/commands/import
|
||||
* @description Import commands — neural import and data import.
|
||||
*
|
||||
* Complete import system exposing ALL Brainy import capabilities:
|
||||
* - UniversalImportAPI: Neural import with AI type matching
|
||||
* - DirectoryImporter: VFS directory imports
|
||||
* - DataAPI: Backup/restore
|
||||
*
|
||||
* Supports: files, directories, URLs, all formats
|
||||
* Supports: files, directories, URLs, all formats. Backup/restore is the
|
||||
* snapshot command family (`brainy snapshot` / `brainy restore`).
|
||||
*/
|
||||
|
||||
import chalk from 'chalk'
|
||||
|
|
|
|||
|
|
@ -85,6 +85,12 @@ ${chalk.cyan('Snapshot:')}
|
|||
`.trim())
|
||||
|
||||
formatOutput({ path: target, generation: db.generation, time: elapsed }, options)
|
||||
|
||||
// close() releases the writer lock and indexes, but global timers
|
||||
// (UnifiedCache bookkeeping, PathResolver stats) keep the event loop
|
||||
// alive. CLI commands are one-shot — exit explicitly.
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
} catch (error: any) {
|
||||
if (spinner) spinner.fail('Snapshot failed')
|
||||
console.error(chalk.red('Error:'), error.message)
|
||||
|
|
@ -119,7 +125,7 @@ ${chalk.cyan('Snapshot:')}
|
|||
])
|
||||
if (!confirm) {
|
||||
console.log(chalk.yellow('Restore cancelled'))
|
||||
return
|
||||
process.exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -136,6 +142,10 @@ ${chalk.cyan('Snapshot:')}
|
|||
)
|
||||
|
||||
formatOutput({ path: target, generation: brain.generation(), time: elapsed }, options)
|
||||
|
||||
// One-shot command — see snapshot() for why the explicit exit.
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
} catch (error: any) {
|
||||
if (spinner) spinner.fail('Restore failed')
|
||||
console.error(chalk.red('Error:'), error.message)
|
||||
|
|
@ -161,7 +171,8 @@ ${chalk.cyan('Snapshot:')}
|
|||
if (entries.length === 0) {
|
||||
console.log(chalk.yellow('\nNo committed transactions yet (the log records transact() batches).\n'))
|
||||
formatOutput([], options)
|
||||
return
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
console.log(chalk.cyan(`\nTransaction Log (last ${entries.length}):\n`))
|
||||
|
|
@ -178,6 +189,10 @@ ${chalk.cyan('Snapshot:')}
|
|||
console.log()
|
||||
|
||||
formatOutput(entries, options)
|
||||
|
||||
// One-shot command — see snapshot() for why the explicit exit.
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
} catch (error: any) {
|
||||
console.error(chalk.red('Error:'), error.message)
|
||||
if (options.verbose) console.error(error)
|
||||
|
|
@ -200,6 +215,10 @@ ${chalk.cyan('Snapshot:')}
|
|||
console.log(`${chalk.cyan('Generation:')} ${chalk.green(String(generation))}`)
|
||||
|
||||
formatOutput({ generation }, options)
|
||||
|
||||
// One-shot command — see snapshot() for why the explicit exit.
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
} catch (error: any) {
|
||||
console.error(chalk.red('Error:'), error.message)
|
||||
if (options.verbose) console.error(error)
|
||||
|
|
|
|||
|
|
@ -142,11 +142,11 @@ export const utilityCommands = {
|
|||
|
||||
// Show warning before clearing
|
||||
console.log(chalk.yellow('\n⚠️ WARNING: This will permanently delete ALL data!'))
|
||||
const dataApi = await brain.data()
|
||||
|
||||
// Clear all data
|
||||
// Clear all data (entities, relationships, and every index)
|
||||
spinner.text = 'Clearing all data...'
|
||||
await dataApi.clear({ entities: true, relations: true })
|
||||
await brain.init()
|
||||
await brain.clear()
|
||||
|
||||
spinner.succeed('Database cleared')
|
||||
|
||||
|
|
@ -156,6 +156,12 @@ export const utilityCommands = {
|
|||
} else {
|
||||
formatOutput({ cleared: true, success: true }, options)
|
||||
}
|
||||
|
||||
// close() releases the writer lock and indexes, but global timers
|
||||
// (UnifiedCache bookkeeping, PathResolver stats) keep the event loop
|
||||
// alive. CLI commands are one-shot — exit explicitly.
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
} catch (error: any) {
|
||||
spinner.fail('Cleanup failed')
|
||||
console.error(chalk.red(error.message))
|
||||
|
|
|
|||
|
|
@ -173,12 +173,6 @@ program
|
|||
.option('--skip-node-modules', 'Skip node_modules', true)
|
||||
.action(importCommands.import)
|
||||
|
||||
program
|
||||
.command('export [file]')
|
||||
.description('Export database')
|
||||
.option('-f, --format <format>', 'Output format (json|csv|jsonl)', 'json')
|
||||
.action(coreCommands.export)
|
||||
|
||||
program
|
||||
.command('diagnostics')
|
||||
.alias('diag')
|
||||
|
|
@ -549,11 +543,12 @@ program
|
|||
|
||||
// ===== Data Management Commands =====
|
||||
|
||||
|
||||
program
|
||||
program
|
||||
.command('data-stats')
|
||||
.description('Show detailed database statistics')
|
||||
.option('--verbose', 'List every indexed field name')
|
||||
.option('--json', 'Output as JSON')
|
||||
.option('--pretty', 'Pretty print JSON')
|
||||
.action(dataCommands.stats)
|
||||
|
||||
// ===== Inspect Commands =====
|
||||
|
|
@ -764,7 +759,12 @@ program
|
|||
|
||||
program
|
||||
.command('snapshot <path>')
|
||||
.description('📸 Persist the current generation as a self-contained snapshot (instant hard links)')
|
||||
.alias('export')
|
||||
.description(
|
||||
'📸 Persist the current generation as a self-contained snapshot (instant hard links). ' +
|
||||
'A snapshot is the full-fidelity export format: open it with Brainy.load(path) or ' +
|
||||
'load it wholesale with `brainy restore`. For ingesting external data files, see `brainy import`.'
|
||||
)
|
||||
.action(snapshotCommands.snapshot)
|
||||
|
||||
program
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue