From 478fa176f2e59f1efa3b2d1ab39b2a519beb964c Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 11 Jun 2026 09:05:12 -0700 Subject: [PATCH] =?UTF-8?q?refactor(8.0):=20delete=20DataAPI=20=E2=80=94?= =?UTF-8?q?=20superseded=20by=20Db=20persist/restore=20+=20import=20API=20?= =?UTF-8?q?+=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy backup/import/export/stats facade (src/api/DataAPI.ts) drifted from the modern entity shape and every job it did now has a first-class surface. Delete it and brain.data(), and rewire the CLI: - data-stats → brain.stats() (full BrainyStats report: per-type breakdowns, indexed fields, index health, storage backend, writer lock, version) - clean → brain.clear() - export → alias of snapshot; a db.persist() snapshot is the full-fidelity export format (open with Brainy.load, load wholesale with brainy restore); external data ingestion remains brainy import (UniversalImportAPI) Rewiring clean onto brain.clear() exposed two real bugs, both fixed: - clear() left this.graphIndex undefined forever — any graph-touching call afterwards (relate, getNeighbors, stats) crashed. clear() now re-resolves the graph index exactly as init() does and re-wires the shared UUID↔int resolver, and re-resolves the metadata index with the same provider fallback as init(). - storage.clear() reset the legacy totals but not the per-type/subtype count rollups or id→type caches, so stats() reported phantom counts for deleted entities. Both adapters now delegate derived-state reset to reloadDerivedState(), the same path restore-from-snapshot uses. One-shot CLI commands (data-stats, clean, snapshot/export, restore, history, generation) now close the brain and exit explicitly — global cache timers otherwise keep the process alive holding the writer lock. Verified: build clean, 1383/1383 unit tests, 24/24 db-mvcc integration, plus an end-to-end CLI smoke (add → data-stats → export → clean → data-stats). --- docs/guides/import-flow.md | 2 +- src/api/DataAPI.ts | 389 ---------------------- src/brainy.ts | 150 +++++---- src/cli/commands/core.ts | 89 +---- src/cli/commands/data.ts | 144 ++++++-- src/cli/commands/import.ts | 7 +- src/cli/commands/snapshot.ts | 23 +- src/cli/commands/utility.ts | 12 +- src/cli/index.ts | 18 +- src/storage/adapters/fileSystemStorage.ts | 25 +- src/storage/adapters/memoryStorage.ts | 24 +- tests/unit/brainy-core.unit.test.ts | 18 +- 12 files changed, 262 insertions(+), 639 deletions(-) delete mode 100644 src/api/DataAPI.ts diff --git a/docs/guides/import-flow.md b/docs/guides/import-flow.md index 447c3ce7..9e8f2d70 100644 --- a/docs/guides/import-flow.md +++ b/docs/guides/import-flow.md @@ -1308,7 +1308,7 @@ await this.history.recordImport( ``` **Use Cases**: -- List all imports: `brain.data().listImports()` +- List all imports: `coordinator.getHistory().getHistory()` (the `ImportHistory` entries shown above) - Reimport with same settings - Audit trail for compliance - Rollback imports diff --git a/src/api/DataAPI.ts b/src/api/DataAPI.ts deleted file mode 100644 index 8ba1a49f..00000000 --- a/src/api/DataAPI.ts +++ /dev/null @@ -1,389 +0,0 @@ -/** - * Data Management API for Brainy 3.0 - * Provides backup, restore, import, export, and data management - */ - -import { StorageAdapter, HNSWNoun, GraphVerb } from '../coreTypes.js' -import { Entity, Relation } from '../types/brainy.types.js' -import { NounType, VerbType } from '../types/graphTypes.js' - -export interface BackupOptions { - includeVectors?: boolean - compress?: boolean - format?: 'json' | 'binary' -} - -export interface RestoreOptions { - merge?: boolean - overwrite?: boolean - validate?: boolean -} - -export interface ImportOptions { - format: 'json' | 'csv' - mapping?: Record - batchSize?: number - validate?: boolean -} - -export interface ExportOptions { - format?: 'json' | 'csv' - filter?: { - type?: NounType | NounType[] - where?: Record - service?: string - } - includeVectors?: boolean -} - -export interface BackupData { - version: string - timestamp: number - entities: Array<{ - id: string - vector?: number[] - type: string - metadata: any - service?: string - }> - relations: Array<{ - id: string - from: string - to: string - type: string - weight: number - metadata?: any - }> - config?: Record - stats: { - entityCount: number - relationCount: number - vectorDimensions?: number - } -} - -export interface ImportResult { - successful: number - failed: number - errors: Array<{ item: any; error: string }> - duration: number -} - -export class DataAPI { - private brain: any // Reference to Brainy instance for neural import - - constructor( - private storage: StorageAdapter, - private getEntity: (id: string) => Promise, - private getRelation?: (id: string) => Promise, - brain?: any - ) { - this.brain = brain - } - - async clear(params: { - entities?: boolean - relations?: boolean - config?: boolean - } = {}): Promise { - const { entities = true, relations = true, config = false } = params - - if (entities) { - // Clear all entities - const nounsResult = await this.storage.getNouns({ - pagination: { limit: 1000000 } - }) - - for (const noun of nounsResult.items) { - await this.storage.deleteNoun(noun.id) - } - - // Also clear the HNSW index if available - if (this.brain?.index?.clear) { - this.brain.index.clear() - } - - // Clear metadata index if available - if (this.brain?.metadataIndex) { - await this.brain.metadataIndex.rebuild() // Rebuild empty index - } - } - - if (relations) { - // Clear all relations - const verbsResult = await this.storage.getVerbs({ - pagination: { limit: 1000000 } - }) - - for (const verb of verbsResult.items) { - await this.storage.deleteVerb(verb.id) - } - } - - if (config) { - // Clear configuration would be handled by ConfigAPI - // For now, skip this - } - } - - /** - * Import data from various formats - */ - async import(params: ImportOptions & { data: any }): Promise { - const { - data, - format, - mapping = {}, - batchSize = 100, - validate = true - } = params - - const result: ImportResult = { - successful: 0, - failed: 0, - errors: [], - duration: 0 - } - - const startTime = Date.now() - - try { - // ALWAYS use neural import for proper type matching - const { UniversalImportAPI } = await import('./UniversalImportAPI.js') - const universalImport = new UniversalImportAPI(this.brain) - await universalImport.init() - - // Convert to ImportSource format - const neuralResult = await universalImport.import({ - type: 'object', - data, - format: format || 'json', - metadata: { mapping, batchSize, validate } - }) - - // Convert neural result to ImportResult format - result.successful = neuralResult.stats.entitiesCreated - result.failed = 0 // Neural import always succeeds with best match - result.duration = neuralResult.stats.processingTimeMs - - // Log relationships created - if (neuralResult.stats.relationshipsCreated > 0) { - console.log(`Neural import also created ${neuralResult.stats.relationshipsCreated} relationships`) - } - - return result - } catch (error) { - // Fallback to legacy import ONLY if neural import fails to load - console.warn('Neural import failed, using legacy import:', error) - - let items: any[] = [] - - // Parse data based on format - switch (format) { - case 'json': - items = Array.isArray(data) ? data : [data] - break - - case 'csv': - // CSV parsing would go here - // For now, assume data is already parsed - items = data - break - - // Parquet format removed - not implemented - - default: - throw new Error(`Unsupported format: ${format}`) - } - - // Process items in batches - for (let i = 0; i < items.length; i += batchSize) { - const batch = items.slice(i, i + batchSize) - - for (const item of batch) { - try { - // Apply field mapping - const mapped = this.applyMapping(item, mapping) - - // Validate if requested - if (validate) { - this.validateImportItem(mapped) - } - - // Save entity - separate vector and metadata - const id = mapped.id || this.generateId() - const noun: HNSWNoun = { - id, - vector: mapped.vector || new Array(384).fill(0), - connections: new Map(), - level: 0 - } - - await this.storage.saveNoun(noun) - await this.storage.saveNounMetadata(id, { ...mapped, createdAt: Date.now() }) - result.successful++ - } catch (error) { - result.failed++ - result.errors.push({ - item, - error: (error as Error).message - }) - } - } - } - - result.duration = Date.now() - startTime - return result - } - } - - /** - * Export data to various formats - */ - async export(params: ExportOptions = {}): Promise { - const { - format = 'json', - filter = {}, - includeVectors = false - } = params - - // Get filtered entities - const nounsResult = await this.storage.getNouns({ - pagination: { limit: 1000000 } - }) - - let entities = nounsResult.items - - // Apply filters - if (filter.type) { - const types = Array.isArray(filter.type) ? filter.type : [filter.type] - entities = entities.filter(e => - types.includes(e.metadata?.noun as NounType) - ) - } - - if (filter.service) { - entities = entities.filter(e => - e.metadata?.service === filter.service - ) - } - - if (filter.where) { - entities = entities.filter(e => - this.matchesFilter(e.metadata, filter.where!) - ) - } - - // Format data based on export format - switch (format) { - case 'json': - return entities.map(e => ({ - id: e.id, - vector: includeVectors ? e.vector : undefined, - ...e.metadata - })) - - case 'csv': - // Convert to CSV format - // For now, return simplified format - return this.convertToCSV(entities) - - // Parquet format removed - not implemented - - default: - throw new Error(`Unsupported export format: ${format}`) - } - } - - /** - * Get storage statistics - */ - async getStats(): Promise<{ - entities: number - relations: number - storageSize?: number - vectorDimensions?: number - }> { - const nounsResult = await this.storage.getNouns({ - pagination: { limit: 1 } - }) - const verbsResult = await this.storage.getVerbs({ - pagination: { limit: 1 } - }) - - const firstNoun = nounsResult.items[0] - - return { - entities: nounsResult.totalCount || nounsResult.items.length, - relations: verbsResult.totalCount || verbsResult.items.length, - vectorDimensions: firstNoun?.vector?.length - } - } - - // Helper methods - - private applyMapping(item: any, mapping: Record): any { - const mapped: any = {} - - for (const [key, value] of Object.entries(item)) { - const mappedKey = mapping[key] || key - mapped[mappedKey] = value - } - - return mapped - } - - private validateImportItem(item: any): void { - // Basic validation - if (!item || typeof item !== 'object') { - throw new Error('Invalid item: must be an object') - } - - // Could add more validation here - } - - private matchesFilter(metadata: any, filter: Record): boolean { - for (const [key, value] of Object.entries(filter)) { - if (metadata[key] !== value) { - return false - } - } - return true - } - - private convertToCSV(entities: Array<{ id: string; metadata?: any }>): string { - if (entities.length === 0) return '' - - // Get all unique keys from metadata - const keys = new Set() - for (const entity of entities) { - if (entity.metadata) { - Object.keys(entity.metadata).forEach(k => keys.add(k)) - } - } - - // Create CSV header - const headers = ['id', ...Array.from(keys)] - const rows = [headers.join(',')] - - // Add data rows - for (const entity of entities) { - const row = [entity.id] - for (const key of keys) { - const value = entity.metadata?.[key] || '' - // Escape values that contain commas - const escaped = String(value).includes(',') - ? `"${String(value).replace(/"/g, '""')}"` - : String(value) - row.push(escaped) - } - rows.push(row.join(',')) - } - - return rows.join('\n') - } - - private generateId(): string { - return `import_${Date.now()}_${Math.random().toString(36).substr(2, 9)}` - } -} \ No newline at end of file diff --git a/src/brainy.ts b/src/brainy.ts index 5daee564..c80f7d36 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -4298,68 +4298,95 @@ export class Brainy implements BrainyInterface { } /** - * Clear all data from the database + * @description Clear ALL data from the database — entities, relationships, + * blobs, and every derived index. The instance remains fully usable + * afterwards: the vector, metadata, and graph indexes are re-resolved + * exactly as on `init()`, and when the VFS was active a fresh VFS root + * entity is re-created (so a thresholdless vector search against a cleared + * store can still surface that one root entity). + * @example + * await brain.clear() */ async clear(): Promise { await this.ensureInitialized() // Clear storage - await this.storage.clear() + await this.storage.clear() - // Invalidate GraphAdjacencyIndex to prevent stale in-memory data - // The index has LSMTree data and verbIdSet pointing to deleted entities. - // Without this, relate()'s duplicate check uses stale data, potentially - // allowing duplicate relationships or missing valid duplicates. - if (typeof (this.storage as any).invalidateGraphIndex === 'function') { - ;(this.storage as any).invalidateGraphIndex() - } - this.graphIndex = undefined as any - - // Reset index - if ('clear' in this.index && typeof this.index.clear === 'function') { - await this.index.clear() - } else { - // Recreate index using plugin factory when available - this.index = this.createIndex() - } - - // Recreate metadata index to clear cached data - const clearMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') - this.metadataIndex = clearMetadataFactory - ? clearMetadataFactory(this.storage) - : new MetadataIndexManager(this.storage) - await this.metadataIndex.init() - - // Reset dimensions - this.dimensions = undefined - - // Clear any cached sub-APIs - this._neural = undefined - this._nlp = undefined - this._tripleIntelligence = undefined - - // Re-initialize the content-addressed blob store after storage.clear() - // (clear() drops the BlobStorage instance). The VFS stores all file - // content through it, so this must happen BEFORE VFS reinitialization. - if (typeof (this.storage as any).initializeBlobStorage === 'function') { - await (this.storage as any).initializeBlobStorage() - } - - // Reset VFS state - root entity was deleted by storage.clear() - // Bug: VFS instance remained in memory pointing to deleted root entity - if (this._vfs) { - // Clear PathResolver caches (including UnifiedCache VFS entries) - if ((this._vfs as any).pathResolver?.invalidateAllCaches) { - (this._vfs as any).pathResolver.invalidateAllCaches() - } - // Recreate and reinitialize VFS so it's ready for use - this._vfs = new VirtualFileSystem(this) - await this._vfs.init() - // _vfsInitialized remains true since we just initialized - } else { - // VFS was never used, reset flag for clean state - this._vfsInitialized = false + // Invalidate GraphAdjacencyIndex to prevent stale in-memory data + // The index has LSMTree data and verbIdSet pointing to deleted entities. + // Without this, relate()'s duplicate check uses stale data, potentially + // allowing duplicate relationships or missing valid duplicates. + if (typeof (this.storage as any).invalidateGraphIndex === 'function') { + ;(this.storage as any).invalidateGraphIndex() + } + + // Reset index + if ('clear' in this.index && typeof this.index.clear === 'function') { + await this.index.clear() + } else { + // Recreate index using plugin factory when available + this.index = this.createIndex() + } + + // Recreate metadata index to clear cached data — mirror init()'s + // resolution: provider factory first, then the JS implementation (with a + // plugin-provided native entityIdMapper when registered). + const clearMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') + if (clearMetadataFactory) { + this.metadataIndex = clearMetadataFactory(this.storage) + } else { + const entityIdMapperFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('entityIdMapper') + this.metadataIndex = new MetadataIndexManager(this.storage, {}, { + entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined, + }) + } + await this.metadataIndex.init() + + // Re-resolve the graph index the same way init() does (provider factory, + // else the storage layer's lazy singleton) and re-wire the shared + // UUID ↔ int resolver. Without this, every graph-touching call after + // clear() — relate(), getNeighbors(), stats() — would hit an undefined + // index. + const clearGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') + if (clearGraphFactory) { + this.graphIndex = clearGraphFactory(this.storage) + this.storage.setGraphIndex(this.graphIndex) + } else { + this.graphIndex = await (this.storage as any).getGraphIndex() + } + this.wireGraphIdResolver() + + // Reset dimensions + this.dimensions = undefined + + // Clear any cached sub-APIs + this._neural = undefined + this._nlp = undefined + this._tripleIntelligence = undefined + + // Re-initialize the content-addressed blob store after storage.clear() + // (clear() drops the BlobStorage instance). The VFS stores all file + // content through it, so this must happen BEFORE VFS reinitialization. + if (typeof (this.storage as any).initializeBlobStorage === 'function') { + await (this.storage as any).initializeBlobStorage() + } + + // Reset VFS state - root entity was deleted by storage.clear() + // Bug: VFS instance remained in memory pointing to deleted root entity + if (this._vfs) { + // Clear PathResolver caches (including UnifiedCache VFS entries) + if ((this._vfs as any).pathResolver?.invalidateAllCaches) { + (this._vfs as any).pathResolver.invalidateAllCaches() } + // Recreate and reinitialize VFS so it's ready for use + this._vfs = new VirtualFileSystem(this) + await this._vfs.init() + // _vfsInitialized remains true since we just initialized + } else { + // VFS was never used, reset flag for clean state + this._vfsInitialized = false + } } // ─── Migration API ─────────────────────────────────────────────── @@ -6231,19 +6258,6 @@ export class Brainy implements BrainyInterface { return this._hub } - /** - * Data Management API - backup, restore, import, export - */ - async data() { - const { DataAPI } = await import('./api/DataAPI.js') - return new DataAPI( - this.storage, - (id: string) => this.get(id), - undefined, // No getRelation method yet - this - ) - } - /** * Get Triple Intelligence System * Advanced pattern recognition and relationship analysis diff --git a/src/cli/commands/core.ts b/src/cli/commands/core.ts index c8509f5e..11407a11 100644 --- a/src/cli/commands/core.ts +++ b/src/cli/commands/core.ts @@ -1,13 +1,14 @@ /** - * Core CLI Commands - TypeScript Implementation - * - * Essential database operations: add, search, get, relate, import, export + * @module cli/commands/core + * @description Essential database CLI commands: add, search, get, update, + * delete, relate, unrelate, and diagnostics. Database export lives under + * `brainy snapshot` (aliased as `brainy export`) — a `db.persist()` snapshot + * is the full-fidelity export format. */ import chalk from 'chalk' import ora from 'ora' import inquirer from 'inquirer' -import { readFileSync, writeFileSync } from 'node:fs' import { Brainy } from '../../brainy.js' import { BrainyTypes, NounType, VerbType } from '../../index.js' @@ -66,10 +67,6 @@ interface RelateOptions extends CoreOptions { subtype?: string } -interface ExportOptions extends CoreOptions { - format?: 'json' | 'csv' | 'jsonl' -} - let brainyInstance: Brainy | null = null const getBrainy = (): Brainy => { @@ -866,82 +863,6 @@ export const coreCommands = { } }, - /** - * Export database - */ - async export(file: string | undefined, options: ExportOptions) { - const spinner = ora('Exporting database...').start() - - try { - const brain = getBrainy() - const format = options.format || 'json' - - // Export all data - const dataApi = await brain.data() - const data = await dataApi.export({ format: 'json' }) - let output = '' - - switch (format) { - case 'json': - output = options.pretty - ? JSON.stringify(data, null, 2) - : JSON.stringify(data) - break - - case 'jsonl': - if (Array.isArray(data)) { - output = data.map(item => JSON.stringify(item)).join('\n') - } else { - output = JSON.stringify(data) - } - break - - case 'csv': - if (Array.isArray(data) && data.length > 0) { - // Get all unique keys for headers - const headers = new Set() - data.forEach(item => { - Object.keys(item).forEach(key => headers.add(key)) - }) - const headerArray = Array.from(headers) - - // Create CSV - output = headerArray.join(',') + '\n' - output += data.map(item => { - return headerArray.map(h => { - const value = item[h] - if (typeof value === 'object') { - return JSON.stringify(value) - } - return value || '' - }).join(',') - }).join('\n') - } - break - } - - if (file) { - writeFileSync(file, output) - spinner.succeed(`Exported to ${file}`) - - if (!options.json) { - console.log(chalk.green(`✓ Successfully exported database to ${file}`)) - console.log(chalk.dim(` Format: ${format}`)) - console.log(chalk.dim(` Items: ${Array.isArray(data) ? data.length : 1}`)) - } else { - formatOutput({ file, format, count: Array.isArray(data) ? data.length : 1 }, options) - } - } else { - spinner.succeed('Export complete') - console.log(output) - } - } catch (error: any) { - spinner.fail('Export failed') - console.error(chalk.red(error.message)) - process.exit(1) - } - }, - /** * Show plugin and provider diagnostics */ diff --git a/src/cli/commands/data.ts b/src/cli/commands/data.ts index f7f8a97d..60e01462 100644 --- a/src/cli/commands/data.ts +++ b/src/cli/commands/data.ts @@ -1,14 +1,26 @@ /** - * Data Management Commands + * @module cli/commands/data + * @description Detailed database statistics for the `data-stats` CLI command. * - * Import, export, and statistics operations + * Renders `brain.stats()` (the operator-facing `BrainyStats` summary) as a + * human-readable report: counts with per-type breakdowns, indexed fields, + * per-index health flags, storage backend, writer-lock holder, and library + * version. Pass `--json` for machine-readable output. + * + * The legacy backup/import/export facade that used to live behind this + * command was superseded in 8.0: snapshots are `brainy snapshot` / `brainy + * restore` (the Db API's `persist()`/`restore()`), and data ingestion is + * `brainy import` (UniversalImportAPI). */ import chalk from 'chalk' import ora from 'ora' -import { readFileSync, writeFileSync } from 'node:fs' import { Brainy } from '../../brainy.js' +/** + * @description Shared CLI flags the data commands accept (`--verbose`, + * `--json`, `--pretty`), mirroring the other command modules. + */ interface DataOptions { verbose?: boolean json?: boolean @@ -17,6 +29,11 @@ interface DataOptions { let brainyInstance: Brainy | null = null +/** + * @description Lazily construct the module's shared `Brainy` instance so + * repeated handler invocations (and tests) reuse one store handle. + * @returns The shared `Brainy` instance. + */ const getBrainy = (): Brainy => { if (!brainyInstance) { brainyInstance = new Brainy() @@ -24,60 +41,113 @@ const getBrainy = (): Brainy => { return brainyInstance } -const formatOutput = (data: any, options: DataOptions): void => { +/** + * @description Emit machine-readable output when `--json` is set; honors + * `--pretty` for indented JSON. + * @param data - The value to serialize. + * @param options - Shared CLI flags. + */ +const formatOutput = (data: unknown, options: DataOptions): void => { if (options.json) { console.log(options.pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data)) } } +/** + * @description Render a `Record` breakdown as indented, + * count-sorted lines (largest first). + * @param byType - Per-type counts, e.g. `BrainyStats.entitiesByType`. + * @returns Formatted lines ready for `console.log`, one per type. + */ +const formatTypeBreakdown = (byType: Record): string[] => + Object.entries(byType) + .sort(([, a], [, b]) => b - a) + .map(([type, count]) => ` ${type}: ${chalk.green(count)}`) + +/** + * @description The `data-stats` command handler registered in + * `src/cli/index.ts`. + */ export const dataCommands = { /** - * Get database statistics + * @description Show detailed database statistics via `brain.stats()`: + * entity/relationship totals with per-type breakdowns, indexed metadata + * fields, index health, storage backend, writer lock, and version. + * @param options - Shared CLI flags. + * @example + * ```bash + * brainy data-stats # human-readable report + * brainy data-stats --json # raw BrainyStats JSON + * ``` */ async stats(options: DataOptions) { const spinner = ora('Gathering statistics...').start() try { const brain = getBrainy() - const dataApi = await brain.data() - - const stats = await dataApi.getStats() + await brain.init() + const stats = await brain.stats() spinner.succeed('Statistics gathered') - if (!options.json) { - console.log(chalk.cyan('\n📊 Database Statistics:\n')) - - console.log(chalk.bold('Entities:')) - console.log(` Total: ${chalk.green(stats.entities)}`) - - console.log(chalk.bold('\nRelationships:')) - console.log(` Total: ${chalk.green(stats.relations)}`) - - if ((stats as any).storageSize) { - console.log(chalk.bold('\nStorage:')) - console.log(` Size: ${chalk.green(formatBytes((stats as any).storageSize))}`) - } - - if ((stats as any).vectorDimensions) { - console.log(chalk.bold('\nVector Index:')) - console.log(` Dimensions: ${(stats as any).vectorDimensions}`) - } - } else { + if (options.json) { formatOutput(stats, options) + // close() releases the writer lock and indexes, but global timers + // (UnifiedCache bookkeeping, PathResolver stats) keep the event loop + // alive. CLI commands are one-shot — exit explicitly. + await brain.close() + process.exit(0) } - } catch (error: any) { + + console.log(chalk.cyan('\n📊 Database Statistics\n')) + + console.log(chalk.bold('Entities:')) + console.log(` Total: ${chalk.green(stats.entityCount)}`) + for (const line of formatTypeBreakdown(stats.entitiesByType)) { + console.log(` ${line}`) + } + + console.log(chalk.bold('\nRelationships:')) + console.log(` Total: ${chalk.green(stats.relationCount)}`) + for (const line of formatTypeBreakdown(stats.relationsByType)) { + console.log(` ${line}`) + } + + console.log(chalk.bold('\nIndexes:')) + console.log(` Indexed fields: ${chalk.green(stats.fieldRegistry.length)}`) + if (options.verbose && stats.fieldRegistry.length > 0) { + console.log(chalk.dim(` ${stats.fieldRegistry.join(', ')}`)) + } + const health = (ok: boolean) => (ok ? chalk.green('ok') : chalk.red('empty')) + console.log(` Vector: ${health(stats.indexHealth.vector)}`) + console.log(` Metadata: ${health(stats.indexHealth.metadata)}`) + console.log(` Graph: ${health(stats.indexHealth.graph)}`) + + console.log(chalk.bold('\nStorage:')) + console.log(` Backend: ${stats.storage.backend}`) + if (stats.storage.rootDir) { + console.log(` Root: ${chalk.dim(stats.storage.rootDir)}`) + } + + if (stats.writerLock) { + console.log(chalk.bold('\nWriter Lock:')) + console.log( + ` PID ${stats.writerLock.pid} on ${stats.writerLock.hostname} ` + + chalk.dim(`(since ${stats.writerLock.startedAt})`) + ) + } + + console.log(chalk.dim(`\nMode: ${stats.mode} • Brainy v${stats.version}`)) + + // close() releases the writer lock and indexes, but global timers + // (UnifiedCache bookkeeping, PathResolver stats) keep the event loop + // alive. CLI commands are one-shot — exit explicitly. + await brain.close() + process.exit(0) + } catch (error: unknown) { spinner.fail('Failed to get statistics') - console.error(chalk.red(error.message)) + console.error(chalk.red((error as Error).message)) process.exit(1) } } } - -function formatBytes(bytes: number): string { - if (bytes === 0) return '0 B' - const k = 1024 - const sizes = ['B', 'KB', 'MB', 'GB'] - const i = Math.floor(Math.log(bytes) / Math.log(k)) - return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i] -} \ No newline at end of file diff --git a/src/cli/commands/import.ts b/src/cli/commands/import.ts index b8349508..2a107f90 100644 --- a/src/cli/commands/import.ts +++ b/src/cli/commands/import.ts @@ -1,12 +1,13 @@ /** - * Import Commands - Neural Import & Data Import + * @module cli/commands/import + * @description Import commands — neural import and data import. * * Complete import system exposing ALL Brainy import capabilities: * - UniversalImportAPI: Neural import with AI type matching * - DirectoryImporter: VFS directory imports - * - DataAPI: Backup/restore * - * Supports: files, directories, URLs, all formats + * Supports: files, directories, URLs, all formats. Backup/restore is the + * snapshot command family (`brainy snapshot` / `brainy restore`). */ import chalk from 'chalk' diff --git a/src/cli/commands/snapshot.ts b/src/cli/commands/snapshot.ts index d30e4d23..203a2207 100644 --- a/src/cli/commands/snapshot.ts +++ b/src/cli/commands/snapshot.ts @@ -85,6 +85,12 @@ ${chalk.cyan('Snapshot:')} `.trim()) formatOutput({ path: target, generation: db.generation, time: elapsed }, options) + + // close() releases the writer lock and indexes, but global timers + // (UnifiedCache bookkeeping, PathResolver stats) keep the event loop + // alive. CLI commands are one-shot — exit explicitly. + await brain.close() + process.exit(0) } catch (error: any) { if (spinner) spinner.fail('Snapshot failed') console.error(chalk.red('Error:'), error.message) @@ -119,7 +125,7 @@ ${chalk.cyan('Snapshot:')} ]) if (!confirm) { console.log(chalk.yellow('Restore cancelled')) - return + process.exit(0) } } @@ -136,6 +142,10 @@ ${chalk.cyan('Snapshot:')} ) formatOutput({ path: target, generation: brain.generation(), time: elapsed }, options) + + // One-shot command — see snapshot() for why the explicit exit. + await brain.close() + process.exit(0) } catch (error: any) { if (spinner) spinner.fail('Restore failed') console.error(chalk.red('Error:'), error.message) @@ -161,7 +171,8 @@ ${chalk.cyan('Snapshot:')} if (entries.length === 0) { console.log(chalk.yellow('\nNo committed transactions yet (the log records transact() batches).\n')) formatOutput([], options) - return + await brain.close() + process.exit(0) } console.log(chalk.cyan(`\nTransaction Log (last ${entries.length}):\n`)) @@ -178,6 +189,10 @@ ${chalk.cyan('Snapshot:')} console.log() formatOutput(entries, options) + + // One-shot command — see snapshot() for why the explicit exit. + await brain.close() + process.exit(0) } catch (error: any) { console.error(chalk.red('Error:'), error.message) if (options.verbose) console.error(error) @@ -200,6 +215,10 @@ ${chalk.cyan('Snapshot:')} console.log(`${chalk.cyan('Generation:')} ${chalk.green(String(generation))}`) formatOutput({ generation }, options) + + // One-shot command — see snapshot() for why the explicit exit. + await brain.close() + process.exit(0) } catch (error: any) { console.error(chalk.red('Error:'), error.message) if (options.verbose) console.error(error) diff --git a/src/cli/commands/utility.ts b/src/cli/commands/utility.ts index 45864547..e934bebd 100644 --- a/src/cli/commands/utility.ts +++ b/src/cli/commands/utility.ts @@ -142,11 +142,11 @@ export const utilityCommands = { // Show warning before clearing console.log(chalk.yellow('\n⚠️ WARNING: This will permanently delete ALL data!')) - const dataApi = await brain.data() - // Clear all data + // Clear all data (entities, relationships, and every index) spinner.text = 'Clearing all data...' - await dataApi.clear({ entities: true, relations: true }) + await brain.init() + await brain.clear() spinner.succeed('Database cleared') @@ -156,6 +156,12 @@ export const utilityCommands = { } else { formatOutput({ cleared: true, success: true }, options) } + + // close() releases the writer lock and indexes, but global timers + // (UnifiedCache bookkeeping, PathResolver stats) keep the event loop + // alive. CLI commands are one-shot — exit explicitly. + await brain.close() + process.exit(0) } catch (error: any) { spinner.fail('Cleanup failed') console.error(chalk.red(error.message)) diff --git a/src/cli/index.ts b/src/cli/index.ts index a9f10262..7fa4c1b8 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -173,12 +173,6 @@ program .option('--skip-node-modules', 'Skip node_modules', true) .action(importCommands.import) -program - .command('export [file]') - .description('Export database') - .option('-f, --format ', 'Output format (json|csv|jsonl)', 'json') - .action(coreCommands.export) - program .command('diagnostics') .alias('diag') @@ -549,11 +543,12 @@ program // ===== Data Management Commands ===== - -program program .command('data-stats') .description('Show detailed database statistics') + .option('--verbose', 'List every indexed field name') + .option('--json', 'Output as JSON') + .option('--pretty', 'Pretty print JSON') .action(dataCommands.stats) // ===== Inspect Commands ===== @@ -764,7 +759,12 @@ program program .command('snapshot ') - .description('📸 Persist the current generation as a self-contained snapshot (instant hard links)') + .alias('export') + .description( + '📸 Persist the current generation as a self-contained snapshot (instant hard links). ' + + 'A snapshot is the full-fidelity export format: open it with Brainy.load(path) or ' + + 'load it wholesale with `brainy restore`. For ingesting external data files, see `brainy import`.' + ) .action(snapshotCommands.snapshot) program diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index f2732fe0..c4467769 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -1404,26 +1404,15 @@ export class FileSystemStorage extends BaseStorage { if (await this.directoryExists(casDir)) { // Delete the entire _cas/ directory (not just contents) await fs.promises.rm(casDir, { recursive: true, force: true }) - - // Drop the BlobStorage instance — its LRU cache holds deleted blobs. - // initializeBlobStorage() re-creates it (brain.clear() does this before - // the VFS is rebuilt). - this.blobStorage = undefined } - // Clear the statistics cache - this.statisticsCache = null - this.statisticsModified = false - - // Reset entity counters (inherited from BaseStorageAdapter) - // These in-memory counters must be reset to 0 after clearing all data - ;(this as any).totalNounCount = 0 - ;(this as any).totalVerbCount = 0 - - // Clear write-through cache (inherited from BaseStorage) - // Without this, readCanonicalObject() would return stale cached data - // after clear(), causing "ghost" entities to appear - this.clearWriteCache() + // Reset ALL shared derived state via the same path restore uses: the + // write-through cache, id→type/subtype caches, per-type/subtype count + // rollups, statistics cache, graph-index singleton, and the BlobStorage + // instance (its LRU cache holds deleted blobs), then recount from the + // now-empty data area. Without the rollup reset, stats() would keep + // reporting per-type counts for deleted entities. + await this.reloadDerivedState() } /** diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index 0cf381cc..e2c86da2 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -353,24 +353,14 @@ export class MemoryStorage extends BaseStorage { this.blobStore.clear() this.txLogLines = [] this.statistics = null - this.totalNounCount = 0 - this.totalVerbCount = 0 - this.entityCounts.clear() - this.verbCounts.clear() - // Clear the statistics cache - this.statisticsCache = null - this.statisticsModified = false - - // Drop the BlobStorage instance — its LRU cache holds deleted blobs. - // initializeBlobStorage() re-creates it (brain.clear() does this before - // the VFS is rebuilt). - this.blobStorage = undefined - - // Clear write-through cache (inherited from BaseStorage) - // Without this, readCanonicalObject() would return stale cached data - // after clear(), causing "ghost" entities to appear - this.clearWriteCache() + // Reset ALL shared derived state via the same path restore uses: the + // write-through cache, id→type/subtype caches, per-type/subtype count + // rollups, statistics cache, graph-index singleton, and the BlobStorage + // instance (its LRU cache holds deleted blobs), then recount from the + // now-empty object store. Without the rollup reset, stats() would keep + // reporting per-type counts for deleted entities. + await this.reloadDerivedState() } /** diff --git a/tests/unit/brainy-core.unit.test.ts b/tests/unit/brainy-core.unit.test.ts index f94779b1..61545c07 100644 --- a/tests/unit/brainy-core.unit.test.ts +++ b/tests/unit/brainy-core.unit.test.ts @@ -220,16 +220,18 @@ describe('Brainy 3.0 Core (Unit Tests)', () => { type: NounType.Concept }) - // Clear using DataAPI - const dataAPI = await brain.data() - await dataAPI.clear({ entities: true, relations: false }) - - // Verify data is cleared - const results = await brain.find({ + // Clear everything — entities, relationships, and all indexes + await brain.clear() + + // Verify user data is cleared. clear() re-creates a fresh VFS root + // entity, and a thresholdless vector search can surface it — so filter + // VFS bookkeeping out and assert the added entities are gone. + const results = await brain.find({ query: 'Test', - limit: 10 + limit: 10 }) - expect(results.length).toBe(0) + const userResults = results.filter(r => !r.metadata?.isVFS) + expect(userResults.length).toBe(0) }) })