soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat(8.0): API simplification — remove neural()/Db.search, one storage path key, integration→0

Browse source 
8.0 RC cleanup toward "one place per thing, zero-config, no deprecation":

- Remove the `brain.neural()` clustering namespace (ImprovedNeuralAPI + the dead
  legacy NeuralAPI + the neural CLI + neural-only types). Similarity is `find({vector})`
  / `similar({to})`; attribute grouping is the aggregation `GROUP BY` engine. The separate
  entity-extraction / smart-import feature (NeuralImport, NeuralEntityExtractor, SmartExtractor,
  NaturalLanguageProcessor, `brain.extract()`/`brain.nlp()`) is kept.
- Remove `Db.search()`; `find()` is the one query verb (accepts a bare string or FindParams).
  Fix the bundled MCP client, which called a non-existent `brain.search(query, limit)` →
  now `find({ query, limit })`.
- Storage config: collapse to one canonical top-level `path` key. The pre-8.0 aliases
  (`rootDirectory`, `options.*`, `fileSystemStorage.*`) are removed and now THROW with the
  exact rename instead of silently defaulting to `./brainy-data` on upgrade. A single resolver
  feeds createStorage, the 7.x→8.0 migration probe, and the plugin-factory handoff, so a native
  storage provider resolves the identical root (no split-brain).
- Fix `similar({ threshold })`: the min-similarity filter was silently dropped; it is now
  applied as a post-filter on `result.score` (the documented way to bound semantic results).
- Fix `vfs.rename()` on a directory: child path updates spread the entity vector into `update()`
  and failed dimension validation; they are metadata-only updates now.
- Fix `vfs.move()`: copy+delete orphaned the content-addressed content blob (the destination
  shared the source hash, then unlink removed it). `move()` now delegates to `rename()` — an
  in-place path change that preserves the blob and the entity id, for files and directories.
- Fix streaming import: the bulk fast path never flushed mid-import nor signalled queryability.
  Entity writes are now chunked by a progressive flush interval (100 → 1000 → 5000); each chunk
  flushes and emits `progress.queryable`, so imported data is queryable during the import.
- Sweep all docs, comments, and JSDoc for the removed/changed APIs.

Integration suite: 49 files / 588 passed / 0 failed. Unit: 80 files / 1456 passed, no type errors.
This commit is contained in:
David Snelling 2026-06-20 13:31:11 -07:00
parent 0c4a51c24e
commit 606445cd61
74 changed files with 712 additions and 7470 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/cli/commands/inspect.ts
View file

@ -44,7 +44,7 @@ async function openReader(rootDir: string, options: OpenOptions): Promise<Brainy
// ack lands before we open the main reader and have it rebuild indexes.
try {
const probe = await Brainy.openReadOnly({
storage: { type: 'filesystem', rootDirectory: rootDir }
storage: { type: 'filesystem', path: rootDir }
})
const acked = await probe.requestFlush({ timeoutMs: flushTimeoutMs })
await probe.close()
@ -61,7 +61,7 @@ async function openReader(rootDir: string, options: OpenOptions): Promise<Brainy
}
return Brainy.openReadOnly({
storage: { type: 'filesystem', rootDirectory: rootDir }
storage: { type: 'filesystem', path: rootDir }
})
}
@ -379,7 +379,7 @@ export const inspectCommands = {
const spinner = options.quiet ? null : ora('Opening writer…').start()
try {
const brain = new Brainy({
storage: { type: 'filesystem', rootDirectory: path },
storage: { type: 'filesystem', path: path },
mode: 'writer',
force: options.force
})

518
src/cli/commands/neural.ts
View file

@ -1,518 +0,0 @@
/**
* @module cli/commands/neural
* @description Neural CLI commands: semantic similarity, clustering,
* hierarchy, neighbors, outlier detection, and visualization data export.
* Registered in `src/cli/index.ts` as `similar` / `cluster` / `related` /
* `hierarchy` / `outliers` / `visualize`. Each command is one-shot: it
* initializes the shared Brainy instance, runs the neural operation, then
* closes the store and exits explicitly.
*/
import inquirer from 'inquirer'
import chalk from 'chalk'
import ora, { type Ora } from 'ora'
import fs from 'node:fs'
import path from 'node:path'
import { Brainy } from '../../brainy.js'
import type { NeuralClusterParams } from '../../types/brainy.types.js'
interface CommandArguments {
action?: string;
id?: string;
query?: string;
threshold?: number;
format?: string;
output?: string;
limit?: number;
algorithm?: string;
dimensions?: number;
explain?: boolean;
_: string[];
}
let brainyInstance: Brainy | null = null
const getBrainy = (): Brainy => {
if (!brainyInstance) {
brainyInstance = new Brainy()
}
return brainyInstance
}
async function handleSimilarCommand(neural: any, argv: CommandArguments): Promise<void> {
const spinner = ora('🧠 Calculating semantic similarity...').start()
try {
let itemA: string, itemB: string
if (argv.id && argv.query) {
itemA = argv.id
itemB = argv.query
} else if (argv._ && argv._.length >= 3) {
itemA = argv._[1]
itemB = argv._[2]
} else {
spinner.stop()
const answers = await inquirer.prompt([
{
type: 'input',
name: 'itemA',
message: 'First item (ID or text):',
validate: (input: string) => input.length > 0
},
{
type: 'input',
name: 'itemB',
message: 'Second item (ID or text):',
validate: (input: string) => input.length > 0
}
])
itemA = answers.itemA
itemB = answers.itemB
spinner.start()
}
const result = await neural.similar(itemA, itemB, {
explain: argv.explain,
includeBreakdown: argv.explain
})
spinner.succeed('✅ Similarity calculated')
if (typeof result === 'number') {
console.log(`\n🔗 Similarity: ${chalk.cyan((result * 100).toFixed(1))}%`)
} else {
console.log(`\n🔗 Similarity: ${chalk.cyan((result.score * 100).toFixed(1))}%`)
if (result.explanation) {
console.log(`💭 Explanation: ${result.explanation}`)
}
if (result.breakdown) {
console.log('\n📊 Breakdown:')
console.log(` Semantic: ${chalk.yellow((result.breakdown.semantic! * 100).toFixed(1))}%`)
if (result.breakdown.taxonomic !== undefined) {
console.log(` Taxonomic: ${chalk.yellow((result.breakdown.taxonomic * 100).toFixed(1))}%`)
}
if (result.breakdown.contextual !== undefined) {
console.log(` Contextual: ${chalk.yellow((result.breakdown.contextual * 100).toFixed(1))}%`)
}
}
if (result.hierarchy) {
console.log(`\n🌳 Hierarchy: ${result.hierarchy.sharedParent ?
`Shared parent at distance ${result.hierarchy.distance}` :
'No shared parent found'}`)
}
}
if (argv.output) {
await saveToFile(argv.output, result, argv.format!)
}
} catch (error) {
spinner.fail('💥 Failed to calculate similarity')
throw error
}
}
async function handleClustersCommand(neural: any, argv: CommandArguments): Promise<void> {
let spinner: Ora | null = null
try {
let options: any = {
// --algorithm arrives as a raw CLI string; the interactive prompt below
// and neural.clusters() constrain it to the supported algorithms.
algorithm: argv.algorithm as NeuralClusterParams['algorithm'],
threshold: argv.threshold,
maxClusters: argv.limit
}
// Interactive mode if no algorithm specified or using defaults
if (!argv.algorithm || argv.algorithm === 'hierarchical') {
const answers = await inquirer.prompt([
{
type: 'list',
name: 'algorithm',
message: 'Choose clustering algorithm:',
default: argv.algorithm || 'hierarchical',
choices: [
{ name: '🌳 Hierarchical (Tree-based, best for natural grouping)', value: 'hierarchical' },
{ name: '📊 K-Means (Fixed number of clusters)', value: 'kmeans' },
{ name: '🎯 DBSCAN (Density-based, finds arbitrary shapes)', value: 'dbscan' }
]
},
{
type: 'number',
name: 'maxClusters',
message: 'Maximum number of clusters:',
default: argv.limit || 5,
when: (answers: any) => answers.algorithm === 'kmeans'
},
{
type: 'number',
name: 'threshold',
message: 'Similarity threshold (0-1):',
default: argv.threshold || 0.7,
validate: (input: number) => (input >= 0 && input <= 1) || 'Must be between 0 and 1'
}
])
options = {
algorithm: answers.algorithm,
threshold: answers.threshold,
maxClusters: answers.maxClusters || options.maxClusters
}
}
const spinner = ora('🎯 Finding semantic clusters...').start()
const clusters = await neural.clusters(argv.query || options)
spinner.succeed(`✅ Found ${clusters.length} clusters`)
if (argv.format === 'json') {
console.log(JSON.stringify(clusters, null, 2))
} else {
console.log(`\n🎯 ${chalk.cyan(clusters.length)} Semantic Clusters:\n`)
clusters.forEach((cluster, index) => {
console.log(`${chalk.yellow(`Cluster ${index + 1}:`)} ${cluster.label || cluster.id}`)
console.log(` 📊 Confidence: ${chalk.green((cluster.confidence * 100).toFixed(1))}%`)
console.log(` 👥 Members: ${cluster.members.length}`)
if (cluster.members.length <= 5) {
cluster.members.forEach(member => {
console.log(` • ${member}`)
})
} else {
cluster.members.slice(0, 3).forEach(member => {
console.log(` • ${member}`)
})
console.log(` ... and ${cluster.members.length - 3} more`)
}
console.log()
})
}
if (argv.output) {
await saveToFile(argv.output, clusters, argv.format!)
}
} catch (error) {
if (spinner) spinner.fail('💥 Failed to find clusters')
throw error
}
}
async function handleHierarchyCommand(neural: any, argv: CommandArguments): Promise<void> {
const spinner = ora('🌳 Building semantic hierarchy...').start()
try {
const id = argv.id || argv._[1]
if (!id) {
spinner.stop()
const answer = await inquirer.prompt([{
type: 'input',
name: 'id',
message: 'Enter item ID:',
validate: (input: string) => input.length > 0
}])
spinner.start()
const hierarchy = await neural.hierarchy(answer.id)
displayHierarchy(hierarchy)
} else {
const hierarchy = await neural.hierarchy(id)
spinner.succeed('✅ Hierarchy built')
displayHierarchy(hierarchy)
}
if (argv.output) {
const hierarchy = await neural.hierarchy(id || argv._[1])
await saveToFile(argv.output, hierarchy, argv.format!)
}
} catch (error) {
spinner.fail('💥 Failed to build hierarchy')
throw error
}
}
function displayHierarchy(hierarchy: any): void {
console.log(`\n🌳 Semantic Hierarchy for ${chalk.cyan(hierarchy.self.id)}:`)
if (hierarchy.root) {
console.log(`🔝 Root: ${hierarchy.root.id} (${(hierarchy.root.similarity * 100).toFixed(1)}%)`)
}
if (hierarchy.grandparent) {
console.log(`👴 Grandparent: ${hierarchy.grandparent.id} (${(hierarchy.grandparent.similarity * 100).toFixed(1)}%)`)
}
if (hierarchy.parent) {
console.log(`👨 Parent: ${hierarchy.parent.id} (${(hierarchy.parent.similarity * 100).toFixed(1)}%)`)
}
console.log(`🎯 ${chalk.bold('Self:')} ${hierarchy.self.id}`)
if (hierarchy.siblings && hierarchy.siblings.length > 0) {
console.log(`👥 Siblings: ${hierarchy.siblings.length}`)
hierarchy.siblings.forEach((sibling: any) => {
console.log(` • ${sibling.id} (${(sibling.similarity * 100).toFixed(1)}%)`)
})
}
if (hierarchy.children && hierarchy.children.length > 0) {
console.log(`👶 Children: ${hierarchy.children.length}`)
hierarchy.children.forEach((child: any) => {
console.log(` • ${child.id} (${(child.similarity * 100).toFixed(1)}%)`)
})
}
}
async function handleNeighborsCommand(neural: any, argv: CommandArguments): Promise<void> {
const spinner = ora('🕸️ Finding semantic neighbors...').start()
try {
const id = argv.id || argv._[1]
if (!id) {
spinner.stop()
const answer = await inquirer.prompt([{
type: 'input',
name: 'id',
message: 'Enter item ID:',
validate: (input: string) => input.length > 0
}])
spinner.start()
}
const targetId = id || (await inquirer.prompt([{
type: 'input',
name: 'id',
message: 'Enter item ID:',
validate: (input: string) => input.length > 0
}])).id
const graph = await neural.neighbors(targetId, {
limit: argv.limit,
includeEdges: true
})
spinner.succeed(`✅ Found ${graph.neighbors.length} neighbors`)
console.log(`\n🕸️ Neighbors of ${chalk.cyan(graph.center)}:`)
graph.neighbors.forEach((neighbor, index) => {
console.log(`${index + 1}. ${neighbor.id} (${(neighbor.similarity * 100).toFixed(1)}%)`)
if (neighbor.type) {
console.log(` Type: ${neighbor.type}`)
}
if (neighbor.connections) {
console.log(` Connections: ${neighbor.connections}`)
}
})
if (graph.edges && graph.edges.length > 0) {
console.log(`\n🔗 ${graph.edges.length} semantic connections found`)
}
if (argv.output) {
await saveToFile(argv.output, graph, argv.format!)
}
} catch (error) {
spinner.fail('💥 Failed to find neighbors')
throw error
}
}
async function handleOutliersCommand(neural: any, argv: CommandArguments): Promise<void> {
const spinner = ora('🚨 Detecting semantic outliers...').start()
try {
const outliers = await neural.outliers(argv.threshold)
spinner.succeed(`✅ Found ${outliers.length} outliers`)
if (outliers.length === 0) {
console.log('\n🎉 No outliers detected - all items are well connected!')
} else {
console.log(`\n🚨 ${chalk.red(outliers.length)} Semantic Outliers:`)
outliers.forEach((outlier, index) => {
console.log(`${index + 1}. ${outlier}`)
})
console.log(`\n💡 These items have similarity < ${argv.threshold} to their nearest neighbors`)
}
if (argv.output) {
await saveToFile(argv.output, outliers, argv.format!)
}
} catch (error) {
spinner.fail('💥 Failed to detect outliers')
throw error
}
}
async function handleVisualizeCommand(neural: any, argv: CommandArguments): Promise<void> {
const spinner = ora('📊 Generating visualization data...').start()
try {
const vizData = await neural.visualize({
dimensions: argv.dimensions as 2 | 3,
maxNodes: argv.limit
})
spinner.succeed('✅ Visualization data generated')
console.log(`\n📊 Visualization Data (${vizData.format} layout):`)
console.log(`📍 Nodes: ${vizData.nodes.length}`)
console.log(`🔗 Edges: ${vizData.edges.length}`)
console.log(`🎯 Clusters: ${vizData.clusters?.length || 0}`)
console.log(`📐 Dimensions: ${vizData.layout?.dimensions}D`)
if (argv.format === 'json') {
console.log('\nData:')
console.log(JSON.stringify(vizData, null, 2))
} else {
console.log('\n🎨 Style Settings:')
console.log(` Node Colors: ${vizData.style?.nodeColors}`)
console.log(` Edge Width: ${vizData.style?.edgeWidth}`)
console.log(` Labels: ${vizData.style?.labels}`)
}
if (argv.output) {
await saveToFile(argv.output, vizData, 'json')
console.log(`\n💾 Visualization data saved to: ${chalk.green(argv.output)}`)
} else {
console.log(`\n💡 Use --output to save visualization data for external tools`)
}
} catch (error) {
spinner.fail('💥 Failed to generate visualization')
throw error
}
}
async function saveToFile(filepath: string, data: any, format: string): Promise<void> {
const dir = path.dirname(filepath)
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir, { recursive: true })
}
let output: string
switch (format) {
case 'json':
output = JSON.stringify(data, null, 2)
break
case 'table':
output = formatAsTable(data)
break
default:
output = JSON.stringify(data, null, 2)
}
fs.writeFileSync(filepath, output, 'utf8')
console.log(`💾 Saved to: ${chalk.green(filepath)}`)
}
function formatAsTable(data: any): string {
// Simple table formatting - could be enhanced with a table library
if (Array.isArray(data)) {
return data.map((item, index) => `${index + 1}. ${JSON.stringify(item)}`).join('\n')
}
return JSON.stringify(data, null, 2)
}
/**
* @description Run a neural CLI handler with the one-shot lifecycle every
* Brainy command follows: init → work → `close()` → explicit `process.exit`.
* close() releases the writer lock and indexes, but global timers
* (UnifiedCache bookkeeping, PathResolver stats) keep the event loop alive,
* so one-shot commands must exit explicitly.
* @param work - The handler body, given the initialized neural API.
*/
async function runNeuralCommand(work: (neural: ReturnType<Brainy['neural']>) => Promise<void>): Promise<void> {
try {
const brain = getBrainy()
await brain.init()
await work(brain.neural())
await brain.close()
process.exit(0)
} catch (error) {
console.error(chalk.red('💥 Error:'), error instanceof Error ? error.message : error)
process.exit(1)
}
}
// Commander-compatible wrappers
export const neuralCommands = {
async similar(a?: string, b?: string, options?: any) {
// Build argv-style object for handler
const argv: CommandArguments = {
_: ['neural', 'similar', a || '', b || ''].filter(x => x),
id: a,
query: b,
explain: options?.explain,
...options
}
await runNeuralCommand((neural) => handleSimilarCommand(neural, argv))
},
async cluster(options?: any) {
const argv: CommandArguments = {
_: ['neural', 'cluster'],
algorithm: options?.algorithm || 'hierarchical',
threshold: options?.threshold ? parseFloat(options.threshold) : 0.7,
limit: options?.maxClusters ? parseInt(options.maxClusters) : 10,
query: options?.near,
...options
}
await runNeuralCommand((neural) => handleClustersCommand(neural, argv))
},
async hierarchy(id?: string, options?: any) {
const argv: CommandArguments = {
_: ['neural', 'hierarchy', id || ''].filter(x => x),
id,
...options
}
await runNeuralCommand((neural) => handleHierarchyCommand(neural, argv))
},
async related(id?: string, options?: any) {
const argv: CommandArguments = {
_: ['neural', 'related', id || ''].filter(x => x),
id,
limit: options?.limit ? parseInt(options.limit) : 10,
threshold: options?.radius ? parseFloat(options.radius) : 0.3,
...options
}
await runNeuralCommand((neural) => handleNeighborsCommand(neural, argv))
},
async outliers(options?: any) {
const argv: CommandArguments = {
_: ['neural', 'outliers'],
threshold: options?.threshold ? parseFloat(options.threshold) : 0.3,
explain: options?.explain,
...options
}
await runNeuralCommand((neural) => handleOutliersCommand(neural, argv))
},
async visualize(options?: any) {
const argv: CommandArguments = {
_: ['neural', 'visualize'],
format: options?.format || 'json',
dimensions: options?.dimensions ? parseInt(options.dimensions) : 2,
limit: options?.maxNodes ? parseInt(options.maxNodes) : 500,
output: options?.output,
...options
}
await runNeuralCommand((neural) => handleVisualizeCommand(neural, argv))
}
}

8
src/cli/commands/utility.ts
View file

@ -234,14 +234,6 @@ export const utilityCommands = {
case 'search':
await brain.find({ query: 'test', limit: 10 })
break
case 'similarity':
const neural = brain.neural()
await neural.similar('test1', 'test2')
break
case 'cluster':
const neuralApi = brain.neural()
await neuralApi.clusters()
break
}
times.push(Date.now() - start)

60
src/cli/index.ts
View file

@ -8,7 +8,6 @@
import { Command } from 'commander'
import chalk from 'chalk'
import { neuralCommands } from './commands/neural.js'
import { coreCommands } from './commands/core.js'
import { utilityCommands } from './commands/utility.js'
import { vfsCommands } from './commands/vfs.js'
@ -194,65 +193,6 @@ program
.option('--json', 'Output as JSON')
.action(validateCommand)
// ===== Neural Commands =====
program
.command('similar [a] [b]')
.alias('sim')
.description('Calculate similarity between two items (interactive if parameters missing)')
.option('--explain', 'Show detailed explanation')
.option('--breakdown', 'Show similarity breakdown')
.action(neuralCommands.similar)
program
.command('cluster')
.alias('clusters')
.description('Find semantic clusters in the data (interactive mode available)')
.option('--algorithm <type>', 'Clustering algorithm (hierarchical|kmeans|dbscan)', 'hierarchical')
.option('--threshold <number>', 'Similarity threshold', '0.7')
.option('--min-size <number>', 'Minimum cluster size', '2')
.option('--max-clusters <number>', 'Maximum number of clusters')
.option('--near <query>', 'Find clusters near a query')
.option('--show', 'Show visual representation')
.action(neuralCommands.cluster)
program
.command('related [id]')
.alias('neighbors')
.description('Find semantically related items (interactive if no ID)')
.option('-l, --limit <number>', 'Number of results', '10')
.option('-r, --radius <number>', 'Semantic radius', '0.3')
.option('--with-scores', 'Include similarity scores')
.option('--with-edges', 'Include connections')
.action(neuralCommands.related)
program
.command('hierarchy [id]')
.alias('tree')
.description('Show semantic hierarchy for an item (interactive if no ID)')
.option('-d, --depth <number>', 'Hierarchy depth', '3')
.option('--parents-only', 'Show only parent hierarchy')
.option('--children-only', 'Show only child hierarchy')
.action(neuralCommands.hierarchy)
program
.command('outliers')
.alias('anomalies')
.description('Detect semantic outliers')
.option('-t, --threshold <number>', 'Outlier threshold', '0.3')
.option('--explain', 'Explain why items are outliers')
.action(neuralCommands.outliers)
program
.command('visualize')
.alias('viz')
.description('Generate visualization data')
.option('-f, --format <format>', 'Output format (json|d3|graphml)', 'json')
.option('--max-nodes <number>', 'Maximum nodes', '500')
.option('--dimensions <number>', '2D or 3D', '2')
.option('-o, --output <file>', 'Output file')
.action(neuralCommands.visualize)
// ===== VFS Commands (Subcommand Group) =====
program