feat(8.0): API simplification — remove neural()/Db.search, one storage path key, integration→0
8.0 RC cleanup toward "one place per thing, zero-config, no deprecation":
- Remove the `brain.neural()` clustering namespace (ImprovedNeuralAPI + the dead
legacy NeuralAPI + the neural CLI + neural-only types). Similarity is `find({vector})`
/ `similar({to})`; attribute grouping is the aggregation `GROUP BY` engine. The separate
entity-extraction / smart-import feature (NeuralImport, NeuralEntityExtractor, SmartExtractor,
NaturalLanguageProcessor, `brain.extract()`/`brain.nlp()`) is kept.
- Remove `Db.search()`; `find()` is the one query verb (accepts a bare string or FindParams).
Fix the bundled MCP client, which called a non-existent `brain.search(query, limit)` →
now `find({ query, limit })`.
- Storage config: collapse to one canonical top-level `path` key. The pre-8.0 aliases
(`rootDirectory`, `options.*`, `fileSystemStorage.*`) are removed and now THROW with the
exact rename instead of silently defaulting to `./brainy-data` on upgrade. A single resolver
feeds createStorage, the 7.x→8.0 migration probe, and the plugin-factory handoff, so a native
storage provider resolves the identical root (no split-brain).
- Fix `similar({ threshold })`: the min-similarity filter was silently dropped; it is now
applied as a post-filter on `result.score` (the documented way to bound semantic results).
- Fix `vfs.rename()` on a directory: child path updates spread the entity vector into `update()`
and failed dimension validation; they are metadata-only updates now.
- Fix `vfs.move()`: copy+delete orphaned the content-addressed content blob (the destination
shared the source hash, then unlink removed it). `move()` now delegates to `rename()` — an
in-place path change that preserves the blob and the entity id, for files and directories.
- Fix streaming import: the bulk fast path never flushed mid-import nor signalled queryability.
Entity writes are now chunked by a progressive flush interval (100 → 1000 → 5000); each chunk
flushes and emits `progress.queryable`, so imported data is queryable during the import.
- Sweep all docs, comments, and JSDoc for the removed/changed APIs.
Integration suite: 49 files / 588 passed / 0 failed. Unit: 80 files / 1456 passed, no type errors.
This commit is contained in:
parent
0c4a51c24e
commit
606445cd61
74 changed files with 712 additions and 7470 deletions
|
|
@ -9,7 +9,7 @@ import { v4 as uuidv4 } from './universal/uuid.js'
|
|||
import { JsHnswVectorIndex } from './hnsw/hnswIndex.js'
|
||||
// TypeAwareHNSWIndex removed from default path — single unified JsHnswVectorIndex is faster
|
||||
// for the 99% of queries that don't filter by type (avoids searching 42 separate graphs)
|
||||
import { createStorage } from './storage/storageFactory.js'
|
||||
import { createStorage, resolveFilesystemRoot } from './storage/storageFactory.js'
|
||||
import type { StorageOptions } from './storage/storageFactory.js'
|
||||
import { rebuildCounts } from './utils/rebuildCounts.js'
|
||||
import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js'
|
||||
|
|
@ -23,7 +23,6 @@ import {
|
|||
} from './utils/index.js'
|
||||
import { embeddingManager } from './embeddings/EmbeddingManager.js'
|
||||
import { matchesMetadataFilter } from './utils/metadataFilter.js'
|
||||
import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js'
|
||||
import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
|
||||
import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js'
|
||||
import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js'
|
||||
|
|
@ -348,7 +347,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
private pluginRegistry = new PluginRegistry()
|
||||
|
||||
// Sub-APIs (lazy-loaded)
|
||||
private _neural?: ImprovedNeuralAPI
|
||||
private _nlp?: NaturalLanguageProcessor
|
||||
private _extractor?: NeuralEntityExtractor
|
||||
private _tripleIntelligence?: TripleIntelligenceSystem
|
||||
|
|
@ -482,7 +480,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* @example
|
||||
* ```typescript
|
||||
* const reader = await Brainy.openReadOnly({
|
||||
* storage: { type: 'filesystem', rootDirectory: '/data/brainy-data/tenant' }
|
||||
* storage: { type: 'filesystem', path: '/data/brainy-data/tenant' }
|
||||
* })
|
||||
* const bookings = await reader.find({ where: { entityType: 'booking' } })
|
||||
* await reader.close()
|
||||
|
|
@ -4287,7 +4285,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
// Use find with vector
|
||||
return this.find({
|
||||
const results = await this.find({
|
||||
vector: targetVector,
|
||||
limit: params.limit,
|
||||
type: params.type,
|
||||
|
|
@ -4295,6 +4293,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
service: params.service,
|
||||
excludeVFS: params.excludeVFS // Pass through VFS filtering
|
||||
})
|
||||
|
||||
// A min-similarity `threshold` is applied as a post-filter on result.score
|
||||
// — the canonical way to impose a minimum score on plain semantic results
|
||||
// (top-level vector search does not honor it; see the find({ near }) guidance).
|
||||
// Previously `params.threshold` was silently dropped.
|
||||
return params.threshold != null
|
||||
? results.filter((r) => r.score >= params.threshold!)
|
||||
: results
|
||||
}
|
||||
|
||||
// ============= BATCH OPERATIONS =============
|
||||
|
|
@ -4856,7 +4862,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
this.dimensions = undefined
|
||||
|
||||
// Clear any cached sub-APIs
|
||||
this._neural = undefined
|
||||
this._nlp = undefined
|
||||
this._tripleIntelligence = undefined
|
||||
|
||||
|
|
@ -5692,7 +5697,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* @param config - Standard `BrainyConfig`.
|
||||
* @returns The initialized instance.
|
||||
* @example
|
||||
* const brain = await Brainy.open({ storage: { type: 'filesystem', rootDirectory: './data' } })
|
||||
* const brain = await Brainy.open({ storage: { type: 'filesystem', path: './data' } })
|
||||
*/
|
||||
static async open<T = any>(config?: BrainyConfig): Promise<Brainy<T>> {
|
||||
const brain = new Brainy<T>(config)
|
||||
|
|
@ -5711,7 +5716,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* @returns A `Db` over the snapshot (release it to free resources).
|
||||
* @example
|
||||
* const db = await Brainy.load('/backups/2026-06-01')
|
||||
* const hits = await db.search('quarterly invoices')
|
||||
* const hits = await db.find({ query: 'quarterly invoices' })
|
||||
* await db.release()
|
||||
*/
|
||||
static async load<T = any>(path: string): Promise<Db<T>> {
|
||||
|
|
@ -5724,7 +5729,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
const brain = new Brainy<T>({
|
||||
storage: { type: 'filesystem', rootDirectory: path },
|
||||
storage: { type: 'filesystem', path },
|
||||
mode: 'reader'
|
||||
})
|
||||
await brain.init()
|
||||
|
|
@ -5957,7 +5962,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
/**
|
||||
* Build the at-`generation` index materialization — the brain-side half of
|
||||
* the historical full-query surface (`Db.find`/`Db.search`/`Db.related` at
|
||||
* the historical full-query surface (`Db.find`/`Db.related` at
|
||||
* past pinned generations; see `src/db/db.ts` and
|
||||
* `docs/ADR-001-generational-mvcc.md`):
|
||||
*
|
||||
|
|
@ -6812,16 +6817,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// ============= SUB-APIS =============
|
||||
|
||||
/**
|
||||
* Neural API - Advanced AI operations
|
||||
*/
|
||||
neural(): ImprovedNeuralAPI {
|
||||
if (!this._neural) {
|
||||
this._neural = new ImprovedNeuralAPI(this)
|
||||
}
|
||||
return this._neural
|
||||
}
|
||||
|
||||
/**
|
||||
* Natural Language Processing API
|
||||
*/
|
||||
|
|
@ -7403,7 +7398,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', rootDirectory: '/data/brain' } })
|
||||
* const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '/data/brain' } })
|
||||
* const fresh = await reader.requestFlush({ timeoutMs: 3000 })
|
||||
* if (!fresh) {
|
||||
* console.warn('Writer did not respond; results reflect last natural flush.')
|
||||
|
|
@ -10786,10 +10781,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// skip the probe-storage construction entirely on the init hot path. (The
|
||||
// migration is node-filesystem-only; `fs.existsSync` is absent/throwing
|
||||
// elsewhere, which the catch treats as "nothing to migrate".)
|
||||
const rootDir =
|
||||
(storageConfig.rootDirectory as string) ||
|
||||
(storageConfig.path as string) ||
|
||||
'./brainy-data' // createStorage's filesystem default
|
||||
//
|
||||
// Resolve the root through the SHARED resolver so the migration probe checks
|
||||
// the SAME directory createStorage (and any plugin factory) will open, and a
|
||||
// removed pre-8.0 alias (`rootDirectory`, `options.*`, `fileSystemStorage.*`)
|
||||
// throws here too — consistent with createStorage, never a silent skip.
|
||||
const rootDir = resolveFilesystemRoot(storageConfig)
|
||||
try {
|
||||
if (!fs.existsSync(`${rootDir}/branches`)) return
|
||||
} catch {
|
||||
|
|
@ -10935,10 +10932,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
Record<string, unknown>
|
||||
const storageType = storageConfig.type || 'auto'
|
||||
|
||||
// Check plugin-provided storage factories (e.g., 'filesystem' override from cortex)
|
||||
// Check plugin-provided storage factories (e.g., a native filesystem/mmap
|
||||
// override registered by an acceleration plugin).
|
||||
const pluginFactory = this.pluginRegistry.getStorageFactory(storageType)
|
||||
if (pluginFactory) {
|
||||
const adapter = await pluginFactory.create(storageConfig)
|
||||
// Hand the plugin factory a NORMALIZED config whose canonical `path` is
|
||||
// the SAME root our built-in resolver computed. The plugin's native side
|
||||
// (e.g. getBinaryBlobPath / mmap files) re-resolves the directory itself;
|
||||
// without this, an alias-only config (`{ rootDirectory }`, `{ options:
|
||||
// { path } }`, …) would resolve to the alias here but to ./brainy-data on
|
||||
// the plugin side — a brainy-data/cor split-brain where the two halves
|
||||
// write to different directories. Stamping the resolved `path` keeps both
|
||||
// resolvers on the identical root.
|
||||
const normalizedConfig = {
|
||||
...storageConfig,
|
||||
path: resolveFilesystemRoot(storageConfig)
|
||||
}
|
||||
const adapter = await pluginFactory.create(normalizedConfig)
|
||||
return adapter as BaseStorage
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ async function openReader(rootDir: string, options: OpenOptions): Promise<Brainy
|
|||
// ack lands before we open the main reader and have it rebuild indexes.
|
||||
try {
|
||||
const probe = await Brainy.openReadOnly({
|
||||
storage: { type: 'filesystem', rootDirectory: rootDir }
|
||||
storage: { type: 'filesystem', path: rootDir }
|
||||
})
|
||||
const acked = await probe.requestFlush({ timeoutMs: flushTimeoutMs })
|
||||
await probe.close()
|
||||
|
|
@ -61,7 +61,7 @@ async function openReader(rootDir: string, options: OpenOptions): Promise<Brainy
|
|||
}
|
||||
|
||||
return Brainy.openReadOnly({
|
||||
storage: { type: 'filesystem', rootDirectory: rootDir }
|
||||
storage: { type: 'filesystem', path: rootDir }
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -379,7 +379,7 @@ export const inspectCommands = {
|
|||
const spinner = options.quiet ? null : ora('Opening writer…').start()
|
||||
try {
|
||||
const brain = new Brainy({
|
||||
storage: { type: 'filesystem', rootDirectory: path },
|
||||
storage: { type: 'filesystem', path: path },
|
||||
mode: 'writer',
|
||||
force: options.force
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,518 +0,0 @@
|
|||
/**
|
||||
* @module cli/commands/neural
|
||||
* @description Neural CLI commands: semantic similarity, clustering,
|
||||
* hierarchy, neighbors, outlier detection, and visualization data export.
|
||||
* Registered in `src/cli/index.ts` as `similar` / `cluster` / `related` /
|
||||
* `hierarchy` / `outliers` / `visualize`. Each command is one-shot: it
|
||||
* initializes the shared Brainy instance, runs the neural operation, then
|
||||
* closes the store and exits explicitly.
|
||||
*/
|
||||
|
||||
import inquirer from 'inquirer'
|
||||
import chalk from 'chalk'
|
||||
import ora, { type Ora } from 'ora'
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { Brainy } from '../../brainy.js'
|
||||
import type { NeuralClusterParams } from '../../types/brainy.types.js'
|
||||
|
||||
interface CommandArguments {
|
||||
action?: string;
|
||||
id?: string;
|
||||
query?: string;
|
||||
threshold?: number;
|
||||
format?: string;
|
||||
output?: string;
|
||||
limit?: number;
|
||||
algorithm?: string;
|
||||
dimensions?: number;
|
||||
explain?: boolean;
|
||||
_: string[];
|
||||
}
|
||||
|
||||
let brainyInstance: Brainy | null = null
|
||||
|
||||
const getBrainy = (): Brainy => {
|
||||
if (!brainyInstance) {
|
||||
brainyInstance = new Brainy()
|
||||
}
|
||||
return brainyInstance
|
||||
}
|
||||
|
||||
async function handleSimilarCommand(neural: any, argv: CommandArguments): Promise<void> {
|
||||
const spinner = ora('🧠 Calculating semantic similarity...').start()
|
||||
|
||||
try {
|
||||
let itemA: string, itemB: string
|
||||
|
||||
if (argv.id && argv.query) {
|
||||
itemA = argv.id
|
||||
itemB = argv.query
|
||||
} else if (argv._ && argv._.length >= 3) {
|
||||
itemA = argv._[1]
|
||||
itemB = argv._[2]
|
||||
} else {
|
||||
spinner.stop()
|
||||
const answers = await inquirer.prompt([
|
||||
{
|
||||
type: 'input',
|
||||
name: 'itemA',
|
||||
message: 'First item (ID or text):',
|
||||
validate: (input: string) => input.length > 0
|
||||
},
|
||||
{
|
||||
type: 'input',
|
||||
name: 'itemB',
|
||||
message: 'Second item (ID or text):',
|
||||
validate: (input: string) => input.length > 0
|
||||
}
|
||||
])
|
||||
itemA = answers.itemA
|
||||
itemB = answers.itemB
|
||||
spinner.start()
|
||||
}
|
||||
|
||||
const result = await neural.similar(itemA, itemB, {
|
||||
explain: argv.explain,
|
||||
includeBreakdown: argv.explain
|
||||
})
|
||||
|
||||
spinner.succeed('✅ Similarity calculated')
|
||||
|
||||
if (typeof result === 'number') {
|
||||
console.log(`\n🔗 Similarity: ${chalk.cyan((result * 100).toFixed(1))}%`)
|
||||
} else {
|
||||
console.log(`\n🔗 Similarity: ${chalk.cyan((result.score * 100).toFixed(1))}%`)
|
||||
if (result.explanation) {
|
||||
console.log(`💭 Explanation: ${result.explanation}`)
|
||||
}
|
||||
if (result.breakdown) {
|
||||
console.log('\n📊 Breakdown:')
|
||||
console.log(` Semantic: ${chalk.yellow((result.breakdown.semantic! * 100).toFixed(1))}%`)
|
||||
if (result.breakdown.taxonomic !== undefined) {
|
||||
console.log(` Taxonomic: ${chalk.yellow((result.breakdown.taxonomic * 100).toFixed(1))}%`)
|
||||
}
|
||||
if (result.breakdown.contextual !== undefined) {
|
||||
console.log(` Contextual: ${chalk.yellow((result.breakdown.contextual * 100).toFixed(1))}%`)
|
||||
}
|
||||
}
|
||||
if (result.hierarchy) {
|
||||
console.log(`\n🌳 Hierarchy: ${result.hierarchy.sharedParent ?
|
||||
`Shared parent at distance ${result.hierarchy.distance}` :
|
||||
'No shared parent found'}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (argv.output) {
|
||||
await saveToFile(argv.output, result, argv.format!)
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
spinner.fail('💥 Failed to calculate similarity')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function handleClustersCommand(neural: any, argv: CommandArguments): Promise<void> {
|
||||
let spinner: Ora | null = null
|
||||
try {
|
||||
let options: any = {
|
||||
// --algorithm arrives as a raw CLI string; the interactive prompt below
|
||||
// and neural.clusters() constrain it to the supported algorithms.
|
||||
algorithm: argv.algorithm as NeuralClusterParams['algorithm'],
|
||||
threshold: argv.threshold,
|
||||
maxClusters: argv.limit
|
||||
}
|
||||
|
||||
// Interactive mode if no algorithm specified or using defaults
|
||||
if (!argv.algorithm || argv.algorithm === 'hierarchical') {
|
||||
const answers = await inquirer.prompt([
|
||||
{
|
||||
type: 'list',
|
||||
name: 'algorithm',
|
||||
message: 'Choose clustering algorithm:',
|
||||
default: argv.algorithm || 'hierarchical',
|
||||
choices: [
|
||||
{ name: '🌳 Hierarchical (Tree-based, best for natural grouping)', value: 'hierarchical' },
|
||||
{ name: '📊 K-Means (Fixed number of clusters)', value: 'kmeans' },
|
||||
{ name: '🎯 DBSCAN (Density-based, finds arbitrary shapes)', value: 'dbscan' }
|
||||
]
|
||||
},
|
||||
{
|
||||
type: 'number',
|
||||
name: 'maxClusters',
|
||||
message: 'Maximum number of clusters:',
|
||||
default: argv.limit || 5,
|
||||
when: (answers: any) => answers.algorithm === 'kmeans'
|
||||
},
|
||||
{
|
||||
type: 'number',
|
||||
name: 'threshold',
|
||||
message: 'Similarity threshold (0-1):',
|
||||
default: argv.threshold || 0.7,
|
||||
validate: (input: number) => (input >= 0 && input <= 1) || 'Must be between 0 and 1'
|
||||
}
|
||||
])
|
||||
|
||||
options = {
|
||||
algorithm: answers.algorithm,
|
||||
threshold: answers.threshold,
|
||||
maxClusters: answers.maxClusters || options.maxClusters
|
||||
}
|
||||
}
|
||||
|
||||
const spinner = ora('🎯 Finding semantic clusters...').start()
|
||||
const clusters = await neural.clusters(argv.query || options)
|
||||
|
||||
spinner.succeed(`✅ Found ${clusters.length} clusters`)
|
||||
|
||||
if (argv.format === 'json') {
|
||||
console.log(JSON.stringify(clusters, null, 2))
|
||||
} else {
|
||||
console.log(`\n🎯 ${chalk.cyan(clusters.length)} Semantic Clusters:\n`)
|
||||
|
||||
clusters.forEach((cluster, index) => {
|
||||
console.log(`${chalk.yellow(`Cluster ${index + 1}:`)} ${cluster.label || cluster.id}`)
|
||||
console.log(` 📊 Confidence: ${chalk.green((cluster.confidence * 100).toFixed(1))}%`)
|
||||
console.log(` 👥 Members: ${cluster.members.length}`)
|
||||
if (cluster.members.length <= 5) {
|
||||
cluster.members.forEach(member => {
|
||||
console.log(` • ${member}`)
|
||||
})
|
||||
} else {
|
||||
cluster.members.slice(0, 3).forEach(member => {
|
||||
console.log(` • ${member}`)
|
||||
})
|
||||
console.log(` ... and ${cluster.members.length - 3} more`)
|
||||
}
|
||||
console.log()
|
||||
})
|
||||
}
|
||||
|
||||
if (argv.output) {
|
||||
await saveToFile(argv.output, clusters, argv.format!)
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
if (spinner) spinner.fail('💥 Failed to find clusters')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function handleHierarchyCommand(neural: any, argv: CommandArguments): Promise<void> {
|
||||
const spinner = ora('🌳 Building semantic hierarchy...').start()
|
||||
|
||||
try {
|
||||
const id = argv.id || argv._[1]
|
||||
if (!id) {
|
||||
spinner.stop()
|
||||
const answer = await inquirer.prompt([{
|
||||
type: 'input',
|
||||
name: 'id',
|
||||
message: 'Enter item ID:',
|
||||
validate: (input: string) => input.length > 0
|
||||
}])
|
||||
spinner.start()
|
||||
const hierarchy = await neural.hierarchy(answer.id)
|
||||
displayHierarchy(hierarchy)
|
||||
} else {
|
||||
const hierarchy = await neural.hierarchy(id)
|
||||
spinner.succeed('✅ Hierarchy built')
|
||||
displayHierarchy(hierarchy)
|
||||
}
|
||||
|
||||
if (argv.output) {
|
||||
const hierarchy = await neural.hierarchy(id || argv._[1])
|
||||
await saveToFile(argv.output, hierarchy, argv.format!)
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
spinner.fail('💥 Failed to build hierarchy')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function displayHierarchy(hierarchy: any): void {
|
||||
console.log(`\n🌳 Semantic Hierarchy for ${chalk.cyan(hierarchy.self.id)}:`)
|
||||
|
||||
if (hierarchy.root) {
|
||||
console.log(`🔝 Root: ${hierarchy.root.id} (${(hierarchy.root.similarity * 100).toFixed(1)}%)`)
|
||||
}
|
||||
|
||||
if (hierarchy.grandparent) {
|
||||
console.log(`👴 Grandparent: ${hierarchy.grandparent.id} (${(hierarchy.grandparent.similarity * 100).toFixed(1)}%)`)
|
||||
}
|
||||
|
||||
if (hierarchy.parent) {
|
||||
console.log(`👨 Parent: ${hierarchy.parent.id} (${(hierarchy.parent.similarity * 100).toFixed(1)}%)`)
|
||||
}
|
||||
|
||||
console.log(`🎯 ${chalk.bold('Self:')} ${hierarchy.self.id}`)
|
||||
|
||||
if (hierarchy.siblings && hierarchy.siblings.length > 0) {
|
||||
console.log(`👥 Siblings: ${hierarchy.siblings.length}`)
|
||||
hierarchy.siblings.forEach((sibling: any) => {
|
||||
console.log(` • ${sibling.id} (${(sibling.similarity * 100).toFixed(1)}%)`)
|
||||
})
|
||||
}
|
||||
|
||||
if (hierarchy.children && hierarchy.children.length > 0) {
|
||||
console.log(`👶 Children: ${hierarchy.children.length}`)
|
||||
hierarchy.children.forEach((child: any) => {
|
||||
console.log(` • ${child.id} (${(child.similarity * 100).toFixed(1)}%)`)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function handleNeighborsCommand(neural: any, argv: CommandArguments): Promise<void> {
|
||||
const spinner = ora('🕸️ Finding semantic neighbors...').start()
|
||||
|
||||
try {
|
||||
const id = argv.id || argv._[1]
|
||||
if (!id) {
|
||||
spinner.stop()
|
||||
const answer = await inquirer.prompt([{
|
||||
type: 'input',
|
||||
name: 'id',
|
||||
message: 'Enter item ID:',
|
||||
validate: (input: string) => input.length > 0
|
||||
}])
|
||||
spinner.start()
|
||||
}
|
||||
|
||||
const targetId = id || (await inquirer.prompt([{
|
||||
type: 'input',
|
||||
name: 'id',
|
||||
message: 'Enter item ID:',
|
||||
validate: (input: string) => input.length > 0
|
||||
}])).id
|
||||
|
||||
const graph = await neural.neighbors(targetId, {
|
||||
limit: argv.limit,
|
||||
includeEdges: true
|
||||
})
|
||||
|
||||
spinner.succeed(`✅ Found ${graph.neighbors.length} neighbors`)
|
||||
|
||||
console.log(`\n🕸️ Neighbors of ${chalk.cyan(graph.center)}:`)
|
||||
|
||||
graph.neighbors.forEach((neighbor, index) => {
|
||||
console.log(`${index + 1}. ${neighbor.id} (${(neighbor.similarity * 100).toFixed(1)}%)`)
|
||||
if (neighbor.type) {
|
||||
console.log(` Type: ${neighbor.type}`)
|
||||
}
|
||||
if (neighbor.connections) {
|
||||
console.log(` Connections: ${neighbor.connections}`)
|
||||
}
|
||||
})
|
||||
|
||||
if (graph.edges && graph.edges.length > 0) {
|
||||
console.log(`\n🔗 ${graph.edges.length} semantic connections found`)
|
||||
}
|
||||
|
||||
if (argv.output) {
|
||||
await saveToFile(argv.output, graph, argv.format!)
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
spinner.fail('💥 Failed to find neighbors')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function handleOutliersCommand(neural: any, argv: CommandArguments): Promise<void> {
|
||||
const spinner = ora('🚨 Detecting semantic outliers...').start()
|
||||
|
||||
try {
|
||||
const outliers = await neural.outliers(argv.threshold)
|
||||
|
||||
spinner.succeed(`✅ Found ${outliers.length} outliers`)
|
||||
|
||||
if (outliers.length === 0) {
|
||||
console.log('\n🎉 No outliers detected - all items are well connected!')
|
||||
} else {
|
||||
console.log(`\n🚨 ${chalk.red(outliers.length)} Semantic Outliers:`)
|
||||
outliers.forEach((outlier, index) => {
|
||||
console.log(`${index + 1}. ${outlier}`)
|
||||
})
|
||||
|
||||
console.log(`\n💡 These items have similarity < ${argv.threshold} to their nearest neighbors`)
|
||||
}
|
||||
|
||||
if (argv.output) {
|
||||
await saveToFile(argv.output, outliers, argv.format!)
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
spinner.fail('💥 Failed to detect outliers')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function handleVisualizeCommand(neural: any, argv: CommandArguments): Promise<void> {
|
||||
const spinner = ora('📊 Generating visualization data...').start()
|
||||
|
||||
try {
|
||||
const vizData = await neural.visualize({
|
||||
dimensions: argv.dimensions as 2 | 3,
|
||||
maxNodes: argv.limit
|
||||
})
|
||||
|
||||
spinner.succeed('✅ Visualization data generated')
|
||||
|
||||
console.log(`\n📊 Visualization Data (${vizData.format} layout):`)
|
||||
console.log(`📍 Nodes: ${vizData.nodes.length}`)
|
||||
console.log(`🔗 Edges: ${vizData.edges.length}`)
|
||||
console.log(`🎯 Clusters: ${vizData.clusters?.length || 0}`)
|
||||
console.log(`📐 Dimensions: ${vizData.layout?.dimensions}D`)
|
||||
|
||||
if (argv.format === 'json') {
|
||||
console.log('\nData:')
|
||||
console.log(JSON.stringify(vizData, null, 2))
|
||||
} else {
|
||||
console.log('\n🎨 Style Settings:')
|
||||
console.log(` Node Colors: ${vizData.style?.nodeColors}`)
|
||||
console.log(` Edge Width: ${vizData.style?.edgeWidth}`)
|
||||
console.log(` Labels: ${vizData.style?.labels}`)
|
||||
}
|
||||
|
||||
if (argv.output) {
|
||||
await saveToFile(argv.output, vizData, 'json')
|
||||
console.log(`\n💾 Visualization data saved to: ${chalk.green(argv.output)}`)
|
||||
} else {
|
||||
console.log(`\n💡 Use --output to save visualization data for external tools`)
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
spinner.fail('💥 Failed to generate visualization')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function saveToFile(filepath: string, data: any, format: string): Promise<void> {
|
||||
const dir = path.dirname(filepath)
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true })
|
||||
}
|
||||
|
||||
let output: string
|
||||
switch (format) {
|
||||
case 'json':
|
||||
output = JSON.stringify(data, null, 2)
|
||||
break
|
||||
case 'table':
|
||||
output = formatAsTable(data)
|
||||
break
|
||||
default:
|
||||
output = JSON.stringify(data, null, 2)
|
||||
}
|
||||
|
||||
fs.writeFileSync(filepath, output, 'utf8')
|
||||
console.log(`💾 Saved to: ${chalk.green(filepath)}`)
|
||||
}
|
||||
|
||||
function formatAsTable(data: any): string {
|
||||
// Simple table formatting - could be enhanced with a table library
|
||||
if (Array.isArray(data)) {
|
||||
return data.map((item, index) => `${index + 1}. ${JSON.stringify(item)}`).join('\n')
|
||||
}
|
||||
return JSON.stringify(data, null, 2)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Run a neural CLI handler with the one-shot lifecycle every
|
||||
* Brainy command follows: init → work → `close()` → explicit `process.exit`.
|
||||
* close() releases the writer lock and indexes, but global timers
|
||||
* (UnifiedCache bookkeeping, PathResolver stats) keep the event loop alive,
|
||||
* so one-shot commands must exit explicitly.
|
||||
* @param work - The handler body, given the initialized neural API.
|
||||
*/
|
||||
async function runNeuralCommand(work: (neural: ReturnType<Brainy['neural']>) => Promise<void>): Promise<void> {
|
||||
try {
|
||||
const brain = getBrainy()
|
||||
await brain.init()
|
||||
|
||||
await work(brain.neural())
|
||||
|
||||
await brain.close()
|
||||
process.exit(0)
|
||||
} catch (error) {
|
||||
console.error(chalk.red('💥 Error:'), error instanceof Error ? error.message : error)
|
||||
process.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Commander-compatible wrappers
|
||||
export const neuralCommands = {
|
||||
async similar(a?: string, b?: string, options?: any) {
|
||||
// Build argv-style object for handler
|
||||
const argv: CommandArguments = {
|
||||
_: ['neural', 'similar', a || '', b || ''].filter(x => x),
|
||||
id: a,
|
||||
query: b,
|
||||
explain: options?.explain,
|
||||
...options
|
||||
}
|
||||
|
||||
await runNeuralCommand((neural) => handleSimilarCommand(neural, argv))
|
||||
},
|
||||
|
||||
async cluster(options?: any) {
|
||||
const argv: CommandArguments = {
|
||||
_: ['neural', 'cluster'],
|
||||
algorithm: options?.algorithm || 'hierarchical',
|
||||
threshold: options?.threshold ? parseFloat(options.threshold) : 0.7,
|
||||
limit: options?.maxClusters ? parseInt(options.maxClusters) : 10,
|
||||
query: options?.near,
|
||||
...options
|
||||
}
|
||||
|
||||
await runNeuralCommand((neural) => handleClustersCommand(neural, argv))
|
||||
},
|
||||
|
||||
async hierarchy(id?: string, options?: any) {
|
||||
const argv: CommandArguments = {
|
||||
_: ['neural', 'hierarchy', id || ''].filter(x => x),
|
||||
id,
|
||||
...options
|
||||
}
|
||||
|
||||
await runNeuralCommand((neural) => handleHierarchyCommand(neural, argv))
|
||||
},
|
||||
|
||||
async related(id?: string, options?: any) {
|
||||
const argv: CommandArguments = {
|
||||
_: ['neural', 'related', id || ''].filter(x => x),
|
||||
id,
|
||||
limit: options?.limit ? parseInt(options.limit) : 10,
|
||||
threshold: options?.radius ? parseFloat(options.radius) : 0.3,
|
||||
...options
|
||||
}
|
||||
|
||||
await runNeuralCommand((neural) => handleNeighborsCommand(neural, argv))
|
||||
},
|
||||
|
||||
async outliers(options?: any) {
|
||||
const argv: CommandArguments = {
|
||||
_: ['neural', 'outliers'],
|
||||
threshold: options?.threshold ? parseFloat(options.threshold) : 0.3,
|
||||
explain: options?.explain,
|
||||
...options
|
||||
}
|
||||
|
||||
await runNeuralCommand((neural) => handleOutliersCommand(neural, argv))
|
||||
},
|
||||
|
||||
async visualize(options?: any) {
|
||||
const argv: CommandArguments = {
|
||||
_: ['neural', 'visualize'],
|
||||
format: options?.format || 'json',
|
||||
dimensions: options?.dimensions ? parseInt(options.dimensions) : 2,
|
||||
limit: options?.maxNodes ? parseInt(options.maxNodes) : 500,
|
||||
output: options?.output,
|
||||
...options
|
||||
}
|
||||
|
||||
await runNeuralCommand((neural) => handleVisualizeCommand(neural, argv))
|
||||
}
|
||||
}
|
||||
|
|
@ -234,14 +234,6 @@ export const utilityCommands = {
|
|||
case 'search':
|
||||
await brain.find({ query: 'test', limit: 10 })
|
||||
break
|
||||
case 'similarity':
|
||||
const neural = brain.neural()
|
||||
await neural.similar('test1', 'test2')
|
||||
break
|
||||
case 'cluster':
|
||||
const neuralApi = brain.neural()
|
||||
await neuralApi.clusters()
|
||||
break
|
||||
}
|
||||
|
||||
times.push(Date.now() - start)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
|
||||
import { Command } from 'commander'
|
||||
import chalk from 'chalk'
|
||||
import { neuralCommands } from './commands/neural.js'
|
||||
import { coreCommands } from './commands/core.js'
|
||||
import { utilityCommands } from './commands/utility.js'
|
||||
import { vfsCommands } from './commands/vfs.js'
|
||||
|
|
@ -194,65 +193,6 @@ program
|
|||
.option('--json', 'Output as JSON')
|
||||
.action(validateCommand)
|
||||
|
||||
// ===== Neural Commands =====
|
||||
|
||||
program
|
||||
.command('similar [a] [b]')
|
||||
.alias('sim')
|
||||
.description('Calculate similarity between two items (interactive if parameters missing)')
|
||||
.option('--explain', 'Show detailed explanation')
|
||||
.option('--breakdown', 'Show similarity breakdown')
|
||||
.action(neuralCommands.similar)
|
||||
|
||||
program
|
||||
.command('cluster')
|
||||
.alias('clusters')
|
||||
.description('Find semantic clusters in the data (interactive mode available)')
|
||||
.option('--algorithm <type>', 'Clustering algorithm (hierarchical|kmeans|dbscan)', 'hierarchical')
|
||||
.option('--threshold <number>', 'Similarity threshold', '0.7')
|
||||
.option('--min-size <number>', 'Minimum cluster size', '2')
|
||||
.option('--max-clusters <number>', 'Maximum number of clusters')
|
||||
.option('--near <query>', 'Find clusters near a query')
|
||||
.option('--show', 'Show visual representation')
|
||||
.action(neuralCommands.cluster)
|
||||
|
||||
program
|
||||
.command('related [id]')
|
||||
.alias('neighbors')
|
||||
.description('Find semantically related items (interactive if no ID)')
|
||||
.option('-l, --limit <number>', 'Number of results', '10')
|
||||
.option('-r, --radius <number>', 'Semantic radius', '0.3')
|
||||
.option('--with-scores', 'Include similarity scores')
|
||||
.option('--with-edges', 'Include connections')
|
||||
.action(neuralCommands.related)
|
||||
|
||||
program
|
||||
.command('hierarchy [id]')
|
||||
.alias('tree')
|
||||
.description('Show semantic hierarchy for an item (interactive if no ID)')
|
||||
.option('-d, --depth <number>', 'Hierarchy depth', '3')
|
||||
.option('--parents-only', 'Show only parent hierarchy')
|
||||
.option('--children-only', 'Show only child hierarchy')
|
||||
.action(neuralCommands.hierarchy)
|
||||
|
||||
program
|
||||
.command('outliers')
|
||||
.alias('anomalies')
|
||||
.description('Detect semantic outliers')
|
||||
.option('-t, --threshold <number>', 'Outlier threshold', '0.3')
|
||||
.option('--explain', 'Explain why items are outliers')
|
||||
.action(neuralCommands.outliers)
|
||||
|
||||
program
|
||||
.command('visualize')
|
||||
.alias('viz')
|
||||
.description('Generate visualization data')
|
||||
.option('-f, --format <format>', 'Output format (json|d3|graphml)', 'json')
|
||||
.option('--max-nodes <number>', 'Maximum nodes', '500')
|
||||
.option('--dimensions <number>', '2D or 3D', '2')
|
||||
.option('-o, --output <file>', 'Output file')
|
||||
.action(neuralCommands.visualize)
|
||||
|
||||
// ===== VFS Commands (Subcommand Group) =====
|
||||
|
||||
program
|
||||
|
|
|
|||
27
src/db/db.ts
27
src/db/db.ts
|
|
@ -407,33 +407,6 @@ export class Db<T = any> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Semantic (vector) search **as of this view's generation** —
|
||||
* convenience for `find({ query, ...options })`. At the current generation
|
||||
* the live vector index serves it directly; at a historical generation the
|
||||
* at-generation index materialization serves it (built lazily on first
|
||||
* use — O(n at G) once per `Db`, freed on release; see the module doc).
|
||||
* Speculative overlays throw {@link SpeculativeOverlayError}: overlay
|
||||
* entities carry no embeddings, so the result would silently exclude them.
|
||||
*
|
||||
* @param query - Natural-language search query.
|
||||
* @param options - Additional `FindParams` (limit, filters, …).
|
||||
* @returns Semantic search results as of this generation.
|
||||
* @throws SpeculativeOverlayError on speculative `with()` overlays.
|
||||
*/
|
||||
async search(query: string, options?: Omit<FindParams<T>, 'query'>): Promise<Result<T>[]> {
|
||||
this.assertUsable('search')
|
||||
|
||||
if (this.overlay) {
|
||||
throw new SpeculativeOverlayError('vector search', this.gen)
|
||||
}
|
||||
if (!this.isHistorical()) {
|
||||
return this.host.find({ ...(options ?? {}), query })
|
||||
}
|
||||
const materialized = await this.materialize()
|
||||
return materialized.find({ ...(options ?? {}), query })
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Serialize part or all of this database value into a portable
|
||||
* `PortableGraph` document, read **as of this view's generation**. Because `export`
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ export class GenerationConflictError extends Error {
|
|||
* @example
|
||||
* const spec = await brain.now().with([{ op: 'add', type, data: 'draft' }])
|
||||
* await spec.find({ where: { draft: true } }) // ✅ metadata find — supported
|
||||
* await spec.search('semantic query') // ❌ throws this error
|
||||
* await spec.find({ query: 'semantic query' }) // ❌ throws this error
|
||||
* await brain.transact([{ op: 'add', type, data: 'draft' }]) // → full surface
|
||||
*/
|
||||
export class SpeculativeOverlayError extends Error {
|
||||
|
|
|
|||
|
|
@ -920,7 +920,7 @@ export class ImportCoordinator {
|
|||
// Starts at 100, increases to 1000 at 1K entities, then 5000 at 10K
|
||||
// This works for both known totals (files) and unknown totals (streaming APIs)
|
||||
let currentFlushInterval = 100 // Start with frequent updates for better UX
|
||||
let entitiesSinceFlush = 0
|
||||
let entitiesSinceFlush = 0 // used by the dedup slow path below
|
||||
let totalFlushes = 0
|
||||
|
||||
console.log(
|
||||
|
|
@ -1026,42 +1026,60 @@ export class ImportCoordinator {
|
|||
}
|
||||
})
|
||||
|
||||
// Batch create all entities (storage-aware batching handles rate limits automatically)
|
||||
const addResult = await this.brain.addMany({
|
||||
items: entityParams,
|
||||
continueOnError: true,
|
||||
onProgress: (done, total) => {
|
||||
options.onProgress?.({
|
||||
stage: 'storing-graph',
|
||||
message: `Creating entities: ${done}/${total}`,
|
||||
processed: done,
|
||||
total,
|
||||
entities: done
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
// Map results to entities array and update rows with new IDs
|
||||
for (let i = 0; i < addResult.successful.length; i++) {
|
||||
const entityId = addResult.successful[i]
|
||||
const row = rows[i]
|
||||
const entity = row.entity || row
|
||||
const vfsFile = vfsResult.files.find((f: any) => f.entityId === entity.id)
|
||||
|
||||
entity.id = entityId
|
||||
entities.push({
|
||||
id: entityId,
|
||||
name: entity.name,
|
||||
type: entity.type,
|
||||
vfsPath: vfsFile?.path,
|
||||
metadata: entity.metadata // Include metadata in return (for ImageHandler, etc)
|
||||
// Chunked batch creation with PROGRESSIVE FLUSH so imported data becomes
|
||||
// queryable DURING the import (the always-on streaming contract): after
|
||||
// each chunk we flush the indexes and emit a `queryable: true` progress
|
||||
// event. The interval widens with volume (100 → 1000 → 5000) to keep large
|
||||
// imports fast while small ones stay responsive. (storage-aware batching
|
||||
// inside addMany still handles rate limits within each chunk.)
|
||||
let failedCount = 0
|
||||
for (let offset = 0; offset < entityParams.length; offset += currentFlushInterval) {
|
||||
const chunk = entityParams.slice(offset, offset + currentFlushInterval)
|
||||
const addResult = await this.brain.addMany({
|
||||
items: chunk,
|
||||
continueOnError: true
|
||||
})
|
||||
newCount++
|
||||
|
||||
// Map this chunk's results back to their source rows.
|
||||
for (let j = 0; j < addResult.successful.length; j++) {
|
||||
const entityId = addResult.successful[j]
|
||||
const row = rows[offset + j]
|
||||
const entity = row.entity || row
|
||||
const vfsFile = vfsResult.files.find((f: any) => f.entityId === entity.id)
|
||||
|
||||
entity.id = entityId
|
||||
entities.push({
|
||||
id: entityId,
|
||||
name: entity.name,
|
||||
type: entity.type,
|
||||
vfsPath: vfsFile?.path,
|
||||
metadata: entity.metadata // Include metadata in return (for ImageHandler, etc)
|
||||
})
|
||||
newCount++
|
||||
}
|
||||
failedCount += addResult.failed.length
|
||||
|
||||
// Flush so the just-added chunk is immediately queryable, then signal it.
|
||||
await this.brain.flush()
|
||||
totalFlushes++
|
||||
const processed = Math.min(offset + currentFlushInterval, entityParams.length)
|
||||
options.onProgress?.({
|
||||
stage: 'storing-graph',
|
||||
message: `Creating entities: ${processed}/${entityParams.length}`,
|
||||
processed,
|
||||
total: entityParams.length,
|
||||
entities: entities.length,
|
||||
queryable: true
|
||||
})
|
||||
|
||||
// Progressive interval: widen as the import grows.
|
||||
if (entities.length >= 10000) currentFlushInterval = 5000
|
||||
else if (entities.length >= 1000) currentFlushInterval = 1000
|
||||
}
|
||||
|
||||
// Handle failed entities
|
||||
if (addResult.failed.length > 0) {
|
||||
console.warn(`⚠️ ${addResult.failed.length} entities failed to create`)
|
||||
if (failedCount > 0) {
|
||||
console.warn(`⚠️ ${failedCount} entities failed to create`)
|
||||
}
|
||||
|
||||
// Create provenance links in batch
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
* - Triple Intelligence: Seamless fusion of vector + graph + field search
|
||||
* - Db: Immutable, generation-pinned database values (now/transact/asOf/with)
|
||||
* - Plugins: Extensible plugin system (cortex, storage adapters)
|
||||
* - Neural API: AI-powered clustering and analysis
|
||||
* - Neural Import: AI-powered entity extraction & smart data import
|
||||
*/
|
||||
|
||||
// Export main Brainy class
|
||||
|
|
|
|||
|
|
@ -243,7 +243,7 @@ export class BrainyMCPClient {
|
|||
return []
|
||||
}
|
||||
|
||||
const results = await this.brainy.search(query, limit)
|
||||
const results = await this.brainy.find({ query, limit })
|
||||
return results.map(r => ({
|
||||
...r.metadata,
|
||||
relevance: r.score
|
||||
|
|
@ -260,7 +260,7 @@ export class BrainyMCPClient {
|
|||
}
|
||||
|
||||
// Search for recent activity
|
||||
const results = await this.brainy.search('recent messages communication', limit)
|
||||
const results = await this.brainy.find({ query: 'recent messages communication', limit })
|
||||
return results
|
||||
.map(r => r.metadata)
|
||||
.sort((a: any, b: any) => b.timestamp - a.timestamp)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1010,10 +1010,11 @@ export class NaturalLanguageProcessor {
|
|||
// This would integrate with Brainy's search to find similar query patterns
|
||||
|
||||
// Future implementation could search a query_history noun type:
|
||||
// const similarQueries = await this.brainy.search(queryEmbedding, {
|
||||
// const similarQueries = await this.brainy.find({
|
||||
// vector: queryEmbedding,
|
||||
// limit: 5,
|
||||
// metadata: { type: 'successful_query' },
|
||||
// nounTypes: ['query_history']
|
||||
// where: { type: 'successful_query' },
|
||||
// type: ['query_history']
|
||||
// })
|
||||
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -1,887 +0,0 @@
|
|||
/**
|
||||
* Neural API - Unified Semantic Intelligence
|
||||
*
|
||||
* Best-of-both: Complete functionality + Enterprise performance
|
||||
* Combines rich features with O(n) algorithms for millions of items
|
||||
*/
|
||||
|
||||
import { Vector, HNSWNoun } from '../coreTypes.js'
|
||||
import { cosineDistance } from '../utils/distance.js'
|
||||
|
||||
// === Rich Result Types (from original neuralAPI) ===
|
||||
|
||||
export interface SimilarityResult {
|
||||
score: number
|
||||
method?: string
|
||||
confidence?: number
|
||||
explanation?: string
|
||||
hierarchy?: {
|
||||
sharedParent?: string
|
||||
distance?: number
|
||||
}
|
||||
breakdown?: {
|
||||
semantic?: number
|
||||
taxonomic?: number
|
||||
contextual?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface SimilarityOptions {
|
||||
explain?: boolean
|
||||
includeBreakdown?: boolean
|
||||
method?: 'cosine' | 'euclidean' | 'hybrid'
|
||||
}
|
||||
|
||||
export interface SemanticCluster {
|
||||
id: string
|
||||
centroid: Vector
|
||||
members: string[]
|
||||
label?: string
|
||||
confidence: number
|
||||
depth?: number
|
||||
// Enterprise additions
|
||||
size?: number
|
||||
level?: number
|
||||
center?: any
|
||||
}
|
||||
|
||||
export interface SemanticHierarchy {
|
||||
self: { id: string; type?: string; vector: Vector }
|
||||
parent?: { id: string; type?: string; similarity: number }
|
||||
grandparent?: { id: string; type?: string; similarity: number }
|
||||
root?: { id: string; type?: string; similarity: number }
|
||||
siblings?: Array<{ id: string; similarity: number }>
|
||||
children?: Array<{ id: string; similarity: number }>
|
||||
depth?: number
|
||||
}
|
||||
|
||||
export interface NeighborGraph {
|
||||
center: string
|
||||
neighbors: Array<{
|
||||
id: string
|
||||
similarity: number
|
||||
type?: string
|
||||
connections?: number
|
||||
}>
|
||||
edges?: Array<{
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
type?: string
|
||||
}>
|
||||
}
|
||||
|
||||
export interface ClusterOptions {
|
||||
algorithm?: 'hierarchical' | 'kmeans' | 'sample' | 'stream'
|
||||
maxClusters?: number
|
||||
threshold?: number
|
||||
// Enterprise options
|
||||
sampleSize?: number
|
||||
strategy?: 'random' | 'diverse' | 'recent'
|
||||
level?: number
|
||||
batchSize?: number
|
||||
}
|
||||
|
||||
export interface VisualizationData {
|
||||
format: 'force-directed' | 'hierarchical' | 'radial'
|
||||
nodes: Array<{
|
||||
id: string
|
||||
x: number
|
||||
y: number
|
||||
z?: number
|
||||
type?: string
|
||||
cluster?: string
|
||||
size?: number
|
||||
}>
|
||||
edges: Array<{
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
type?: string
|
||||
}>
|
||||
layout?: {
|
||||
dimensions: number
|
||||
algorithm: string
|
||||
bounds?: { width: number; height: number; depth?: number }
|
||||
}
|
||||
clusters?: Array<{
|
||||
id: string
|
||||
color: string
|
||||
label?: string
|
||||
size: number
|
||||
}>
|
||||
}
|
||||
|
||||
// === Enterprise Types (from neuralOptimized) ===
|
||||
|
||||
export interface ClusteringStrategy {
|
||||
type: 'sample' | 'hierarchical' | 'stream' | 'hybrid'
|
||||
sampleSize?: number
|
||||
maxClusters?: number
|
||||
minClusterSize?: number
|
||||
}
|
||||
|
||||
export interface LODConfig {
|
||||
levels: number
|
||||
itemsPerLevel: number[]
|
||||
zoomThresholds: number[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Neural API - Unified best-of-both implementation
|
||||
*/
|
||||
export class NeuralAPI {
|
||||
private brain: any // Brainy instance
|
||||
private distanceFn: (a: Vector, b: Vector) => number
|
||||
private similarityCache: Map<string, number> = new Map()
|
||||
private clusterCache: Map<string, any> = new Map() // Enhanced for enterprise
|
||||
private hierarchyCache: Map<string, SemanticHierarchy> = new Map()
|
||||
|
||||
constructor(brain: any) {
|
||||
this.brain = brain
|
||||
this.distanceFn = brain.distance || cosineDistance
|
||||
}
|
||||
|
||||
// ===== SMART USER-FRIENDLY API =====
|
||||
|
||||
/**
|
||||
* Calculate similarity between any two items (smart detection)
|
||||
*/
|
||||
async similar(a: any, b: any, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
// Auto-detect input types
|
||||
if (typeof a === 'string' && typeof b === 'string') {
|
||||
if (this.isId(a) && this.isId(b)) {
|
||||
return this.similarityById(a, b, options)
|
||||
} else {
|
||||
return this.similarityByText(a, b, options)
|
||||
}
|
||||
} else if (Array.isArray(a) && Array.isArray(b)) {
|
||||
return this.similarityByVector(a as Vector, b as Vector, options)
|
||||
}
|
||||
|
||||
// Handle mixed types
|
||||
return this.smartSimilarity(a, b, options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Find semantic clusters (auto-detects best approach)
|
||||
* Now with enterprise performance!
|
||||
*/
|
||||
async clusters(input?: any): Promise<SemanticCluster[]> {
|
||||
// No input? Use enterprise fast clustering
|
||||
if (!input) {
|
||||
return this.clusterFast()
|
||||
}
|
||||
|
||||
// Array? Cluster these items (use large clustering for big arrays)
|
||||
if (Array.isArray(input)) {
|
||||
if (input.length > 1000) {
|
||||
return this.clusterLarge({ sampleSize: Math.min(input.length, 1000) })
|
||||
}
|
||||
return this.clusterItems(input)
|
||||
}
|
||||
|
||||
// String? Find clusters near this
|
||||
if (typeof input === 'string') {
|
||||
return this.clustersNear(input)
|
||||
}
|
||||
|
||||
// Object? Use as config with enterprise algorithms
|
||||
if (typeof input === 'object' && !Array.isArray(input)) {
|
||||
return this.clusterWithConfig(input as ClusterOptions)
|
||||
}
|
||||
|
||||
throw new Error('Invalid input for clustering')
|
||||
}
|
||||
|
||||
/**
|
||||
* Get semantic hierarchy for an item
|
||||
*/
|
||||
async hierarchy(id: string): Promise<SemanticHierarchy> {
|
||||
// Check cache first
|
||||
if (this.hierarchyCache.has(id)) {
|
||||
return this.hierarchyCache.get(id)!
|
||||
}
|
||||
|
||||
const item = await this.brain.get(id)
|
||||
if (!item) {
|
||||
throw new Error(`Item not found: ${id}`)
|
||||
}
|
||||
|
||||
// Find semantic relationships
|
||||
const hierarchy = await this.buildHierarchy(item)
|
||||
|
||||
// Cache result
|
||||
this.hierarchyCache.set(id, hierarchy)
|
||||
|
||||
return hierarchy
|
||||
}
|
||||
|
||||
/**
|
||||
* Find semantic neighbors for visualization
|
||||
*/
|
||||
async neighbors(id: string, options?: {
|
||||
radius?: number
|
||||
limit?: number
|
||||
includeEdges?: boolean
|
||||
}): Promise<NeighborGraph> {
|
||||
const radius = options?.radius ?? 0.3
|
||||
const limit = options?.limit ?? 50
|
||||
|
||||
// Search for nearby items
|
||||
const results = await this.brain.search(id, limit * 2)
|
||||
|
||||
// Filter by semantic radius
|
||||
const neighbors = results
|
||||
.filter((r: any) => r.similarity >= (1 - radius))
|
||||
.slice(0, limit)
|
||||
.map((r: any) => ({
|
||||
id: r.id,
|
||||
similarity: r.similarity,
|
||||
type: r.metadata?.type,
|
||||
connections: r.metadata?.connections?.size || 0
|
||||
}))
|
||||
|
||||
const graph: NeighborGraph = {
|
||||
center: id,
|
||||
neighbors
|
||||
}
|
||||
|
||||
// Add edges if requested
|
||||
if (options?.includeEdges) {
|
||||
graph.edges = await this.buildEdges(id, neighbors)
|
||||
}
|
||||
|
||||
return graph
|
||||
}
|
||||
|
||||
/**
|
||||
* Find semantic path between two items
|
||||
*/
|
||||
async semanticPath(fromId: string, toId: string, options?: {
|
||||
maxHops?: number
|
||||
algorithm?: 'breadth' | 'dijkstra'
|
||||
}): Promise<Array<{
|
||||
id: string
|
||||
similarity: number
|
||||
hop: number
|
||||
}>> {
|
||||
const maxHops = options?.maxHops ?? 5
|
||||
const algorithm = options?.algorithm ?? 'breadth'
|
||||
|
||||
if (algorithm === 'dijkstra') {
|
||||
return this.dijkstraPath(fromId, toId, maxHops)
|
||||
} else {
|
||||
return this.breadthFirstPath(fromId, toId, maxHops)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect semantic outliers
|
||||
*/
|
||||
async outliers(threshold: number = 0.3): Promise<string[]> {
|
||||
// Get all items
|
||||
const stats = this.brain.getStats()
|
||||
const totalItems = stats.entities.total
|
||||
|
||||
if (totalItems === 0) return []
|
||||
|
||||
// For large datasets, use sampling
|
||||
if (totalItems > 10000) {
|
||||
return this.outliersViaSampling(threshold, 1000)
|
||||
}
|
||||
|
||||
return this.outliersByDistance(threshold)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate visualization data
|
||||
*/
|
||||
async visualize(options?: {
|
||||
maxNodes?: number
|
||||
dimensions?: 2 | 3
|
||||
algorithm?: 'force' | 'hierarchical' | 'radial'
|
||||
includeEdges?: boolean
|
||||
}): Promise<VisualizationData> {
|
||||
const maxNodes = options?.maxNodes ?? 100
|
||||
const dimensions = options?.dimensions ?? 2
|
||||
const algorithm = options?.algorithm ?? 'force'
|
||||
|
||||
// Get representative nodes
|
||||
const nodes = await this.getVisualizationNodes(maxNodes)
|
||||
|
||||
// Apply layout algorithm
|
||||
const positioned = await this.applyLayout(nodes, algorithm, dimensions)
|
||||
|
||||
// Build edges if requested
|
||||
const edges = options?.includeEdges !== false ?
|
||||
await this.buildVisualizationEdges(positioned) : []
|
||||
|
||||
// Detect optimal format
|
||||
const format = this.detectOptimalFormat(positioned, edges)
|
||||
|
||||
return {
|
||||
format,
|
||||
nodes: positioned,
|
||||
edges,
|
||||
layout: {
|
||||
dimensions,
|
||||
algorithm,
|
||||
bounds: this.calculateBounds(positioned, dimensions)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== ENTERPRISE PERFORMANCE ALGORITHMS =====
|
||||
|
||||
/**
|
||||
* Fast clustering using HNSW levels - O(n) instead of O(n²)
|
||||
*/
|
||||
async clusterFast(options: {
|
||||
level?: number
|
||||
maxClusters?: number
|
||||
} = {}): Promise<SemanticCluster[]> {
|
||||
const cacheKey = `hierarchical-${options.level}-${options.maxClusters}`
|
||||
if (this.clusterCache.has(cacheKey)) {
|
||||
return this.clusterCache.get(cacheKey)
|
||||
}
|
||||
|
||||
// Use HNSW's natural hierarchy - auto-select optimal level
|
||||
const level = options.level ?? await this.getOptimalClusteringLevel()
|
||||
const maxClusters = options.maxClusters ?? 100
|
||||
|
||||
// Get representative nodes from HNSW level
|
||||
const representatives = await this.getHNSWLevelNodes(level)
|
||||
|
||||
// Each representative is a natural cluster center
|
||||
const clusters = []
|
||||
for (const rep of representatives.slice(0, maxClusters)) {
|
||||
const members = await this.findClusterMembers(rep, level - 1)
|
||||
clusters.push({
|
||||
id: `cluster-${rep.id}`,
|
||||
centroid: rep.vector,
|
||||
center: rep,
|
||||
members: members.map(m => m.id),
|
||||
size: members.length,
|
||||
level,
|
||||
confidence: 0.8 + (members.length / 100) * 0.2 // Size-based confidence
|
||||
} as SemanticCluster)
|
||||
}
|
||||
|
||||
this.clusterCache.set(cacheKey, clusters)
|
||||
return clusters
|
||||
}
|
||||
|
||||
/**
|
||||
* Large-scale clustering for massive datasets (millions of items)
|
||||
*/
|
||||
async clusterLarge(options: {
|
||||
sampleSize?: number
|
||||
strategy?: 'random' | 'diverse' | 'recent'
|
||||
} = {}): Promise<SemanticCluster[]> {
|
||||
const sampleSize = options.sampleSize ?? 1000
|
||||
const strategy = options.strategy ?? 'diverse'
|
||||
|
||||
// Get representative sample
|
||||
const sample = await this.getSample(sampleSize, strategy)
|
||||
|
||||
// Cluster the sample (fast on small set)
|
||||
const sampleClusters = await this.performFastClustering(sample)
|
||||
|
||||
// Project clusters to full dataset
|
||||
return this.projectClustersToFullDataset(sampleClusters)
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming clustering for progressive refinement
|
||||
*/
|
||||
async* clusterStream(options: {
|
||||
batchSize?: number
|
||||
maxBatches?: number
|
||||
} = {}): AsyncGenerator<SemanticCluster[]> {
|
||||
const batchSize = options.batchSize ?? 1000
|
||||
const maxBatches = options.maxBatches ?? Infinity
|
||||
|
||||
let offset = 0
|
||||
let batchCount = 0
|
||||
let globalClusters: SemanticCluster[] = []
|
||||
|
||||
while (batchCount < maxBatches) {
|
||||
// Get next batch
|
||||
const batch = await this.getBatch(offset, batchSize)
|
||||
if (batch.length === 0) break
|
||||
|
||||
// Cluster this batch
|
||||
const batchClusters = await this.performFastClustering(batch)
|
||||
|
||||
// Merge with global clusters
|
||||
globalClusters = await this.mergeClusters(globalClusters, batchClusters)
|
||||
|
||||
// Yield current state
|
||||
yield globalClusters
|
||||
|
||||
offset += batchSize
|
||||
batchCount++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Level-of-detail for massive visualization
|
||||
*/
|
||||
async getLOD(zoomLevel: number, viewport?: {
|
||||
center: Vector
|
||||
radius: number
|
||||
}): Promise<any> {
|
||||
// Define LOD levels based on zoom
|
||||
const lodLevels = [
|
||||
{ zoom: 0, maxNodes: 50, clusterLevel: 3 },
|
||||
{ zoom: 1, maxNodes: 200, clusterLevel: 2 },
|
||||
{ zoom: 2, maxNodes: 1000, clusterLevel: 1 },
|
||||
{ zoom: 3, maxNodes: 5000, clusterLevel: 0 }
|
||||
]
|
||||
|
||||
const lod = lodLevels.find(l => zoomLevel <= l.zoom) || lodLevels[lodLevels.length - 1]
|
||||
|
||||
if (viewport) {
|
||||
return this.getViewportLOD(viewport, lod)
|
||||
} else {
|
||||
return this.getGlobalLOD(lod)
|
||||
}
|
||||
}
|
||||
|
||||
// ===== IMPLEMENTATION HELPERS =====
|
||||
|
||||
private isId(str: string): boolean {
|
||||
// Check if string looks like an ID (UUID pattern, etc.)
|
||||
return (str.length === 36 && str.includes('-')) || !!str.match(/^[a-f0-9]{24}$/)
|
||||
}
|
||||
|
||||
private async similarityById(idA: string, idB: string, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
const cacheKey = `${idA}-${idB}`
|
||||
if (this.similarityCache.has(cacheKey)) {
|
||||
return this.similarityCache.get(cacheKey)!
|
||||
}
|
||||
|
||||
// Get items
|
||||
const [itemA, itemB] = await Promise.all([
|
||||
this.brain.get(idA),
|
||||
this.brain.get(idB)
|
||||
])
|
||||
|
||||
if (!itemA || !itemB) {
|
||||
throw new Error('One or both items not found')
|
||||
}
|
||||
|
||||
// Calculate similarity
|
||||
const score = this.distanceFn(itemA.vector, itemB.vector)
|
||||
|
||||
this.similarityCache.set(cacheKey, score)
|
||||
|
||||
if (options?.explain) {
|
||||
return {
|
||||
score,
|
||||
method: 'cosine',
|
||||
confidence: 0.9,
|
||||
explanation: `Semantic similarity between ${idA} and ${idB}`
|
||||
}
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
private async similarityByText(textA: string, textB: string, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
// Generate embeddings
|
||||
const [vectorA, vectorB] = await Promise.all([
|
||||
this.brain.embed(textA),
|
||||
this.brain.embed(textB)
|
||||
])
|
||||
|
||||
return this.similarityByVector(vectorA, vectorB, options)
|
||||
}
|
||||
|
||||
private async similarityByVector(vectorA: Vector, vectorB: Vector, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
const score = this.distanceFn(vectorA, vectorB)
|
||||
|
||||
if (options?.explain) {
|
||||
return {
|
||||
score,
|
||||
method: options.method || 'cosine',
|
||||
confidence: 0.95,
|
||||
explanation: 'Direct vector similarity calculation'
|
||||
}
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
private async smartSimilarity(a: any, b: any, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
// Convert both to vectors and compare
|
||||
const vectorA = await this.toVector(a)
|
||||
const vectorB = await this.toVector(b)
|
||||
|
||||
return this.similarityByVector(vectorA, vectorB, options)
|
||||
}
|
||||
|
||||
private async toVector(item: any): Promise<Vector> {
|
||||
if (Array.isArray(item)) return item
|
||||
if (typeof item === 'string') {
|
||||
if (this.isId(item)) {
|
||||
const found = await this.brain.get(item)
|
||||
return found?.vector || await this.brain.embed(item)
|
||||
}
|
||||
return await this.brain.embed(item)
|
||||
}
|
||||
if (typeof item === 'object' && item.vector) {
|
||||
return item.vector
|
||||
}
|
||||
// Convert object to string and embed
|
||||
return await this.brain.embed(JSON.stringify(item))
|
||||
}
|
||||
|
||||
// Enterprise clustering implementations
|
||||
private async getOptimalClusteringLevel(): Promise<number> {
|
||||
// Analyze dataset size and return optimal HNSW level
|
||||
const stats = this.brain.getStats()
|
||||
const itemCount = stats.entities.total
|
||||
|
||||
if (itemCount < 1000) return 0
|
||||
if (itemCount < 10000) return 1
|
||||
if (itemCount < 100000) return 2
|
||||
return 3
|
||||
}
|
||||
|
||||
private async getHNSWLevelNodes(level: number): Promise<any[]> {
|
||||
// Get nodes from specific HNSW level
|
||||
// For now, use search to get a representative sample
|
||||
const stats = this.brain.getStats()
|
||||
const sampleSize = Math.min(100, Math.floor(stats.entities.total / (level + 1)))
|
||||
|
||||
// Use search with a general query to get representative items
|
||||
const queryVector = await this.brain.embed('data information content')
|
||||
const allItems = await this.brain.search(queryVector, sampleSize * 2)
|
||||
return allItems.slice(0, sampleSize)
|
||||
}
|
||||
|
||||
private async findClusterMembers(center: any, level: number): Promise<any[]> {
|
||||
// Find all items that belong to this cluster
|
||||
const results = await this.brain.search(center.vector, 50)
|
||||
return results.filter((r: any) => r.similarity > 0.7)
|
||||
}
|
||||
|
||||
private async getSample(size: number, strategy: string): Promise<any[]> {
|
||||
// Use search to get a sample of items
|
||||
const stats = this.brain.getStats()
|
||||
const maxSize = Math.min(size * 3, stats.entities.total) // Get more than needed for sampling
|
||||
const queryVector = await this.brain.embed('sample data content')
|
||||
const allItems = await this.brain.search(queryVector, maxSize)
|
||||
|
||||
switch (strategy) {
|
||||
case 'random':
|
||||
return this.shuffleArray(allItems).slice(0, size)
|
||||
case 'diverse':
|
||||
return this.getDiverseSample(allItems, size)
|
||||
case 'recent':
|
||||
return allItems.slice(-size)
|
||||
default:
|
||||
return allItems.slice(0, size)
|
||||
}
|
||||
}
|
||||
|
||||
private shuffleArray(array: any[]): any[] {
|
||||
const shuffled = [...array]
|
||||
for (let i = shuffled.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]
|
||||
}
|
||||
return shuffled
|
||||
}
|
||||
|
||||
private async getDiverseSample(items: any[], size: number): Promise<any[]> {
|
||||
// Select diverse items using maximum distance sampling
|
||||
if (items.length <= size) return items
|
||||
|
||||
const sample = [items[0]] // Start with first item
|
||||
|
||||
for (let i = 1; i < size; i++) {
|
||||
let maxMinDistance = -1
|
||||
let bestItem = null
|
||||
|
||||
for (const candidate of items) {
|
||||
if (sample.includes(candidate)) continue
|
||||
|
||||
// Find minimum distance to existing sample
|
||||
let minDistance = Infinity
|
||||
for (const selected of sample) {
|
||||
const distance = this.distanceFn(candidate.vector, selected.vector)
|
||||
minDistance = Math.min(minDistance, distance)
|
||||
}
|
||||
|
||||
// Select item with maximum minimum distance
|
||||
if (minDistance > maxMinDistance) {
|
||||
maxMinDistance = minDistance
|
||||
bestItem = candidate
|
||||
}
|
||||
}
|
||||
|
||||
if (bestItem) sample.push(bestItem)
|
||||
}
|
||||
|
||||
return sample
|
||||
}
|
||||
|
||||
private async performFastClustering(items: any[]): Promise<SemanticCluster[]> {
|
||||
// Simple k-means clustering for the sample
|
||||
const k = Math.min(10, Math.floor(items.length / 3))
|
||||
if (k <= 1) {
|
||||
return [{
|
||||
id: 'cluster-0',
|
||||
centroid: items[0]?.vector || [],
|
||||
members: items.map(i => i.id),
|
||||
confidence: 1.0
|
||||
}]
|
||||
}
|
||||
|
||||
// Initialize centroids randomly
|
||||
const centroids = items.slice(0, k).map(item => item.vector)
|
||||
|
||||
// Run k-means iterations (simplified)
|
||||
for (let iter = 0; iter < 10; iter++) {
|
||||
const clusters: (typeof items)[] = Array(k).fill(null).map(() => [])
|
||||
|
||||
// Assign items to nearest centroid
|
||||
for (const item of items) {
|
||||
let bestCluster = 0
|
||||
let bestDistance = Infinity
|
||||
|
||||
for (let c = 0; c < k; c++) {
|
||||
const distance = this.distanceFn(item.vector, centroids[c])
|
||||
if (distance < bestDistance) {
|
||||
bestDistance = distance
|
||||
bestCluster = c
|
||||
}
|
||||
}
|
||||
|
||||
clusters[bestCluster].push(item)
|
||||
}
|
||||
|
||||
// Update centroids
|
||||
for (let c = 0; c < k; c++) {
|
||||
if (clusters[c].length > 0) {
|
||||
const newCentroid = this.calculateCentroid(clusters[c])
|
||||
centroids[c] = newCentroid
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to SemanticCluster format
|
||||
const result: SemanticCluster[] = []
|
||||
for (let c = 0; c < k; c++) {
|
||||
const members = items.filter(item => {
|
||||
let bestCluster = 0
|
||||
let bestDistance = Infinity
|
||||
|
||||
for (let cc = 0; cc < k; cc++) {
|
||||
const distance = this.distanceFn(item.vector, centroids[cc])
|
||||
if (distance < bestDistance) {
|
||||
bestDistance = distance
|
||||
bestCluster = cc
|
||||
}
|
||||
}
|
||||
|
||||
return bestCluster === c
|
||||
})
|
||||
|
||||
if (members.length > 0) {
|
||||
result.push({
|
||||
id: `cluster-${c}`,
|
||||
centroid: centroids[c],
|
||||
members: members.map(m => m.id),
|
||||
confidence: Math.min(0.9, members.length / items.length * 2)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private calculateCentroid(items: any[]): Vector {
|
||||
if (items.length === 0) return []
|
||||
|
||||
const dimensions = items[0].vector.length
|
||||
const centroid = new Array(dimensions).fill(0)
|
||||
|
||||
for (const item of items) {
|
||||
for (let d = 0; d < dimensions; d++) {
|
||||
centroid[d] += item.vector[d]
|
||||
}
|
||||
}
|
||||
|
||||
for (let d = 0; d < dimensions; d++) {
|
||||
centroid[d] /= items.length
|
||||
}
|
||||
|
||||
return centroid
|
||||
}
|
||||
|
||||
private async projectClustersToFullDataset(sampleClusters: SemanticCluster[]): Promise<SemanticCluster[]> {
|
||||
// Project sample clusters to full dataset
|
||||
const result: SemanticCluster[] = []
|
||||
|
||||
for (const cluster of sampleClusters) {
|
||||
// Find all items similar to this cluster's centroid
|
||||
const similar = await this.brain.search(cluster.centroid, 1000)
|
||||
const members = similar
|
||||
.filter((s: any) => s.similarity > 0.6)
|
||||
.map((s: any) => s.id)
|
||||
|
||||
result.push({
|
||||
...cluster,
|
||||
members,
|
||||
size: members.length
|
||||
})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private async mergeClusters(globalClusters: SemanticCluster[], batchClusters: SemanticCluster[]): Promise<SemanticCluster[]> {
|
||||
// Simple merge strategy - combine similar clusters
|
||||
const result = [...globalClusters]
|
||||
|
||||
for (const batchCluster of batchClusters) {
|
||||
let merged = false
|
||||
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
const similarity = this.distanceFn(result[i].centroid, batchCluster.centroid)
|
||||
|
||||
if (similarity > 0.8) {
|
||||
// Merge clusters
|
||||
const newMembers = [...new Set([...result[i].members, ...batchCluster.members])]
|
||||
result[i] = {
|
||||
...result[i],
|
||||
members: newMembers,
|
||||
size: newMembers.length,
|
||||
centroid: this.averageVectors(result[i].centroid, batchCluster.centroid)
|
||||
}
|
||||
merged = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (!merged) {
|
||||
result.push(batchCluster)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private averageVectors(v1: Vector, v2: Vector): Vector {
|
||||
const result = new Array(v1.length)
|
||||
for (let i = 0; i < v1.length; i++) {
|
||||
result[i] = (v1[i] + v2[i]) / 2
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private async getBatch(offset: number, size: number): Promise<any[]> {
|
||||
// Get batch of items for streaming using search with offset
|
||||
const queryVector = await this.brain.embed('batch data content')
|
||||
const items = await this.brain.search(queryVector, size, { offset })
|
||||
return items
|
||||
}
|
||||
|
||||
// Additional methods needed for full compatibility...
|
||||
private async clusterAll(): Promise<SemanticCluster[]> {
|
||||
return this.clusterFast()
|
||||
}
|
||||
|
||||
private async clusterItems(items: any[]): Promise<SemanticCluster[]> {
|
||||
return this.performFastClustering(items)
|
||||
}
|
||||
|
||||
private async clustersNear(id: string): Promise<SemanticCluster[]> {
|
||||
const neighbors = await this.neighbors(id, { limit: 100 })
|
||||
return this.performFastClustering(neighbors.neighbors)
|
||||
}
|
||||
|
||||
private async clusterWithConfig(config: ClusterOptions): Promise<SemanticCluster[]> {
|
||||
switch (config.algorithm) {
|
||||
case 'hierarchical':
|
||||
return this.clusterFast(config)
|
||||
case 'sample':
|
||||
return this.clusterLarge(config)
|
||||
case 'stream':
|
||||
const generator = this.clusterStream(config)
|
||||
const results = []
|
||||
for await (const batch of generator) {
|
||||
results.push(...batch)
|
||||
}
|
||||
return results
|
||||
default:
|
||||
return this.clusterFast(config)
|
||||
}
|
||||
}
|
||||
|
||||
// Placeholder implementations for remaining methods
|
||||
private async buildHierarchy(item: any): Promise<SemanticHierarchy> {
|
||||
// Implementation for hierarchy building
|
||||
return {
|
||||
self: { id: item.id, vector: item.vector }
|
||||
}
|
||||
}
|
||||
|
||||
private async buildEdges(centerId: string, neighbors: any[]): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async dijkstraPath(from: string, to: string, maxHops: number): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async breadthFirstPath(from: string, to: string, maxHops: number): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async outliersViaSampling(threshold: number, sampleSize: number): Promise<string[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async outliersByDistance(threshold: number): Promise<string[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async getVisualizationNodes(maxNodes: number): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async applyLayout(nodes: any[], algorithm: string, dimensions: number): Promise<any[]> {
|
||||
return nodes
|
||||
}
|
||||
|
||||
private async buildVisualizationEdges(nodes: any[]): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private detectOptimalFormat(nodes: any[], edges: any[]): 'force-directed' | 'hierarchical' | 'radial' {
|
||||
return 'force-directed'
|
||||
}
|
||||
|
||||
private calculateBounds(nodes: any[], dimensions: number): any {
|
||||
return { width: 100, height: 100 }
|
||||
}
|
||||
|
||||
private async getViewportLOD(viewport: any, lod: any): Promise<any> {
|
||||
// LOD visualization is an optional advanced feature
|
||||
// Return default view without LOD optimization
|
||||
console.warn('Viewport LOD optimization not available. Using standard view.')
|
||||
return { nodes: [], edges: [], optimized: false }
|
||||
}
|
||||
|
||||
private async getGlobalLOD(lod: any): Promise<any> {
|
||||
// LOD visualization is an optional advanced feature
|
||||
// Return default view without LOD optimization
|
||||
console.warn('Global LOD optimization not available. Using standard view.')
|
||||
return { nodes: [], edges: [], optimized: false }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,342 +0,0 @@
|
|||
/**
|
||||
* Neural API Type Definitions
|
||||
* Comprehensive interfaces for clustering, similarity, and analysis
|
||||
*/
|
||||
|
||||
export interface Vector {
|
||||
[index: number]: number
|
||||
length: number
|
||||
}
|
||||
|
||||
// ===== CORE CLUSTERING INTERFACES =====
|
||||
|
||||
export interface SemanticCluster {
|
||||
id: string
|
||||
centroid: Vector
|
||||
members: string[]
|
||||
size: number
|
||||
confidence: number
|
||||
label?: string
|
||||
metadata?: Record<string, any>
|
||||
cohesion?: number
|
||||
level?: number
|
||||
}
|
||||
|
||||
export interface ClusterEdge {
|
||||
id: string
|
||||
source: string
|
||||
target: string
|
||||
type: string
|
||||
weight?: number
|
||||
isInterCluster: boolean
|
||||
sourceCluster?: string
|
||||
targetCluster?: string
|
||||
}
|
||||
|
||||
export interface EnhancedSemanticCluster extends SemanticCluster {
|
||||
intraClusterEdges: ClusterEdge[]
|
||||
interClusterEdges: ClusterEdge[]
|
||||
relationshipSummary: {
|
||||
totalEdges: number
|
||||
intraClusterEdges: number
|
||||
interClusterEdges: number
|
||||
edgeTypes: Record<string, number>
|
||||
}
|
||||
}
|
||||
|
||||
export interface DomainCluster extends SemanticCluster {
|
||||
domain: string
|
||||
domainConfidence: number
|
||||
crossDomainMembers?: string[]
|
||||
}
|
||||
|
||||
export interface TemporalCluster extends SemanticCluster {
|
||||
timeWindow: TimeWindow
|
||||
trend?: 'increasing' | 'decreasing' | 'stable'
|
||||
temporal: {
|
||||
startTime: Date
|
||||
endTime: Date
|
||||
peakTime?: Date
|
||||
frequency?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface ExplainableCluster extends SemanticCluster {
|
||||
explanation: {
|
||||
primaryFeatures: string[]
|
||||
commonTerms: string[]
|
||||
reasoning: string
|
||||
confidence: number
|
||||
}
|
||||
subClusters?: ExplainableCluster[]
|
||||
}
|
||||
|
||||
export interface ConfidentCluster extends SemanticCluster {
|
||||
minConfidence: number
|
||||
uncertainMembers: string[]
|
||||
certainMembers: string[]
|
||||
}
|
||||
|
||||
// ===== CLUSTERING OPTIONS =====
|
||||
|
||||
export interface BaseClusteringOptions {
|
||||
maxClusters?: number
|
||||
minClusterSize?: number
|
||||
threshold?: number
|
||||
cacheResults?: boolean
|
||||
}
|
||||
|
||||
export interface ClusteringOptions extends BaseClusteringOptions {
|
||||
algorithm?: 'auto' | 'hierarchical' | 'kmeans' | 'dbscan' | 'sample' | 'semantic' | 'graph' | 'multimodal'
|
||||
sampleSize?: number
|
||||
strategy?: 'random' | 'diverse' | 'recent' | 'important'
|
||||
memoryLimit?: string // e.g., '512MB'
|
||||
includeOutliers?: boolean
|
||||
// K-means specific options
|
||||
maxIterations?: number
|
||||
tolerance?: number
|
||||
}
|
||||
|
||||
export interface DomainClusteringOptions extends BaseClusteringOptions {
|
||||
domainField?: string
|
||||
crossDomainThreshold?: number
|
||||
preserveDomainBoundaries?: boolean
|
||||
}
|
||||
|
||||
export interface TemporalClusteringOptions extends BaseClusteringOptions {
|
||||
timeField: string
|
||||
windows: TimeWindow[]
|
||||
overlapStrategy?: 'merge' | 'separate' | 'hierarchical'
|
||||
trendAnalysis?: boolean
|
||||
}
|
||||
|
||||
export interface StreamClusteringOptions extends BaseClusteringOptions {
|
||||
batchSize?: number
|
||||
updateInterval?: number
|
||||
adaptiveThreshold?: boolean
|
||||
decayFactor?: number // For aging old clusters
|
||||
}
|
||||
|
||||
// ===== SIMILARITY & NEIGHBORS =====
|
||||
|
||||
export interface SimilarityOptions {
|
||||
detailed?: boolean
|
||||
metric?: 'cosine' | 'euclidean' | 'manhattan' | 'jaccard'
|
||||
normalized?: boolean
|
||||
}
|
||||
|
||||
export interface SimilarityResult {
|
||||
score: number
|
||||
confidence: number
|
||||
explanation?: string
|
||||
metric?: string
|
||||
}
|
||||
|
||||
export interface NeighborOptions {
|
||||
limit?: number
|
||||
radius?: number
|
||||
minSimilarity?: number
|
||||
includeMetadata?: boolean
|
||||
sortBy?: 'similarity' | 'importance' | 'recency'
|
||||
}
|
||||
|
||||
export interface Neighbor {
|
||||
id: string
|
||||
similarity: number
|
||||
data?: any
|
||||
metadata?: Record<string, any>
|
||||
distance?: number
|
||||
}
|
||||
|
||||
export interface NeighborsResult {
|
||||
neighbors: Neighbor[]
|
||||
queryId: string
|
||||
totalFound: number
|
||||
averageSimilarity: number
|
||||
}
|
||||
|
||||
// ===== HIERARCHY & ANALYSIS =====
|
||||
|
||||
export interface SemanticHierarchy {
|
||||
self?: { id: string; vector?: Vector; metadata?: any }
|
||||
root?: { id: string; vector?: Vector; metadata?: any } | null
|
||||
levels?: any[]
|
||||
parent?: { id: string; similarity: number }
|
||||
children?: Array<{ id: string; similarity: number }>
|
||||
siblings?: Array<{ id: string; similarity: number }>
|
||||
level?: number
|
||||
depth?: number
|
||||
}
|
||||
|
||||
export interface HierarchyOptions {
|
||||
maxDepth?: number
|
||||
minSimilarity?: number
|
||||
includeMetadata?: boolean
|
||||
buildStrategy?: 'similarity' | 'metadata' | 'mixed'
|
||||
}
|
||||
|
||||
// ===== VISUALIZATION =====
|
||||
|
||||
export interface VisualizationOptions {
|
||||
maxNodes?: number
|
||||
dimensions?: 2 | 3
|
||||
algorithm?: 'force' | 'spring' | 'circular' | 'hierarchical'
|
||||
includeEdges?: boolean
|
||||
clusterColors?: boolean
|
||||
nodeSize?: 'uniform' | 'importance' | 'connections'
|
||||
}
|
||||
|
||||
export interface VisualizationNode {
|
||||
id: string
|
||||
x: number
|
||||
y: number
|
||||
z?: number
|
||||
cluster?: string
|
||||
size?: number
|
||||
color?: string
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface VisualizationEdge {
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
color?: string
|
||||
type?: string
|
||||
}
|
||||
|
||||
export interface VisualizationResult {
|
||||
nodes: VisualizationNode[]
|
||||
edges: VisualizationEdge[]
|
||||
clusters?: Array<{
|
||||
id: string
|
||||
color: string
|
||||
size: number
|
||||
label?: string
|
||||
}>
|
||||
metadata: {
|
||||
algorithm: string
|
||||
dimensions: number
|
||||
totalNodes: number
|
||||
totalEdges: number
|
||||
generatedAt: Date
|
||||
}
|
||||
}
|
||||
|
||||
// ===== UTILITY TYPES =====
|
||||
|
||||
export interface TimeWindow {
|
||||
start: Date
|
||||
end: Date
|
||||
label?: string
|
||||
weight?: number
|
||||
}
|
||||
|
||||
export interface ClusterFeedback {
|
||||
clusterId: string
|
||||
action: 'merge' | 'split' | 'relabel' | 'adjust'
|
||||
parameters?: Record<string, any>
|
||||
confidence?: number
|
||||
}
|
||||
|
||||
export interface OutlierOptions {
|
||||
threshold?: number
|
||||
method?: 'isolation' | 'statistical' | 'cluster-based'
|
||||
minNeighbors?: number
|
||||
includeReasons?: boolean
|
||||
}
|
||||
|
||||
export interface Outlier {
|
||||
id: string
|
||||
score: number
|
||||
reasons?: string[]
|
||||
nearestNeighbors?: Neighbor[]
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
// ===== PERFORMANCE & MONITORING =====
|
||||
|
||||
export interface PerformanceMetrics {
|
||||
executionTime: number
|
||||
memoryUsed: number
|
||||
itemsProcessed: number
|
||||
cacheHits: number
|
||||
cacheMisses: number
|
||||
algorithm: string
|
||||
}
|
||||
|
||||
export interface ClusteringResult<T = SemanticCluster> {
|
||||
clusters: T[]
|
||||
metrics: PerformanceMetrics
|
||||
metadata: {
|
||||
totalItems: number
|
||||
clustersFound: number
|
||||
averageClusterSize: number
|
||||
silhouetteScore?: number
|
||||
timestamp: Date
|
||||
// Additional clustering-specific metadata
|
||||
semanticTypes?: number
|
||||
hnswLevel?: number
|
||||
kValue?: number
|
||||
hasConverged?: boolean
|
||||
outlierCount?: number
|
||||
eps?: number
|
||||
minPts?: number
|
||||
averageModularity?: number
|
||||
fusionMethod?: string
|
||||
componentAlgorithms?: string[]
|
||||
sampleSize?: number
|
||||
samplingStrategy?: string
|
||||
}
|
||||
}
|
||||
|
||||
// ===== STREAMING =====
|
||||
|
||||
export interface StreamingBatch<T = SemanticCluster> {
|
||||
clusters: T[]
|
||||
batchNumber: number
|
||||
isComplete: boolean
|
||||
progress: {
|
||||
processed: number
|
||||
total: number
|
||||
percentage: number
|
||||
}
|
||||
metrics: PerformanceMetrics
|
||||
}
|
||||
|
||||
// ===== ERROR TYPES =====
|
||||
|
||||
export class NeuralAPIError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public code: string,
|
||||
public context?: Record<string, any>
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'NeuralAPIError'
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusteringError extends NeuralAPIError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'CLUSTERING_ERROR', context)
|
||||
}
|
||||
}
|
||||
|
||||
export class SimilarityError extends NeuralAPIError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'SIMILARITY_ERROR', context)
|
||||
}
|
||||
}
|
||||
|
||||
// ===== CONFIGURATION =====
|
||||
|
||||
export interface NeuralAPIConfig {
|
||||
cacheSize?: number
|
||||
defaultAlgorithm?: string
|
||||
similarityMetric?: 'cosine' | 'euclidean' | 'manhattan'
|
||||
performanceTracking?: boolean
|
||||
maxMemoryUsage?: string
|
||||
parallelProcessing?: boolean
|
||||
streamingBatchSize?: number
|
||||
}
|
||||
|
|
@ -1622,7 +1622,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
` Version: ${existing.version}\n` +
|
||||
` Directory: ${this.rootDir}\n\n` +
|
||||
`For diagnostic queries against this live store, use:\n` +
|
||||
` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', rootDirectory: '${this.rootDir}' } })\n\n` +
|
||||
` const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '${this.rootDir}' } })\n\n` +
|
||||
`If you have verified the existing lock is stale (e.g. a crashed writer on a different host that PID liveness cannot reach), pass { force: true }.`
|
||||
) as Error & { code: string; lockInfo: WriterLockInfo }
|
||||
err.code = 'BRAINY_WRITER_LOCKED'
|
||||
|
|
|
|||
|
|
@ -30,6 +30,9 @@ export interface StorageOptions {
|
|||
* in a browser.
|
||||
* - `'memory'` — in-memory; ephemeral.
|
||||
* - `'filesystem'` — persistent disk storage; Node-like runtimes only.
|
||||
*
|
||||
* A top-level `path` implies `'filesystem'`, so `{ path: '/data' }` works
|
||||
* without an explicit `type`.
|
||||
*/
|
||||
type?: 'auto' | 'memory' | 'filesystem'
|
||||
|
||||
|
|
@ -39,20 +42,25 @@ export interface StorageOptions {
|
|||
/** Force filesystem storage. Throws in a browser environment. */
|
||||
forceFileSystemStorage?: boolean
|
||||
|
||||
/** Root directory for filesystem storage. */
|
||||
rootDirectory?: string
|
||||
|
||||
/**
|
||||
* Alias for `rootDirectory`. The `storage: { type: 'filesystem', path: '…' }`
|
||||
* shape is widely used (and shown throughout the docs), so it is honored as a
|
||||
* first-class top-level key — not silently ignored in favour of the default
|
||||
* directory.
|
||||
* **Canonical** directory for filesystem storage. This is the one key the
|
||||
* rest of the API already speaks (`persist(path)`, `Brainy.load(path)`,
|
||||
* `asOf(path)`, `restore(path)`). Specifying it implies `type: 'filesystem'`.
|
||||
*
|
||||
* @example
|
||||
* new Brainy({ storage: { path: '/var/lib/app/data' } })
|
||||
*/
|
||||
path?: string
|
||||
|
||||
/**
|
||||
* Nested options block (backward-compat for `BrainyConfig.storage.options`).
|
||||
* Recognized keys: `rootDirectory`, `path`.
|
||||
* @deprecated REMOVED in 8.0 — passing `rootDirectory` THROWS. Use the
|
||||
* canonical {@link StorageOptions.path} (`storage: { path: '/data' }`).
|
||||
*/
|
||||
rootDirectory?: string
|
||||
|
||||
/**
|
||||
* @deprecated REMOVED in 8.0 — a nested `options.path` / `options.rootDirectory`
|
||||
* THROWS. Use the top-level {@link StorageOptions.path}.
|
||||
*/
|
||||
options?: {
|
||||
rootDirectory?: string
|
||||
|
|
@ -60,10 +68,120 @@ export interface StorageOptions {
|
|||
[key: string]: any
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated REMOVED in 8.0 — `fileSystemStorage.path` / `.rootDirectory`
|
||||
* THROWS. Use the top-level {@link StorageOptions.path}.
|
||||
*/
|
||||
fileSystemStorage?: {
|
||||
rootDirectory?: string
|
||||
path?: string
|
||||
[key: string]: any
|
||||
}
|
||||
|
||||
/** Operation tuning (timeouts, retry budgets) passed through to the adapter. */
|
||||
operationConfig?: OperationConfig
|
||||
}
|
||||
|
||||
/** Default on-disk root when a filesystem store is requested without a path. */
|
||||
export const DEFAULT_FILESYSTEM_ROOT = './brainy-data'
|
||||
|
||||
/**
|
||||
* @description Throw a clear migration error for a removed legacy storage-config
|
||||
* key. 8.0 is a clean break: the pre-8.0 aliases were REMOVED (not deprecated),
|
||||
* so a stale config fails loudly with the exact rename rather than silently
|
||||
* writing to the wrong directory.
|
||||
* @param key - The removed key path (e.g. `'rootDirectory'`, `'options.path'`).
|
||||
* @throws Always — naming the canonical `path` replacement.
|
||||
*/
|
||||
function throwRemovedStorageKey(key: string): never {
|
||||
throw new Error(
|
||||
`[brainy] storage config '${key}' was removed in 8.0 — use the top-level ` +
|
||||
`'path' instead (e.g. storage: { path: '/data' }).`
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Resolve any supported filesystem storage config shape to the ONE
|
||||
* canonical on-disk root, encoding the full precedence in a single place so the
|
||||
* factory, the 7.x→8.0 migration probe, and any plugin storage factory all agree
|
||||
* on the IDENTICAL directory (no `./brainy-data` split-brain).
|
||||
*
|
||||
* Behavior:
|
||||
* 1. `path` — the one supported key. Returned as-is.
|
||||
* 2. A removed pre-8.0 alias (`rootDirectory`, `options.*`, `fileSystemStorage.*`) →
|
||||
* THROW with the exact rename (never silently fall through to the default,
|
||||
* which would misplace a 7.x consumer's data on upgrade).
|
||||
* 3. No path at all → `./brainy-data` (the zero-config default).
|
||||
*
|
||||
* @param config - A storage config object (`StorageOptions`-shaped; tolerant of
|
||||
* extra keys for the plugin-factory `Record<string, unknown>` contract).
|
||||
* @returns The resolved directory string.
|
||||
* @throws If a removed pre-8.0 alias is present (message names the `path` rename).
|
||||
* @example
|
||||
* resolveFilesystemRoot({ path: '/data' }) // → '/data'
|
||||
* resolveFilesystemRoot({ rootDirectory: '/data' }) // → throws (use `path`)
|
||||
* resolveFilesystemRoot({ type: 'filesystem' }) // → './brainy-data'
|
||||
*/
|
||||
export function resolveFilesystemRoot(
|
||||
config: StorageOptions & Record<string, unknown> = {}
|
||||
): string {
|
||||
// 1. Canonical top-level path — the one and only supported key.
|
||||
if (typeof config.path === 'string' && config.path.length > 0) {
|
||||
return config.path
|
||||
}
|
||||
|
||||
// 2. Removed pre-8.0 aliases → throw with the exact rename. Detect them even
|
||||
// though they're no longer the path, so a stale config fails loudly
|
||||
// instead of silently landing on the default and misplacing data.
|
||||
if (typeof config.rootDirectory === 'string' && config.rootDirectory.length > 0) {
|
||||
throwRemovedStorageKey('rootDirectory')
|
||||
}
|
||||
const opts = config.options
|
||||
if (
|
||||
opts &&
|
||||
typeof opts === 'object' &&
|
||||
((typeof opts.path === 'string' && opts.path.length > 0) ||
|
||||
(typeof opts.rootDirectory === 'string' && opts.rootDirectory.length > 0))
|
||||
) {
|
||||
throwRemovedStorageKey('options.path')
|
||||
}
|
||||
const fss = config.fileSystemStorage
|
||||
if (
|
||||
fss &&
|
||||
typeof fss === 'object' &&
|
||||
((typeof fss.path === 'string' && fss.path.length > 0) ||
|
||||
(typeof fss.rootDirectory === 'string' && fss.rootDirectory.length > 0))
|
||||
) {
|
||||
throwRemovedStorageKey('fileSystemStorage.path')
|
||||
}
|
||||
|
||||
// 3. Zero-config default. A `type: 'filesystem'` with no path lands here
|
||||
// intentionally ("persist, default location").
|
||||
return DEFAULT_FILESYSTEM_ROOT
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Whether a storage config (no explicit/`'auto'` type) names a
|
||||
* filesystem directory through ANY supported shape. Used by `pickAdapter` so
|
||||
* `{ path: '/data' }` (or a deprecated alias) implies filesystem on a Node
|
||||
* runtime without the caller writing `type: 'filesystem'`.
|
||||
* @param config - A storage config object.
|
||||
* @returns `true` if a directory was specified through any recognized key.
|
||||
*/
|
||||
function hasExplicitFilesystemPath(
|
||||
config: StorageOptions & Record<string, unknown>
|
||||
): boolean {
|
||||
const nonEmpty = (v: unknown): boolean => typeof v === 'string' && v.length > 0
|
||||
return (
|
||||
nonEmpty(config.path) ||
|
||||
nonEmpty(config.rootDirectory) ||
|
||||
nonEmpty(config.options?.path) ||
|
||||
nonEmpty(config.options?.rootDirectory) ||
|
||||
nonEmpty(config.fileSystemStorage?.path) ||
|
||||
nonEmpty(config.fileSystemStorage?.rootDirectory)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `StorageOptions` to a concrete storage adapter.
|
||||
*
|
||||
|
|
@ -91,7 +209,17 @@ async function pickAdapter(options: StorageOptions): Promise<StorageAdapter> {
|
|||
return await createFilesystemStorage(options)
|
||||
}
|
||||
|
||||
// 'auto': prefer filesystem, fall back to memory if init fails.
|
||||
// 'auto' (or no type) with an explicit filesystem path → filesystem. A naked
|
||||
// `{ path: '/data' }` should land on disk at that path, not silently fall to
|
||||
// memory on a runtime where the `auto` filesystem attempt happens to fail.
|
||||
if (
|
||||
requestedType === 'auto' &&
|
||||
hasExplicitFilesystemPath(options as StorageOptions & Record<string, unknown>)
|
||||
) {
|
||||
return await createFilesystemStorage(options)
|
||||
}
|
||||
|
||||
// 'auto' with no path: prefer filesystem, fall back to memory if init fails.
|
||||
try {
|
||||
return await createFilesystemStorage(options)
|
||||
} catch {
|
||||
|
|
@ -100,12 +228,8 @@ async function pickAdapter(options: StorageOptions): Promise<StorageAdapter> {
|
|||
}
|
||||
|
||||
async function createFilesystemStorage(options: StorageOptions): Promise<StorageAdapter> {
|
||||
const rootDir =
|
||||
options.rootDirectory ??
|
||||
options.path ??
|
||||
options.options?.rootDirectory ??
|
||||
options.options?.path ??
|
||||
'./brainy-data'
|
||||
// Single source of truth for the on-disk root across every config shape.
|
||||
const rootDir = resolveFilesystemRoot(options as StorageOptions & Record<string, unknown>)
|
||||
|
||||
// Dynamic import so browser bundles don't pull node:fs.
|
||||
const { FileSystemStorage } = await import('./adapters/fileSystemStorage.js')
|
||||
|
|
|
|||
|
|
@ -1282,14 +1282,37 @@ export interface BrainyConfig {
|
|||
*/
|
||||
storage?:
|
||||
| {
|
||||
type: 'auto' | 'memory' | 'filesystem'
|
||||
/** Root directory for filesystem storage. Passed through to storage factories
|
||||
* including plugin-provided factories (e.g. native mmap providers). */
|
||||
rootDirectory?: string
|
||||
/** Alias for `rootDirectory`. The `storage: { type: 'filesystem', path: '…' }`
|
||||
* shape is honored as a first-class key so existing configs keep working. */
|
||||
/**
|
||||
* Storage backend. Optional — a top-level `path` implies `'filesystem'`,
|
||||
* so `storage: { path: '/data' }` works without it. `'auto'` (the
|
||||
* default) picks filesystem on Node, memory in a browser.
|
||||
*/
|
||||
type?: 'auto' | 'memory' | 'filesystem'
|
||||
/**
|
||||
* **Canonical** directory for filesystem storage. The rest of the API
|
||||
* already speaks `path` (`persist(path)`, `Brainy.load(path)`,
|
||||
* `asOf(path)`, `restore(path)`). Specifying it implies
|
||||
* `type: 'filesystem'`. Passed through to plugin-provided storage
|
||||
* factories (e.g. native mmap providers) so they resolve the same root.
|
||||
* @example
|
||||
* new Brainy({ storage: { path: '/var/lib/app/data' } })
|
||||
*/
|
||||
path?: string
|
||||
/**
|
||||
* @deprecated REMOVED in 8.0 — passing `rootDirectory` THROWS. Use the
|
||||
* canonical {@link path}.
|
||||
*/
|
||||
rootDirectory?: string
|
||||
/**
|
||||
* @deprecated REMOVED in 8.0 — a nested `options.path` / `options.rootDirectory`
|
||||
* THROWS. Use the top-level {@link path}.
|
||||
*/
|
||||
options?: any
|
||||
/**
|
||||
* @deprecated REMOVED in 8.0 — `fileSystemStorage.path` / `.rootDirectory`
|
||||
* THROWS. Use the top-level {@link path}.
|
||||
*/
|
||||
fileSystemStorage?: { path?: string; rootDirectory?: string; [key: string]: any }
|
||||
}
|
||||
| StorageAdapter
|
||||
|
||||
|
|
|
|||
|
|
@ -1482,18 +1482,17 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
const relativePath = oldChildPath.substring(oldParentPath.length)
|
||||
const newChildPath = newParentPath + relativePath
|
||||
|
||||
// Update child entity
|
||||
const updatedChild = {
|
||||
...child,
|
||||
// Update child entity — metadata-only (mirrors rename() above). Spreading
|
||||
// the whole child forwards its `vector` field into update(), which fails
|
||||
// dimension validation when the child was fetched without vectors (and
|
||||
// would needlessly touch the vector index when it wasn't).
|
||||
await this.brain.update({
|
||||
id: child.id,
|
||||
metadata: {
|
||||
...child.metadata,
|
||||
path: newChildPath,
|
||||
modified: Date.now()
|
||||
}
|
||||
}
|
||||
await this.brain.update({
|
||||
...updatedChild,
|
||||
id: child.id
|
||||
})
|
||||
|
||||
// Update path cache
|
||||
|
|
@ -1923,18 +1922,14 @@ export class VirtualFileSystem implements IVirtualFileSystem {
|
|||
async move(src: string, dest: string): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Move is just copy + delete
|
||||
await this.copy(src, dest, { overwrite: false })
|
||||
|
||||
// Delete source after successful copy
|
||||
const srcEntityId = await this.pathResolver.resolve(src)
|
||||
const srcEntity = await this.brain.get(srcEntityId)
|
||||
|
||||
if (srcEntity!.metadata.vfsType === 'file') {
|
||||
await this.unlink(src)
|
||||
} else {
|
||||
await this.rmdir(src, { recursive: true })
|
||||
}
|
||||
// A move is a RENAME (in-place path change), NOT copy + delete. The old
|
||||
// copy+delete path was broken on the content-addressed blob store: copy()
|
||||
// makes the destination reference the SAME content-hash as the source, then
|
||||
// unlink(src) deletes that shared blob — orphaning the destination
|
||||
// ("Blob metadata not found" on the next read). rename() updates the path
|
||||
// in place (preserving the blob, keeping the same entity id) and already
|
||||
// handles both files and directories, including child path updates.
|
||||
await this.rename(src, dest)
|
||||
}
|
||||
|
||||
async symlink(target: string, path: string): Promise<void> {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue