feat(8.0): API simplification — remove neural()/Db.search, one storage path key, integration→0
8.0 RC cleanup toward "one place per thing, zero-config, no deprecation":
- Remove the `brain.neural()` clustering namespace (ImprovedNeuralAPI + the dead
legacy NeuralAPI + the neural CLI + neural-only types). Similarity is `find({vector})`
/ `similar({to})`; attribute grouping is the aggregation `GROUP BY` engine. The separate
entity-extraction / smart-import feature (NeuralImport, NeuralEntityExtractor, SmartExtractor,
NaturalLanguageProcessor, `brain.extract()`/`brain.nlp()`) is kept.
- Remove `Db.search()`; `find()` is the one query verb (accepts a bare string or FindParams).
Fix the bundled MCP client, which called a non-existent `brain.search(query, limit)` →
now `find({ query, limit })`.
- Storage config: collapse to one canonical top-level `path` key. The pre-8.0 aliases
(`rootDirectory`, `options.*`, `fileSystemStorage.*`) are removed and now THROW with the
exact rename instead of silently defaulting to `./brainy-data` on upgrade. A single resolver
feeds createStorage, the 7.x→8.0 migration probe, and the plugin-factory handoff, so a native
storage provider resolves the identical root (no split-brain).
- Fix `similar({ threshold })`: the min-similarity filter was silently dropped; it is now
applied as a post-filter on `result.score` (the documented way to bound semantic results).
- Fix `vfs.rename()` on a directory: child path updates spread the entity vector into `update()`
and failed dimension validation; they are metadata-only updates now.
- Fix `vfs.move()`: copy+delete orphaned the content-addressed content blob (the destination
shared the source hash, then unlink removed it). `move()` now delegates to `rename()` — an
in-place path change that preserves the blob and the entity id, for files and directories.
- Fix streaming import: the bulk fast path never flushed mid-import nor signalled queryability.
Entity writes are now chunked by a progressive flush interval (100 → 1000 → 5000); each chunk
flushes and emits `progress.queryable`, so imported data is queryable during the import.
- Sweep all docs, comments, and JSDoc for the removed/changed APIs.
Integration suite: 49 files / 588 passed / 0 failed. Unit: 80 files / 1456 passed, no type errors.
This commit is contained in:
parent
0c4a51c24e
commit
606445cd61
74 changed files with 712 additions and 7470 deletions
File diff suppressed because it is too large
Load diff
|
|
@ -1010,10 +1010,11 @@ export class NaturalLanguageProcessor {
|
|||
// This would integrate with Brainy's search to find similar query patterns
|
||||
|
||||
// Future implementation could search a query_history noun type:
|
||||
// const similarQueries = await this.brainy.search(queryEmbedding, {
|
||||
// const similarQueries = await this.brainy.find({
|
||||
// vector: queryEmbedding,
|
||||
// limit: 5,
|
||||
// metadata: { type: 'successful_query' },
|
||||
// nounTypes: ['query_history']
|
||||
// where: { type: 'successful_query' },
|
||||
// type: ['query_history']
|
||||
// })
|
||||
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -1,887 +0,0 @@
|
|||
/**
|
||||
* Neural API - Unified Semantic Intelligence
|
||||
*
|
||||
* Best-of-both: Complete functionality + Enterprise performance
|
||||
* Combines rich features with O(n) algorithms for millions of items
|
||||
*/
|
||||
|
||||
import { Vector, HNSWNoun } from '../coreTypes.js'
|
||||
import { cosineDistance } from '../utils/distance.js'
|
||||
|
||||
// === Rich Result Types (from original neuralAPI) ===
|
||||
|
||||
export interface SimilarityResult {
|
||||
score: number
|
||||
method?: string
|
||||
confidence?: number
|
||||
explanation?: string
|
||||
hierarchy?: {
|
||||
sharedParent?: string
|
||||
distance?: number
|
||||
}
|
||||
breakdown?: {
|
||||
semantic?: number
|
||||
taxonomic?: number
|
||||
contextual?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface SimilarityOptions {
|
||||
explain?: boolean
|
||||
includeBreakdown?: boolean
|
||||
method?: 'cosine' | 'euclidean' | 'hybrid'
|
||||
}
|
||||
|
||||
export interface SemanticCluster {
|
||||
id: string
|
||||
centroid: Vector
|
||||
members: string[]
|
||||
label?: string
|
||||
confidence: number
|
||||
depth?: number
|
||||
// Enterprise additions
|
||||
size?: number
|
||||
level?: number
|
||||
center?: any
|
||||
}
|
||||
|
||||
export interface SemanticHierarchy {
|
||||
self: { id: string; type?: string; vector: Vector }
|
||||
parent?: { id: string; type?: string; similarity: number }
|
||||
grandparent?: { id: string; type?: string; similarity: number }
|
||||
root?: { id: string; type?: string; similarity: number }
|
||||
siblings?: Array<{ id: string; similarity: number }>
|
||||
children?: Array<{ id: string; similarity: number }>
|
||||
depth?: number
|
||||
}
|
||||
|
||||
export interface NeighborGraph {
|
||||
center: string
|
||||
neighbors: Array<{
|
||||
id: string
|
||||
similarity: number
|
||||
type?: string
|
||||
connections?: number
|
||||
}>
|
||||
edges?: Array<{
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
type?: string
|
||||
}>
|
||||
}
|
||||
|
||||
export interface ClusterOptions {
|
||||
algorithm?: 'hierarchical' | 'kmeans' | 'sample' | 'stream'
|
||||
maxClusters?: number
|
||||
threshold?: number
|
||||
// Enterprise options
|
||||
sampleSize?: number
|
||||
strategy?: 'random' | 'diverse' | 'recent'
|
||||
level?: number
|
||||
batchSize?: number
|
||||
}
|
||||
|
||||
export interface VisualizationData {
|
||||
format: 'force-directed' | 'hierarchical' | 'radial'
|
||||
nodes: Array<{
|
||||
id: string
|
||||
x: number
|
||||
y: number
|
||||
z?: number
|
||||
type?: string
|
||||
cluster?: string
|
||||
size?: number
|
||||
}>
|
||||
edges: Array<{
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
type?: string
|
||||
}>
|
||||
layout?: {
|
||||
dimensions: number
|
||||
algorithm: string
|
||||
bounds?: { width: number; height: number; depth?: number }
|
||||
}
|
||||
clusters?: Array<{
|
||||
id: string
|
||||
color: string
|
||||
label?: string
|
||||
size: number
|
||||
}>
|
||||
}
|
||||
|
||||
// === Enterprise Types (from neuralOptimized) ===
|
||||
|
||||
export interface ClusteringStrategy {
|
||||
type: 'sample' | 'hierarchical' | 'stream' | 'hybrid'
|
||||
sampleSize?: number
|
||||
maxClusters?: number
|
||||
minClusterSize?: number
|
||||
}
|
||||
|
||||
export interface LODConfig {
|
||||
levels: number
|
||||
itemsPerLevel: number[]
|
||||
zoomThresholds: number[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Neural API - Unified best-of-both implementation
|
||||
*/
|
||||
export class NeuralAPI {
|
||||
private brain: any // Brainy instance
|
||||
private distanceFn: (a: Vector, b: Vector) => number
|
||||
private similarityCache: Map<string, number> = new Map()
|
||||
private clusterCache: Map<string, any> = new Map() // Enhanced for enterprise
|
||||
private hierarchyCache: Map<string, SemanticHierarchy> = new Map()
|
||||
|
||||
constructor(brain: any) {
|
||||
this.brain = brain
|
||||
this.distanceFn = brain.distance || cosineDistance
|
||||
}
|
||||
|
||||
// ===== SMART USER-FRIENDLY API =====
|
||||
|
||||
/**
|
||||
* Calculate similarity between any two items (smart detection)
|
||||
*/
|
||||
async similar(a: any, b: any, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
// Auto-detect input types
|
||||
if (typeof a === 'string' && typeof b === 'string') {
|
||||
if (this.isId(a) && this.isId(b)) {
|
||||
return this.similarityById(a, b, options)
|
||||
} else {
|
||||
return this.similarityByText(a, b, options)
|
||||
}
|
||||
} else if (Array.isArray(a) && Array.isArray(b)) {
|
||||
return this.similarityByVector(a as Vector, b as Vector, options)
|
||||
}
|
||||
|
||||
// Handle mixed types
|
||||
return this.smartSimilarity(a, b, options)
|
||||
}
|
||||
|
||||
/**
|
||||
* Find semantic clusters (auto-detects best approach)
|
||||
* Now with enterprise performance!
|
||||
*/
|
||||
async clusters(input?: any): Promise<SemanticCluster[]> {
|
||||
// No input? Use enterprise fast clustering
|
||||
if (!input) {
|
||||
return this.clusterFast()
|
||||
}
|
||||
|
||||
// Array? Cluster these items (use large clustering for big arrays)
|
||||
if (Array.isArray(input)) {
|
||||
if (input.length > 1000) {
|
||||
return this.clusterLarge({ sampleSize: Math.min(input.length, 1000) })
|
||||
}
|
||||
return this.clusterItems(input)
|
||||
}
|
||||
|
||||
// String? Find clusters near this
|
||||
if (typeof input === 'string') {
|
||||
return this.clustersNear(input)
|
||||
}
|
||||
|
||||
// Object? Use as config with enterprise algorithms
|
||||
if (typeof input === 'object' && !Array.isArray(input)) {
|
||||
return this.clusterWithConfig(input as ClusterOptions)
|
||||
}
|
||||
|
||||
throw new Error('Invalid input for clustering')
|
||||
}
|
||||
|
||||
/**
|
||||
* Get semantic hierarchy for an item
|
||||
*/
|
||||
async hierarchy(id: string): Promise<SemanticHierarchy> {
|
||||
// Check cache first
|
||||
if (this.hierarchyCache.has(id)) {
|
||||
return this.hierarchyCache.get(id)!
|
||||
}
|
||||
|
||||
const item = await this.brain.get(id)
|
||||
if (!item) {
|
||||
throw new Error(`Item not found: ${id}`)
|
||||
}
|
||||
|
||||
// Find semantic relationships
|
||||
const hierarchy = await this.buildHierarchy(item)
|
||||
|
||||
// Cache result
|
||||
this.hierarchyCache.set(id, hierarchy)
|
||||
|
||||
return hierarchy
|
||||
}
|
||||
|
||||
/**
|
||||
* Find semantic neighbors for visualization
|
||||
*/
|
||||
async neighbors(id: string, options?: {
|
||||
radius?: number
|
||||
limit?: number
|
||||
includeEdges?: boolean
|
||||
}): Promise<NeighborGraph> {
|
||||
const radius = options?.radius ?? 0.3
|
||||
const limit = options?.limit ?? 50
|
||||
|
||||
// Search for nearby items
|
||||
const results = await this.brain.search(id, limit * 2)
|
||||
|
||||
// Filter by semantic radius
|
||||
const neighbors = results
|
||||
.filter((r: any) => r.similarity >= (1 - radius))
|
||||
.slice(0, limit)
|
||||
.map((r: any) => ({
|
||||
id: r.id,
|
||||
similarity: r.similarity,
|
||||
type: r.metadata?.type,
|
||||
connections: r.metadata?.connections?.size || 0
|
||||
}))
|
||||
|
||||
const graph: NeighborGraph = {
|
||||
center: id,
|
||||
neighbors
|
||||
}
|
||||
|
||||
// Add edges if requested
|
||||
if (options?.includeEdges) {
|
||||
graph.edges = await this.buildEdges(id, neighbors)
|
||||
}
|
||||
|
||||
return graph
|
||||
}
|
||||
|
||||
/**
|
||||
* Find semantic path between two items
|
||||
*/
|
||||
async semanticPath(fromId: string, toId: string, options?: {
|
||||
maxHops?: number
|
||||
algorithm?: 'breadth' | 'dijkstra'
|
||||
}): Promise<Array<{
|
||||
id: string
|
||||
similarity: number
|
||||
hop: number
|
||||
}>> {
|
||||
const maxHops = options?.maxHops ?? 5
|
||||
const algorithm = options?.algorithm ?? 'breadth'
|
||||
|
||||
if (algorithm === 'dijkstra') {
|
||||
return this.dijkstraPath(fromId, toId, maxHops)
|
||||
} else {
|
||||
return this.breadthFirstPath(fromId, toId, maxHops)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect semantic outliers
|
||||
*/
|
||||
async outliers(threshold: number = 0.3): Promise<string[]> {
|
||||
// Get all items
|
||||
const stats = this.brain.getStats()
|
||||
const totalItems = stats.entities.total
|
||||
|
||||
if (totalItems === 0) return []
|
||||
|
||||
// For large datasets, use sampling
|
||||
if (totalItems > 10000) {
|
||||
return this.outliersViaSampling(threshold, 1000)
|
||||
}
|
||||
|
||||
return this.outliersByDistance(threshold)
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate visualization data
|
||||
*/
|
||||
async visualize(options?: {
|
||||
maxNodes?: number
|
||||
dimensions?: 2 | 3
|
||||
algorithm?: 'force' | 'hierarchical' | 'radial'
|
||||
includeEdges?: boolean
|
||||
}): Promise<VisualizationData> {
|
||||
const maxNodes = options?.maxNodes ?? 100
|
||||
const dimensions = options?.dimensions ?? 2
|
||||
const algorithm = options?.algorithm ?? 'force'
|
||||
|
||||
// Get representative nodes
|
||||
const nodes = await this.getVisualizationNodes(maxNodes)
|
||||
|
||||
// Apply layout algorithm
|
||||
const positioned = await this.applyLayout(nodes, algorithm, dimensions)
|
||||
|
||||
// Build edges if requested
|
||||
const edges = options?.includeEdges !== false ?
|
||||
await this.buildVisualizationEdges(positioned) : []
|
||||
|
||||
// Detect optimal format
|
||||
const format = this.detectOptimalFormat(positioned, edges)
|
||||
|
||||
return {
|
||||
format,
|
||||
nodes: positioned,
|
||||
edges,
|
||||
layout: {
|
||||
dimensions,
|
||||
algorithm,
|
||||
bounds: this.calculateBounds(positioned, dimensions)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== ENTERPRISE PERFORMANCE ALGORITHMS =====
|
||||
|
||||
/**
|
||||
* Fast clustering using HNSW levels - O(n) instead of O(n²)
|
||||
*/
|
||||
async clusterFast(options: {
|
||||
level?: number
|
||||
maxClusters?: number
|
||||
} = {}): Promise<SemanticCluster[]> {
|
||||
const cacheKey = `hierarchical-${options.level}-${options.maxClusters}`
|
||||
if (this.clusterCache.has(cacheKey)) {
|
||||
return this.clusterCache.get(cacheKey)
|
||||
}
|
||||
|
||||
// Use HNSW's natural hierarchy - auto-select optimal level
|
||||
const level = options.level ?? await this.getOptimalClusteringLevel()
|
||||
const maxClusters = options.maxClusters ?? 100
|
||||
|
||||
// Get representative nodes from HNSW level
|
||||
const representatives = await this.getHNSWLevelNodes(level)
|
||||
|
||||
// Each representative is a natural cluster center
|
||||
const clusters = []
|
||||
for (const rep of representatives.slice(0, maxClusters)) {
|
||||
const members = await this.findClusterMembers(rep, level - 1)
|
||||
clusters.push({
|
||||
id: `cluster-${rep.id}`,
|
||||
centroid: rep.vector,
|
||||
center: rep,
|
||||
members: members.map(m => m.id),
|
||||
size: members.length,
|
||||
level,
|
||||
confidence: 0.8 + (members.length / 100) * 0.2 // Size-based confidence
|
||||
} as SemanticCluster)
|
||||
}
|
||||
|
||||
this.clusterCache.set(cacheKey, clusters)
|
||||
return clusters
|
||||
}
|
||||
|
||||
/**
|
||||
* Large-scale clustering for massive datasets (millions of items)
|
||||
*/
|
||||
async clusterLarge(options: {
|
||||
sampleSize?: number
|
||||
strategy?: 'random' | 'diverse' | 'recent'
|
||||
} = {}): Promise<SemanticCluster[]> {
|
||||
const sampleSize = options.sampleSize ?? 1000
|
||||
const strategy = options.strategy ?? 'diverse'
|
||||
|
||||
// Get representative sample
|
||||
const sample = await this.getSample(sampleSize, strategy)
|
||||
|
||||
// Cluster the sample (fast on small set)
|
||||
const sampleClusters = await this.performFastClustering(sample)
|
||||
|
||||
// Project clusters to full dataset
|
||||
return this.projectClustersToFullDataset(sampleClusters)
|
||||
}
|
||||
|
||||
/**
|
||||
* Streaming clustering for progressive refinement
|
||||
*/
|
||||
async* clusterStream(options: {
|
||||
batchSize?: number
|
||||
maxBatches?: number
|
||||
} = {}): AsyncGenerator<SemanticCluster[]> {
|
||||
const batchSize = options.batchSize ?? 1000
|
||||
const maxBatches = options.maxBatches ?? Infinity
|
||||
|
||||
let offset = 0
|
||||
let batchCount = 0
|
||||
let globalClusters: SemanticCluster[] = []
|
||||
|
||||
while (batchCount < maxBatches) {
|
||||
// Get next batch
|
||||
const batch = await this.getBatch(offset, batchSize)
|
||||
if (batch.length === 0) break
|
||||
|
||||
// Cluster this batch
|
||||
const batchClusters = await this.performFastClustering(batch)
|
||||
|
||||
// Merge with global clusters
|
||||
globalClusters = await this.mergeClusters(globalClusters, batchClusters)
|
||||
|
||||
// Yield current state
|
||||
yield globalClusters
|
||||
|
||||
offset += batchSize
|
||||
batchCount++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Level-of-detail for massive visualization
|
||||
*/
|
||||
async getLOD(zoomLevel: number, viewport?: {
|
||||
center: Vector
|
||||
radius: number
|
||||
}): Promise<any> {
|
||||
// Define LOD levels based on zoom
|
||||
const lodLevels = [
|
||||
{ zoom: 0, maxNodes: 50, clusterLevel: 3 },
|
||||
{ zoom: 1, maxNodes: 200, clusterLevel: 2 },
|
||||
{ zoom: 2, maxNodes: 1000, clusterLevel: 1 },
|
||||
{ zoom: 3, maxNodes: 5000, clusterLevel: 0 }
|
||||
]
|
||||
|
||||
const lod = lodLevels.find(l => zoomLevel <= l.zoom) || lodLevels[lodLevels.length - 1]
|
||||
|
||||
if (viewport) {
|
||||
return this.getViewportLOD(viewport, lod)
|
||||
} else {
|
||||
return this.getGlobalLOD(lod)
|
||||
}
|
||||
}
|
||||
|
||||
// ===== IMPLEMENTATION HELPERS =====
|
||||
|
||||
private isId(str: string): boolean {
|
||||
// Check if string looks like an ID (UUID pattern, etc.)
|
||||
return (str.length === 36 && str.includes('-')) || !!str.match(/^[a-f0-9]{24}$/)
|
||||
}
|
||||
|
||||
private async similarityById(idA: string, idB: string, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
const cacheKey = `${idA}-${idB}`
|
||||
if (this.similarityCache.has(cacheKey)) {
|
||||
return this.similarityCache.get(cacheKey)!
|
||||
}
|
||||
|
||||
// Get items
|
||||
const [itemA, itemB] = await Promise.all([
|
||||
this.brain.get(idA),
|
||||
this.brain.get(idB)
|
||||
])
|
||||
|
||||
if (!itemA || !itemB) {
|
||||
throw new Error('One or both items not found')
|
||||
}
|
||||
|
||||
// Calculate similarity
|
||||
const score = this.distanceFn(itemA.vector, itemB.vector)
|
||||
|
||||
this.similarityCache.set(cacheKey, score)
|
||||
|
||||
if (options?.explain) {
|
||||
return {
|
||||
score,
|
||||
method: 'cosine',
|
||||
confidence: 0.9,
|
||||
explanation: `Semantic similarity between ${idA} and ${idB}`
|
||||
}
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
private async similarityByText(textA: string, textB: string, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
// Generate embeddings
|
||||
const [vectorA, vectorB] = await Promise.all([
|
||||
this.brain.embed(textA),
|
||||
this.brain.embed(textB)
|
||||
])
|
||||
|
||||
return this.similarityByVector(vectorA, vectorB, options)
|
||||
}
|
||||
|
||||
private async similarityByVector(vectorA: Vector, vectorB: Vector, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
const score = this.distanceFn(vectorA, vectorB)
|
||||
|
||||
if (options?.explain) {
|
||||
return {
|
||||
score,
|
||||
method: options.method || 'cosine',
|
||||
confidence: 0.95,
|
||||
explanation: 'Direct vector similarity calculation'
|
||||
}
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
private async smartSimilarity(a: any, b: any, options?: SimilarityOptions): Promise<number | SimilarityResult> {
|
||||
// Convert both to vectors and compare
|
||||
const vectorA = await this.toVector(a)
|
||||
const vectorB = await this.toVector(b)
|
||||
|
||||
return this.similarityByVector(vectorA, vectorB, options)
|
||||
}
|
||||
|
||||
private async toVector(item: any): Promise<Vector> {
|
||||
if (Array.isArray(item)) return item
|
||||
if (typeof item === 'string') {
|
||||
if (this.isId(item)) {
|
||||
const found = await this.brain.get(item)
|
||||
return found?.vector || await this.brain.embed(item)
|
||||
}
|
||||
return await this.brain.embed(item)
|
||||
}
|
||||
if (typeof item === 'object' && item.vector) {
|
||||
return item.vector
|
||||
}
|
||||
// Convert object to string and embed
|
||||
return await this.brain.embed(JSON.stringify(item))
|
||||
}
|
||||
|
||||
// Enterprise clustering implementations
|
||||
private async getOptimalClusteringLevel(): Promise<number> {
|
||||
// Analyze dataset size and return optimal HNSW level
|
||||
const stats = this.brain.getStats()
|
||||
const itemCount = stats.entities.total
|
||||
|
||||
if (itemCount < 1000) return 0
|
||||
if (itemCount < 10000) return 1
|
||||
if (itemCount < 100000) return 2
|
||||
return 3
|
||||
}
|
||||
|
||||
private async getHNSWLevelNodes(level: number): Promise<any[]> {
|
||||
// Get nodes from specific HNSW level
|
||||
// For now, use search to get a representative sample
|
||||
const stats = this.brain.getStats()
|
||||
const sampleSize = Math.min(100, Math.floor(stats.entities.total / (level + 1)))
|
||||
|
||||
// Use search with a general query to get representative items
|
||||
const queryVector = await this.brain.embed('data information content')
|
||||
const allItems = await this.brain.search(queryVector, sampleSize * 2)
|
||||
return allItems.slice(0, sampleSize)
|
||||
}
|
||||
|
||||
private async findClusterMembers(center: any, level: number): Promise<any[]> {
|
||||
// Find all items that belong to this cluster
|
||||
const results = await this.brain.search(center.vector, 50)
|
||||
return results.filter((r: any) => r.similarity > 0.7)
|
||||
}
|
||||
|
||||
private async getSample(size: number, strategy: string): Promise<any[]> {
|
||||
// Use search to get a sample of items
|
||||
const stats = this.brain.getStats()
|
||||
const maxSize = Math.min(size * 3, stats.entities.total) // Get more than needed for sampling
|
||||
const queryVector = await this.brain.embed('sample data content')
|
||||
const allItems = await this.brain.search(queryVector, maxSize)
|
||||
|
||||
switch (strategy) {
|
||||
case 'random':
|
||||
return this.shuffleArray(allItems).slice(0, size)
|
||||
case 'diverse':
|
||||
return this.getDiverseSample(allItems, size)
|
||||
case 'recent':
|
||||
return allItems.slice(-size)
|
||||
default:
|
||||
return allItems.slice(0, size)
|
||||
}
|
||||
}
|
||||
|
||||
private shuffleArray(array: any[]): any[] {
|
||||
const shuffled = [...array]
|
||||
for (let i = shuffled.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]
|
||||
}
|
||||
return shuffled
|
||||
}
|
||||
|
||||
private async getDiverseSample(items: any[], size: number): Promise<any[]> {
|
||||
// Select diverse items using maximum distance sampling
|
||||
if (items.length <= size) return items
|
||||
|
||||
const sample = [items[0]] // Start with first item
|
||||
|
||||
for (let i = 1; i < size; i++) {
|
||||
let maxMinDistance = -1
|
||||
let bestItem = null
|
||||
|
||||
for (const candidate of items) {
|
||||
if (sample.includes(candidate)) continue
|
||||
|
||||
// Find minimum distance to existing sample
|
||||
let minDistance = Infinity
|
||||
for (const selected of sample) {
|
||||
const distance = this.distanceFn(candidate.vector, selected.vector)
|
||||
minDistance = Math.min(minDistance, distance)
|
||||
}
|
||||
|
||||
// Select item with maximum minimum distance
|
||||
if (minDistance > maxMinDistance) {
|
||||
maxMinDistance = minDistance
|
||||
bestItem = candidate
|
||||
}
|
||||
}
|
||||
|
||||
if (bestItem) sample.push(bestItem)
|
||||
}
|
||||
|
||||
return sample
|
||||
}
|
||||
|
||||
private async performFastClustering(items: any[]): Promise<SemanticCluster[]> {
|
||||
// Simple k-means clustering for the sample
|
||||
const k = Math.min(10, Math.floor(items.length / 3))
|
||||
if (k <= 1) {
|
||||
return [{
|
||||
id: 'cluster-0',
|
||||
centroid: items[0]?.vector || [],
|
||||
members: items.map(i => i.id),
|
||||
confidence: 1.0
|
||||
}]
|
||||
}
|
||||
|
||||
// Initialize centroids randomly
|
||||
const centroids = items.slice(0, k).map(item => item.vector)
|
||||
|
||||
// Run k-means iterations (simplified)
|
||||
for (let iter = 0; iter < 10; iter++) {
|
||||
const clusters: (typeof items)[] = Array(k).fill(null).map(() => [])
|
||||
|
||||
// Assign items to nearest centroid
|
||||
for (const item of items) {
|
||||
let bestCluster = 0
|
||||
let bestDistance = Infinity
|
||||
|
||||
for (let c = 0; c < k; c++) {
|
||||
const distance = this.distanceFn(item.vector, centroids[c])
|
||||
if (distance < bestDistance) {
|
||||
bestDistance = distance
|
||||
bestCluster = c
|
||||
}
|
||||
}
|
||||
|
||||
clusters[bestCluster].push(item)
|
||||
}
|
||||
|
||||
// Update centroids
|
||||
for (let c = 0; c < k; c++) {
|
||||
if (clusters[c].length > 0) {
|
||||
const newCentroid = this.calculateCentroid(clusters[c])
|
||||
centroids[c] = newCentroid
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to SemanticCluster format
|
||||
const result: SemanticCluster[] = []
|
||||
for (let c = 0; c < k; c++) {
|
||||
const members = items.filter(item => {
|
||||
let bestCluster = 0
|
||||
let bestDistance = Infinity
|
||||
|
||||
for (let cc = 0; cc < k; cc++) {
|
||||
const distance = this.distanceFn(item.vector, centroids[cc])
|
||||
if (distance < bestDistance) {
|
||||
bestDistance = distance
|
||||
bestCluster = cc
|
||||
}
|
||||
}
|
||||
|
||||
return bestCluster === c
|
||||
})
|
||||
|
||||
if (members.length > 0) {
|
||||
result.push({
|
||||
id: `cluster-${c}`,
|
||||
centroid: centroids[c],
|
||||
members: members.map(m => m.id),
|
||||
confidence: Math.min(0.9, members.length / items.length * 2)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private calculateCentroid(items: any[]): Vector {
|
||||
if (items.length === 0) return []
|
||||
|
||||
const dimensions = items[0].vector.length
|
||||
const centroid = new Array(dimensions).fill(0)
|
||||
|
||||
for (const item of items) {
|
||||
for (let d = 0; d < dimensions; d++) {
|
||||
centroid[d] += item.vector[d]
|
||||
}
|
||||
}
|
||||
|
||||
for (let d = 0; d < dimensions; d++) {
|
||||
centroid[d] /= items.length
|
||||
}
|
||||
|
||||
return centroid
|
||||
}
|
||||
|
||||
private async projectClustersToFullDataset(sampleClusters: SemanticCluster[]): Promise<SemanticCluster[]> {
|
||||
// Project sample clusters to full dataset
|
||||
const result: SemanticCluster[] = []
|
||||
|
||||
for (const cluster of sampleClusters) {
|
||||
// Find all items similar to this cluster's centroid
|
||||
const similar = await this.brain.search(cluster.centroid, 1000)
|
||||
const members = similar
|
||||
.filter((s: any) => s.similarity > 0.6)
|
||||
.map((s: any) => s.id)
|
||||
|
||||
result.push({
|
||||
...cluster,
|
||||
members,
|
||||
size: members.length
|
||||
})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private async mergeClusters(globalClusters: SemanticCluster[], batchClusters: SemanticCluster[]): Promise<SemanticCluster[]> {
|
||||
// Simple merge strategy - combine similar clusters
|
||||
const result = [...globalClusters]
|
||||
|
||||
for (const batchCluster of batchClusters) {
|
||||
let merged = false
|
||||
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
const similarity = this.distanceFn(result[i].centroid, batchCluster.centroid)
|
||||
|
||||
if (similarity > 0.8) {
|
||||
// Merge clusters
|
||||
const newMembers = [...new Set([...result[i].members, ...batchCluster.members])]
|
||||
result[i] = {
|
||||
...result[i],
|
||||
members: newMembers,
|
||||
size: newMembers.length,
|
||||
centroid: this.averageVectors(result[i].centroid, batchCluster.centroid)
|
||||
}
|
||||
merged = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (!merged) {
|
||||
result.push(batchCluster)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private averageVectors(v1: Vector, v2: Vector): Vector {
|
||||
const result = new Array(v1.length)
|
||||
for (let i = 0; i < v1.length; i++) {
|
||||
result[i] = (v1[i] + v2[i]) / 2
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
private async getBatch(offset: number, size: number): Promise<any[]> {
|
||||
// Get batch of items for streaming using search with offset
|
||||
const queryVector = await this.brain.embed('batch data content')
|
||||
const items = await this.brain.search(queryVector, size, { offset })
|
||||
return items
|
||||
}
|
||||
|
||||
// Additional methods needed for full compatibility...
|
||||
private async clusterAll(): Promise<SemanticCluster[]> {
|
||||
return this.clusterFast()
|
||||
}
|
||||
|
||||
private async clusterItems(items: any[]): Promise<SemanticCluster[]> {
|
||||
return this.performFastClustering(items)
|
||||
}
|
||||
|
||||
private async clustersNear(id: string): Promise<SemanticCluster[]> {
|
||||
const neighbors = await this.neighbors(id, { limit: 100 })
|
||||
return this.performFastClustering(neighbors.neighbors)
|
||||
}
|
||||
|
||||
private async clusterWithConfig(config: ClusterOptions): Promise<SemanticCluster[]> {
|
||||
switch (config.algorithm) {
|
||||
case 'hierarchical':
|
||||
return this.clusterFast(config)
|
||||
case 'sample':
|
||||
return this.clusterLarge(config)
|
||||
case 'stream':
|
||||
const generator = this.clusterStream(config)
|
||||
const results = []
|
||||
for await (const batch of generator) {
|
||||
results.push(...batch)
|
||||
}
|
||||
return results
|
||||
default:
|
||||
return this.clusterFast(config)
|
||||
}
|
||||
}
|
||||
|
||||
// Placeholder implementations for remaining methods
|
||||
private async buildHierarchy(item: any): Promise<SemanticHierarchy> {
|
||||
// Implementation for hierarchy building
|
||||
return {
|
||||
self: { id: item.id, vector: item.vector }
|
||||
}
|
||||
}
|
||||
|
||||
private async buildEdges(centerId: string, neighbors: any[]): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async dijkstraPath(from: string, to: string, maxHops: number): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async breadthFirstPath(from: string, to: string, maxHops: number): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async outliersViaSampling(threshold: number, sampleSize: number): Promise<string[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async outliersByDistance(threshold: number): Promise<string[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async getVisualizationNodes(maxNodes: number): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private async applyLayout(nodes: any[], algorithm: string, dimensions: number): Promise<any[]> {
|
||||
return nodes
|
||||
}
|
||||
|
||||
private async buildVisualizationEdges(nodes: any[]): Promise<any[]> {
|
||||
return []
|
||||
}
|
||||
|
||||
private detectOptimalFormat(nodes: any[], edges: any[]): 'force-directed' | 'hierarchical' | 'radial' {
|
||||
return 'force-directed'
|
||||
}
|
||||
|
||||
private calculateBounds(nodes: any[], dimensions: number): any {
|
||||
return { width: 100, height: 100 }
|
||||
}
|
||||
|
||||
private async getViewportLOD(viewport: any, lod: any): Promise<any> {
|
||||
// LOD visualization is an optional advanced feature
|
||||
// Return default view without LOD optimization
|
||||
console.warn('Viewport LOD optimization not available. Using standard view.')
|
||||
return { nodes: [], edges: [], optimized: false }
|
||||
}
|
||||
|
||||
private async getGlobalLOD(lod: any): Promise<any> {
|
||||
// LOD visualization is an optional advanced feature
|
||||
// Return default view without LOD optimization
|
||||
console.warn('Global LOD optimization not available. Using standard view.')
|
||||
return { nodes: [], edges: [], optimized: false }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,342 +0,0 @@
|
|||
/**
|
||||
* Neural API Type Definitions
|
||||
* Comprehensive interfaces for clustering, similarity, and analysis
|
||||
*/
|
||||
|
||||
export interface Vector {
|
||||
[index: number]: number
|
||||
length: number
|
||||
}
|
||||
|
||||
// ===== CORE CLUSTERING INTERFACES =====
|
||||
|
||||
export interface SemanticCluster {
|
||||
id: string
|
||||
centroid: Vector
|
||||
members: string[]
|
||||
size: number
|
||||
confidence: number
|
||||
label?: string
|
||||
metadata?: Record<string, any>
|
||||
cohesion?: number
|
||||
level?: number
|
||||
}
|
||||
|
||||
export interface ClusterEdge {
|
||||
id: string
|
||||
source: string
|
||||
target: string
|
||||
type: string
|
||||
weight?: number
|
||||
isInterCluster: boolean
|
||||
sourceCluster?: string
|
||||
targetCluster?: string
|
||||
}
|
||||
|
||||
export interface EnhancedSemanticCluster extends SemanticCluster {
|
||||
intraClusterEdges: ClusterEdge[]
|
||||
interClusterEdges: ClusterEdge[]
|
||||
relationshipSummary: {
|
||||
totalEdges: number
|
||||
intraClusterEdges: number
|
||||
interClusterEdges: number
|
||||
edgeTypes: Record<string, number>
|
||||
}
|
||||
}
|
||||
|
||||
export interface DomainCluster extends SemanticCluster {
|
||||
domain: string
|
||||
domainConfidence: number
|
||||
crossDomainMembers?: string[]
|
||||
}
|
||||
|
||||
export interface TemporalCluster extends SemanticCluster {
|
||||
timeWindow: TimeWindow
|
||||
trend?: 'increasing' | 'decreasing' | 'stable'
|
||||
temporal: {
|
||||
startTime: Date
|
||||
endTime: Date
|
||||
peakTime?: Date
|
||||
frequency?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface ExplainableCluster extends SemanticCluster {
|
||||
explanation: {
|
||||
primaryFeatures: string[]
|
||||
commonTerms: string[]
|
||||
reasoning: string
|
||||
confidence: number
|
||||
}
|
||||
subClusters?: ExplainableCluster[]
|
||||
}
|
||||
|
||||
export interface ConfidentCluster extends SemanticCluster {
|
||||
minConfidence: number
|
||||
uncertainMembers: string[]
|
||||
certainMembers: string[]
|
||||
}
|
||||
|
||||
// ===== CLUSTERING OPTIONS =====
|
||||
|
||||
export interface BaseClusteringOptions {
|
||||
maxClusters?: number
|
||||
minClusterSize?: number
|
||||
threshold?: number
|
||||
cacheResults?: boolean
|
||||
}
|
||||
|
||||
export interface ClusteringOptions extends BaseClusteringOptions {
|
||||
algorithm?: 'auto' | 'hierarchical' | 'kmeans' | 'dbscan' | 'sample' | 'semantic' | 'graph' | 'multimodal'
|
||||
sampleSize?: number
|
||||
strategy?: 'random' | 'diverse' | 'recent' | 'important'
|
||||
memoryLimit?: string // e.g., '512MB'
|
||||
includeOutliers?: boolean
|
||||
// K-means specific options
|
||||
maxIterations?: number
|
||||
tolerance?: number
|
||||
}
|
||||
|
||||
export interface DomainClusteringOptions extends BaseClusteringOptions {
|
||||
domainField?: string
|
||||
crossDomainThreshold?: number
|
||||
preserveDomainBoundaries?: boolean
|
||||
}
|
||||
|
||||
export interface TemporalClusteringOptions extends BaseClusteringOptions {
|
||||
timeField: string
|
||||
windows: TimeWindow[]
|
||||
overlapStrategy?: 'merge' | 'separate' | 'hierarchical'
|
||||
trendAnalysis?: boolean
|
||||
}
|
||||
|
||||
export interface StreamClusteringOptions extends BaseClusteringOptions {
|
||||
batchSize?: number
|
||||
updateInterval?: number
|
||||
adaptiveThreshold?: boolean
|
||||
decayFactor?: number // For aging old clusters
|
||||
}
|
||||
|
||||
// ===== SIMILARITY & NEIGHBORS =====
|
||||
|
||||
export interface SimilarityOptions {
|
||||
detailed?: boolean
|
||||
metric?: 'cosine' | 'euclidean' | 'manhattan' | 'jaccard'
|
||||
normalized?: boolean
|
||||
}
|
||||
|
||||
export interface SimilarityResult {
|
||||
score: number
|
||||
confidence: number
|
||||
explanation?: string
|
||||
metric?: string
|
||||
}
|
||||
|
||||
export interface NeighborOptions {
|
||||
limit?: number
|
||||
radius?: number
|
||||
minSimilarity?: number
|
||||
includeMetadata?: boolean
|
||||
sortBy?: 'similarity' | 'importance' | 'recency'
|
||||
}
|
||||
|
||||
export interface Neighbor {
|
||||
id: string
|
||||
similarity: number
|
||||
data?: any
|
||||
metadata?: Record<string, any>
|
||||
distance?: number
|
||||
}
|
||||
|
||||
export interface NeighborsResult {
|
||||
neighbors: Neighbor[]
|
||||
queryId: string
|
||||
totalFound: number
|
||||
averageSimilarity: number
|
||||
}
|
||||
|
||||
// ===== HIERARCHY & ANALYSIS =====
|
||||
|
||||
export interface SemanticHierarchy {
|
||||
self?: { id: string; vector?: Vector; metadata?: any }
|
||||
root?: { id: string; vector?: Vector; metadata?: any } | null
|
||||
levels?: any[]
|
||||
parent?: { id: string; similarity: number }
|
||||
children?: Array<{ id: string; similarity: number }>
|
||||
siblings?: Array<{ id: string; similarity: number }>
|
||||
level?: number
|
||||
depth?: number
|
||||
}
|
||||
|
||||
export interface HierarchyOptions {
|
||||
maxDepth?: number
|
||||
minSimilarity?: number
|
||||
includeMetadata?: boolean
|
||||
buildStrategy?: 'similarity' | 'metadata' | 'mixed'
|
||||
}
|
||||
|
||||
// ===== VISUALIZATION =====
|
||||
|
||||
export interface VisualizationOptions {
|
||||
maxNodes?: number
|
||||
dimensions?: 2 | 3
|
||||
algorithm?: 'force' | 'spring' | 'circular' | 'hierarchical'
|
||||
includeEdges?: boolean
|
||||
clusterColors?: boolean
|
||||
nodeSize?: 'uniform' | 'importance' | 'connections'
|
||||
}
|
||||
|
||||
export interface VisualizationNode {
|
||||
id: string
|
||||
x: number
|
||||
y: number
|
||||
z?: number
|
||||
cluster?: string
|
||||
size?: number
|
||||
color?: string
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface VisualizationEdge {
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
color?: string
|
||||
type?: string
|
||||
}
|
||||
|
||||
export interface VisualizationResult {
|
||||
nodes: VisualizationNode[]
|
||||
edges: VisualizationEdge[]
|
||||
clusters?: Array<{
|
||||
id: string
|
||||
color: string
|
||||
size: number
|
||||
label?: string
|
||||
}>
|
||||
metadata: {
|
||||
algorithm: string
|
||||
dimensions: number
|
||||
totalNodes: number
|
||||
totalEdges: number
|
||||
generatedAt: Date
|
||||
}
|
||||
}
|
||||
|
||||
// ===== UTILITY TYPES =====
|
||||
|
||||
export interface TimeWindow {
|
||||
start: Date
|
||||
end: Date
|
||||
label?: string
|
||||
weight?: number
|
||||
}
|
||||
|
||||
export interface ClusterFeedback {
|
||||
clusterId: string
|
||||
action: 'merge' | 'split' | 'relabel' | 'adjust'
|
||||
parameters?: Record<string, any>
|
||||
confidence?: number
|
||||
}
|
||||
|
||||
export interface OutlierOptions {
|
||||
threshold?: number
|
||||
method?: 'isolation' | 'statistical' | 'cluster-based'
|
||||
minNeighbors?: number
|
||||
includeReasons?: boolean
|
||||
}
|
||||
|
||||
export interface Outlier {
|
||||
id: string
|
||||
score: number
|
||||
reasons?: string[]
|
||||
nearestNeighbors?: Neighbor[]
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
// ===== PERFORMANCE & MONITORING =====
|
||||
|
||||
export interface PerformanceMetrics {
|
||||
executionTime: number
|
||||
memoryUsed: number
|
||||
itemsProcessed: number
|
||||
cacheHits: number
|
||||
cacheMisses: number
|
||||
algorithm: string
|
||||
}
|
||||
|
||||
export interface ClusteringResult<T = SemanticCluster> {
|
||||
clusters: T[]
|
||||
metrics: PerformanceMetrics
|
||||
metadata: {
|
||||
totalItems: number
|
||||
clustersFound: number
|
||||
averageClusterSize: number
|
||||
silhouetteScore?: number
|
||||
timestamp: Date
|
||||
// Additional clustering-specific metadata
|
||||
semanticTypes?: number
|
||||
hnswLevel?: number
|
||||
kValue?: number
|
||||
hasConverged?: boolean
|
||||
outlierCount?: number
|
||||
eps?: number
|
||||
minPts?: number
|
||||
averageModularity?: number
|
||||
fusionMethod?: string
|
||||
componentAlgorithms?: string[]
|
||||
sampleSize?: number
|
||||
samplingStrategy?: string
|
||||
}
|
||||
}
|
||||
|
||||
// ===== STREAMING =====
|
||||
|
||||
export interface StreamingBatch<T = SemanticCluster> {
|
||||
clusters: T[]
|
||||
batchNumber: number
|
||||
isComplete: boolean
|
||||
progress: {
|
||||
processed: number
|
||||
total: number
|
||||
percentage: number
|
||||
}
|
||||
metrics: PerformanceMetrics
|
||||
}
|
||||
|
||||
// ===== ERROR TYPES =====
|
||||
|
||||
export class NeuralAPIError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public code: string,
|
||||
public context?: Record<string, any>
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'NeuralAPIError'
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusteringError extends NeuralAPIError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'CLUSTERING_ERROR', context)
|
||||
}
|
||||
}
|
||||
|
||||
export class SimilarityError extends NeuralAPIError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'SIMILARITY_ERROR', context)
|
||||
}
|
||||
}
|
||||
|
||||
// ===== CONFIGURATION =====
|
||||
|
||||
export interface NeuralAPIConfig {
|
||||
cacheSize?: number
|
||||
defaultAlgorithm?: string
|
||||
similarityMetric?: 'cosine' | 'euclidean' | 'manhattan'
|
||||
performanceTracking?: boolean
|
||||
maxMemoryUsage?: string
|
||||
parallelProcessing?: boolean
|
||||
streamingBatchSize?: number
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue