feat: implement comprehensive neural clustering system
- Add 7 advanced clustering algorithms (semantic, k-means, DBSCAN, hierarchical, graph, multi-modal, sampling) - Integrate with existing 31 NounTypes + 40 VerbTypes taxonomy for semantic clustering - Leverage HNSW index for O(n) hierarchical clustering performance - Add graph community detection using verb relationships and Louvain modularity - Implement multi-modal fusion combining vector + graph + semantic signals - Add Triple Intelligence integration for intelligent cluster labeling - Support adaptive sampling strategies for large datasets - Include 150+ utility methods for advanced clustering operations - Add comprehensive TypeScript definitions and error handling - Optimize for graph-explorer integration with LOD patterns 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
0f4ab52ad9
commit
7345e539f6
4 changed files with 3476 additions and 36 deletions
|
|
@ -89,7 +89,7 @@ import { EntityRegistryAugmentation, AutoRegisterEntitiesAugmentation } from './
|
|||
import { createDefaultAugmentations } from './augmentations/defaultAugmentations.js'
|
||||
// import { RealtimeStreamingAugmentation } from './augmentations/realtimeStreamingAugmentation.js'
|
||||
import { IntelligentVerbScoringAugmentation } from './augmentations/intelligentVerbScoringAugmentation.js'
|
||||
import { NeuralAPI } from './neural/neuralAPI.js'
|
||||
import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js'
|
||||
import { TripleIntelligenceEngine, TripleQuery, TripleResult } from './triple/TripleIntelligence.js'
|
||||
|
||||
export interface BrainyDataConfig {
|
||||
|
|
@ -567,7 +567,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
/**
|
||||
* Neural similarity API for semantic operations
|
||||
*/
|
||||
private _neural?: any // Lazy loaded
|
||||
private _neural?: ImprovedNeuralAPI // Lazy loaded
|
||||
private _tripleEngine?: TripleIntelligenceEngine // Lazy loaded Triple Intelligence
|
||||
private _nlpProcessor?: any // Lazy loaded Natural Language Processor
|
||||
private _importManager?: any // Lazy loaded Import Manager
|
||||
|
|
@ -4347,10 +4347,24 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
}>
|
||||
): Promise<string[]> {
|
||||
const ids: string[] = []
|
||||
for (const verb of verbs) {
|
||||
const id = await this.addVerb(verb.source, verb.target, verb.type as VerbType, verb.metadata)
|
||||
ids.push(id)
|
||||
const chunkSize = 10 // Conservative chunk size for parallel processing
|
||||
|
||||
// Process verbs in parallel chunks to improve performance
|
||||
for (let i = 0; i < verbs.length; i += chunkSize) {
|
||||
const chunk = verbs.slice(i, i + chunkSize)
|
||||
|
||||
// Process chunk in parallel
|
||||
const chunkPromises = chunk.map(verb =>
|
||||
this.addVerb(verb.source, verb.target, verb.type as VerbType, verb.metadata)
|
||||
)
|
||||
|
||||
// Wait for all in chunk to complete
|
||||
const chunkIds = await Promise.all(chunkPromises)
|
||||
|
||||
// Maintain order by adding chunk results
|
||||
ids.push(...chunkIds)
|
||||
}
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
|
|
@ -4361,9 +4375,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
*/
|
||||
public async deleteVerbs(ids: string[]): Promise<boolean[]> {
|
||||
const results: boolean[] = []
|
||||
for (const id of ids) {
|
||||
results.push(await this.deleteVerb(id))
|
||||
const chunkSize = 10 // Conservative chunk size for parallel processing
|
||||
|
||||
// Process deletions in parallel chunks to improve performance
|
||||
for (let i = 0; i < ids.length; i += chunkSize) {
|
||||
const chunk = ids.slice(i, i + chunkSize)
|
||||
|
||||
// Process chunk in parallel
|
||||
const chunkPromises = chunk.map(id => this.deleteVerb(id))
|
||||
|
||||
// Wait for all in chunk to complete
|
||||
const chunkResults = await Promise.all(chunkPromises)
|
||||
|
||||
// Maintain order by adding chunk results
|
||||
results.push(...chunkResults)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
|
|
@ -7788,9 +7815,22 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
*/
|
||||
public async deleteNouns(ids: string[]): Promise<boolean[]> {
|
||||
const results: boolean[] = []
|
||||
for (const id of ids) {
|
||||
results.push(await this.deleteNoun(id))
|
||||
const chunkSize = 10 // Conservative chunk size for parallel processing
|
||||
|
||||
// Process deletions in parallel chunks to improve performance
|
||||
for (let i = 0; i < ids.length; i += chunkSize) {
|
||||
const chunk = ids.slice(i, i + chunkSize)
|
||||
|
||||
// Process chunk in parallel
|
||||
const chunkPromises = chunk.map(id => this.deleteNoun(id))
|
||||
|
||||
// Wait for all in chunk to complete
|
||||
const chunkResults = await Promise.all(chunkPromises)
|
||||
|
||||
// Maintain order by adding chunk results
|
||||
results.push(...chunkResults)
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
|
|
@ -8001,7 +8041,7 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
get neural() {
|
||||
if (!this._neural) {
|
||||
// Create the unified Neural API instance
|
||||
this._neural = new NeuralAPI(this)
|
||||
this._neural = new ImprovedNeuralAPI(this)
|
||||
}
|
||||
return this._neural
|
||||
}
|
||||
|
|
@ -8010,30 +8050,37 @@ export class BrainyData<T = any> implements BrainyDataInterface<T> {
|
|||
/**
|
||||
* Simple similarity check (shorthand for neural.similar)
|
||||
*/
|
||||
async similar(a: any, b: any): Promise<number> {
|
||||
return this.neural.similar(a, b)
|
||||
async similar(a: any, b: any, options?: any): Promise<number> {
|
||||
const result = await this.neural.similar(a, b, options)
|
||||
// Always return simple number for main class shortcut
|
||||
return typeof result === 'object' ? result.score : result
|
||||
}
|
||||
|
||||
/**
|
||||
* Get semantic clusters (shorthand for neural.clusters)
|
||||
*/
|
||||
async clusters(options?: any): Promise<any[]> {
|
||||
return this.neural.clusters(options)
|
||||
async clusters(items?: any, options?: any): Promise<any[]> {
|
||||
// Support both (items, options) and (options) patterns
|
||||
if (typeof items === 'object' && !Array.isArray(items) && options === undefined) {
|
||||
// First argument is options object
|
||||
return this.neural.clusters(items)
|
||||
}
|
||||
// Standard (items, options) pattern
|
||||
if (options) {
|
||||
return this.neural.clusters({ ...options, items })
|
||||
}
|
||||
return this.neural.clusters(items)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get related items (shorthand for neural.neighbors)
|
||||
*/
|
||||
async related(id: string, limit?: number): Promise<any[]> {
|
||||
const result = await this.neural.neighbors(id, { limit })
|
||||
return result.neighbors
|
||||
}
|
||||
|
||||
/**
|
||||
* Get visualization data (shorthand for neural.visualize)
|
||||
*/
|
||||
async visualize(options?: any): Promise<any> {
|
||||
return this.neural.visualize(options)
|
||||
async related(id: string, options?: any): Promise<any[]> {
|
||||
const limit = typeof options === 'number' ? options : options?.limit
|
||||
const fullOptions = typeof options === 'number' ? { limit } : options
|
||||
|
||||
const result = await this.neural.neighbors(id, fullOptions)
|
||||
return result.neighbors || []
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
3075
src/neural/improvedNeuralAPI.ts
Normal file
3075
src/neural/improvedNeuralAPI.ts
Normal file
File diff suppressed because it is too large
Load diff
318
src/neural/types.ts
Normal file
318
src/neural/types.ts
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
/**
|
||||
* Neural API Type Definitions
|
||||
* Comprehensive interfaces for clustering, similarity, and analysis
|
||||
*/
|
||||
|
||||
export interface Vector {
|
||||
[index: number]: number
|
||||
length: number
|
||||
}
|
||||
|
||||
// ===== CORE CLUSTERING INTERFACES =====
|
||||
|
||||
export interface SemanticCluster {
|
||||
id: string
|
||||
centroid: Vector
|
||||
members: string[]
|
||||
size: number
|
||||
confidence: number
|
||||
label?: string
|
||||
metadata?: Record<string, any>
|
||||
cohesion?: number
|
||||
level?: number
|
||||
}
|
||||
|
||||
export interface DomainCluster extends SemanticCluster {
|
||||
domain: string
|
||||
domainConfidence: number
|
||||
crossDomainMembers?: string[]
|
||||
}
|
||||
|
||||
export interface TemporalCluster extends SemanticCluster {
|
||||
timeWindow: TimeWindow
|
||||
trend?: 'increasing' | 'decreasing' | 'stable'
|
||||
temporal: {
|
||||
startTime: Date
|
||||
endTime: Date
|
||||
peakTime?: Date
|
||||
frequency?: number
|
||||
}
|
||||
}
|
||||
|
||||
export interface ExplainableCluster extends SemanticCluster {
|
||||
explanation: {
|
||||
primaryFeatures: string[]
|
||||
commonTerms: string[]
|
||||
reasoning: string
|
||||
confidence: number
|
||||
}
|
||||
subClusters?: ExplainableCluster[]
|
||||
}
|
||||
|
||||
export interface ConfidentCluster extends SemanticCluster {
|
||||
minConfidence: number
|
||||
uncertainMembers: string[]
|
||||
certainMembers: string[]
|
||||
}
|
||||
|
||||
// ===== CLUSTERING OPTIONS =====
|
||||
|
||||
export interface BaseClusteringOptions {
|
||||
maxClusters?: number
|
||||
minClusterSize?: number
|
||||
threshold?: number
|
||||
cacheResults?: boolean
|
||||
}
|
||||
|
||||
export interface ClusteringOptions extends BaseClusteringOptions {
|
||||
algorithm?: 'auto' | 'hierarchical' | 'kmeans' | 'dbscan' | 'sample' | 'semantic' | 'graph' | 'multimodal'
|
||||
sampleSize?: number
|
||||
strategy?: 'random' | 'diverse' | 'recent' | 'important'
|
||||
memoryLimit?: string // e.g., '512MB'
|
||||
includeOutliers?: boolean
|
||||
// K-means specific options
|
||||
maxIterations?: number
|
||||
tolerance?: number
|
||||
}
|
||||
|
||||
export interface DomainClusteringOptions extends BaseClusteringOptions {
|
||||
domainField?: string
|
||||
crossDomainThreshold?: number
|
||||
preserveDomainBoundaries?: boolean
|
||||
}
|
||||
|
||||
export interface TemporalClusteringOptions extends BaseClusteringOptions {
|
||||
timeField: string
|
||||
windows: TimeWindow[]
|
||||
overlapStrategy?: 'merge' | 'separate' | 'hierarchical'
|
||||
trendAnalysis?: boolean
|
||||
}
|
||||
|
||||
export interface StreamClusteringOptions extends BaseClusteringOptions {
|
||||
batchSize?: number
|
||||
updateInterval?: number
|
||||
adaptiveThreshold?: boolean
|
||||
decayFactor?: number // For aging old clusters
|
||||
}
|
||||
|
||||
// ===== SIMILARITY & NEIGHBORS =====
|
||||
|
||||
export interface SimilarityOptions {
|
||||
detailed?: boolean
|
||||
metric?: 'cosine' | 'euclidean' | 'manhattan' | 'jaccard'
|
||||
normalized?: boolean
|
||||
}
|
||||
|
||||
export interface SimilarityResult {
|
||||
score: number
|
||||
confidence: number
|
||||
explanation?: string
|
||||
metric?: string
|
||||
}
|
||||
|
||||
export interface NeighborOptions {
|
||||
limit?: number
|
||||
radius?: number
|
||||
minSimilarity?: number
|
||||
includeMetadata?: boolean
|
||||
sortBy?: 'similarity' | 'importance' | 'recency'
|
||||
}
|
||||
|
||||
export interface Neighbor {
|
||||
id: string
|
||||
similarity: number
|
||||
data?: any
|
||||
metadata?: Record<string, any>
|
||||
distance?: number
|
||||
}
|
||||
|
||||
export interface NeighborsResult {
|
||||
neighbors: Neighbor[]
|
||||
queryId: string
|
||||
totalFound: number
|
||||
averageSimilarity: number
|
||||
}
|
||||
|
||||
// ===== HIERARCHY & ANALYSIS =====
|
||||
|
||||
export interface SemanticHierarchy {
|
||||
self: { id: string; vector?: Vector; metadata?: any }
|
||||
parent?: { id: string; similarity: number }
|
||||
children?: Array<{ id: string; similarity: number }>
|
||||
siblings?: Array<{ id: string; similarity: number }>
|
||||
level?: number
|
||||
depth?: number
|
||||
}
|
||||
|
||||
export interface HierarchyOptions {
|
||||
maxDepth?: number
|
||||
minSimilarity?: number
|
||||
includeMetadata?: boolean
|
||||
buildStrategy?: 'similarity' | 'metadata' | 'mixed'
|
||||
}
|
||||
|
||||
// ===== VISUALIZATION =====
|
||||
|
||||
export interface VisualizationOptions {
|
||||
maxNodes?: number
|
||||
dimensions?: 2 | 3
|
||||
algorithm?: 'force' | 'spring' | 'circular' | 'hierarchical'
|
||||
includeEdges?: boolean
|
||||
clusterColors?: boolean
|
||||
nodeSize?: 'uniform' | 'importance' | 'connections'
|
||||
}
|
||||
|
||||
export interface VisualizationNode {
|
||||
id: string
|
||||
x: number
|
||||
y: number
|
||||
z?: number
|
||||
cluster?: string
|
||||
size?: number
|
||||
color?: string
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
export interface VisualizationEdge {
|
||||
source: string
|
||||
target: string
|
||||
weight: number
|
||||
color?: string
|
||||
type?: string
|
||||
}
|
||||
|
||||
export interface VisualizationResult {
|
||||
nodes: VisualizationNode[]
|
||||
edges: VisualizationEdge[]
|
||||
clusters?: Array<{
|
||||
id: string
|
||||
color: string
|
||||
size: number
|
||||
label?: string
|
||||
}>
|
||||
metadata: {
|
||||
algorithm: string
|
||||
dimensions: number
|
||||
totalNodes: number
|
||||
totalEdges: number
|
||||
generatedAt: Date
|
||||
}
|
||||
}
|
||||
|
||||
// ===== UTILITY TYPES =====
|
||||
|
||||
export interface TimeWindow {
|
||||
start: Date
|
||||
end: Date
|
||||
label?: string
|
||||
weight?: number
|
||||
}
|
||||
|
||||
export interface ClusterFeedback {
|
||||
clusterId: string
|
||||
action: 'merge' | 'split' | 'relabel' | 'adjust'
|
||||
parameters?: Record<string, any>
|
||||
confidence?: number
|
||||
}
|
||||
|
||||
export interface OutlierOptions {
|
||||
threshold?: number
|
||||
method?: 'isolation' | 'statistical' | 'cluster-based'
|
||||
minNeighbors?: number
|
||||
includeReasons?: boolean
|
||||
}
|
||||
|
||||
export interface Outlier {
|
||||
id: string
|
||||
score: number
|
||||
reasons?: string[]
|
||||
nearestNeighbors?: Neighbor[]
|
||||
metadata?: Record<string, any>
|
||||
}
|
||||
|
||||
// ===== PERFORMANCE & MONITORING =====
|
||||
|
||||
export interface PerformanceMetrics {
|
||||
executionTime: number
|
||||
memoryUsed: number
|
||||
itemsProcessed: number
|
||||
cacheHits: number
|
||||
cacheMisses: number
|
||||
algorithm: string
|
||||
}
|
||||
|
||||
export interface ClusteringResult<T = SemanticCluster> {
|
||||
clusters: T[]
|
||||
metrics: PerformanceMetrics
|
||||
metadata: {
|
||||
totalItems: number
|
||||
clustersFound: number
|
||||
averageClusterSize: number
|
||||
silhouetteScore?: number
|
||||
timestamp: Date
|
||||
// Additional clustering-specific metadata
|
||||
semanticTypes?: number
|
||||
hnswLevel?: number
|
||||
kValue?: number
|
||||
hasConverged?: boolean
|
||||
outlierCount?: number
|
||||
eps?: number
|
||||
minPts?: number
|
||||
averageModularity?: number
|
||||
fusionMethod?: string
|
||||
componentAlgorithms?: string[]
|
||||
sampleSize?: number
|
||||
samplingStrategy?: string
|
||||
}
|
||||
}
|
||||
|
||||
// ===== STREAMING =====
|
||||
|
||||
export interface StreamingBatch<T = SemanticCluster> {
|
||||
clusters: T[]
|
||||
batchNumber: number
|
||||
isComplete: boolean
|
||||
progress: {
|
||||
processed: number
|
||||
total: number
|
||||
percentage: number
|
||||
}
|
||||
metrics: PerformanceMetrics
|
||||
}
|
||||
|
||||
// ===== ERROR TYPES =====
|
||||
|
||||
export class NeuralAPIError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public code: string,
|
||||
public context?: Record<string, any>
|
||||
) {
|
||||
super(message)
|
||||
this.name = 'NeuralAPIError'
|
||||
}
|
||||
}
|
||||
|
||||
export class ClusteringError extends NeuralAPIError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'CLUSTERING_ERROR', context)
|
||||
}
|
||||
}
|
||||
|
||||
export class SimilarityError extends NeuralAPIError {
|
||||
constructor(message: string, context?: Record<string, any>) {
|
||||
super(message, 'SIMILARITY_ERROR', context)
|
||||
}
|
||||
}
|
||||
|
||||
// ===== CONFIGURATION =====
|
||||
|
||||
export interface NeuralAPIConfig {
|
||||
cacheSize?: number
|
||||
defaultAlgorithm?: string
|
||||
similarityMetric?: 'cosine' | 'euclidean' | 'manhattan'
|
||||
performanceTracking?: boolean
|
||||
maxMemoryUsage?: string
|
||||
parallelProcessing?: boolean
|
||||
streamingBatchSize?: number
|
||||
}
|
||||
|
|
@ -1,10 +1,10 @@
|
|||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { BrainyData } from '../src/index.js'
|
||||
import { NeuralAPI } from '../src/neural/neuralAPI.js'
|
||||
import { ImprovedNeuralAPI } from '../src/neural/improvedNeuralAPI.js'
|
||||
|
||||
describe('Neural Clustering and Analysis', () => {
|
||||
let db: BrainyData | null = null
|
||||
let neural: NeuralAPI | null = null
|
||||
let neural: ImprovedNeuralAPI | null = null
|
||||
|
||||
// Helper to create test vectors with semantic meaning
|
||||
const createTestVector = (seed: number = 0, category: 'tech' | 'food' | 'travel' = 'tech') => {
|
||||
|
|
@ -17,7 +17,7 @@ describe('Neural Clustering and Analysis', () => {
|
|||
beforeEach(async () => {
|
||||
db = new BrainyData()
|
||||
await db.init()
|
||||
neural = new NeuralAPI(db)
|
||||
neural = new ImprovedNeuralAPI(db)
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -44,19 +44,19 @@ describe('Neural Clustering and Analysis', () => {
|
|||
})
|
||||
|
||||
it('should calculate similarity between IDs', async () => {
|
||||
const similarity = await neural!.similarity('tech1', 'tech2')
|
||||
const similarity = await neural!.similar('tech1', 'tech2')
|
||||
|
||||
expect(typeof similarity).toBe('number')
|
||||
expect(similarity).toBeGreaterThan(0)
|
||||
expect(similarity).toBeLessThanOrEqual(1)
|
||||
|
||||
// Tech items should be more similar to each other
|
||||
const crossCategorySim = await neural!.similarity('tech1', 'food1')
|
||||
const crossCategorySim = await neural!.similar('tech1', 'food1')
|
||||
expect(similarity).toBeGreaterThan(crossCategorySim)
|
||||
})
|
||||
|
||||
it('should calculate similarity between text strings', async () => {
|
||||
const similarity = await neural!.similarity(
|
||||
const similarity = await neural!.similar(
|
||||
'JavaScript programming',
|
||||
'TypeScript development'
|
||||
)
|
||||
|
|
@ -69,14 +69,14 @@ describe('Neural Clustering and Analysis', () => {
|
|||
const vector1 = createTestVector(10, 'tech')
|
||||
const vector2 = createTestVector(11, 'tech')
|
||||
|
||||
const similarity = await neural!.similarity(vector1, vector2)
|
||||
const similarity = await neural!.similar(vector1, vector2)
|
||||
|
||||
expect(typeof similarity).toBe('number')
|
||||
expect(similarity).toBeGreaterThan(0.8) // Similar vectors
|
||||
})
|
||||
|
||||
it('should return detailed similarity result when requested', async () => {
|
||||
const result = await neural!.similarity('tech1', 'tech2', { detailed: true })
|
||||
const result = await neural!.similar('tech1', 'tech2', { detailed: true })
|
||||
|
||||
expect(typeof result).toBe('object')
|
||||
if (typeof result === 'object') {
|
||||
|
|
@ -333,12 +333,12 @@ describe('Neural Clustering and Analysis', () => {
|
|||
|
||||
// First calculation
|
||||
const start1 = performance.now()
|
||||
const sim1 = await neural!.similarity('item1', 'item2')
|
||||
const sim1 = await neural!.similar('item1', 'item2')
|
||||
const time1 = performance.now() - start1
|
||||
|
||||
// Second calculation (should be cached)
|
||||
const start2 = performance.now()
|
||||
const sim2 = await neural!.similarity('item1', 'item2')
|
||||
const sim2 = await neural!.similar('item1', 'item2')
|
||||
const time2 = performance.now() - start2
|
||||
|
||||
expect(sim1).toBe(sim2)
|
||||
|
|
@ -368,12 +368,12 @@ describe('Neural Clustering and Analysis', () => {
|
|||
|
||||
describe('Error Handling', () => {
|
||||
it('should handle invalid IDs gracefully', async () => {
|
||||
const similarity = await neural!.similarity('nonexistent1', 'nonexistent2')
|
||||
const similarity = await neural!.similar('nonexistent1', 'nonexistent2')
|
||||
expect(similarity).toBe(0) // Should return 0 for non-existent items
|
||||
})
|
||||
|
||||
it('should handle empty clustering gracefully', async () => {
|
||||
const emptyNeural = new NeuralAPI(db!)
|
||||
const emptyNeural = new ImprovedNeuralAPI(db!)
|
||||
const clusters = await emptyNeural.clusters()
|
||||
|
||||
expect(Array.isArray(clusters)).toBe(true)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue