David Snelling
8ff382ca3b
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
2630 lines
No EOL
112 KiB
JavaScript
2630 lines
No EOL
112 KiB
JavaScript
/**
|
|
* Improved Neural API - Clean, Consistent, Performant
|
|
*
|
|
* Public API Surface:
|
|
* - brain.neural.similar(a, b, options?) // Similarity calculation
|
|
* - brain.neural.clusters(items?, options?) // Semantic clustering
|
|
* - brain.neural.neighbors(id, options?) // K-nearest neighbors
|
|
* - brain.neural.hierarchy(id, options?) // Semantic hierarchy
|
|
* - brain.neural.outliers(options?) // Anomaly detection
|
|
* - brain.neural.visualize(options?) // Visualization data
|
|
*
|
|
* Advanced Clustering:
|
|
* - brain.neural.clusterByDomain(field, options?) // Domain-aware clustering
|
|
* - brain.neural.clusterByTime(field, windows, options?) // Temporal clustering
|
|
* - brain.neural.clusterStream(options?) // AsyncIterator for streaming
|
|
* - brain.neural.updateClusters(items, options?) // Incremental clustering
|
|
*
|
|
* Private methods are prefixed with _ and not exposed in public API
|
|
*/
|
|
import { cosineDistance, euclideanDistance } from '../utils/distance.js';
|
|
import { NeuralAPIError, ClusteringError, SimilarityError } from './types.js';
|
|
export class ImprovedNeuralAPI {
|
|
constructor(brain, config = {}) {
|
|
// Caching for performance
|
|
this.similarityCache = new Map();
|
|
this.clusterCache = new Map();
|
|
this.hierarchyCache = new Map();
|
|
this.neighborsCache = new Map();
|
|
// Performance tracking
|
|
this.performanceMetrics = new Map();
|
|
this.brain = brain;
|
|
this.config = {
|
|
cacheSize: 1000,
|
|
defaultAlgorithm: 'auto',
|
|
similarityMetric: 'cosine',
|
|
performanceTracking: true,
|
|
maxMemoryUsage: '1GB',
|
|
parallelProcessing: true,
|
|
streamingBatchSize: 100,
|
|
...config
|
|
};
|
|
this._initializeCleanupTimer();
|
|
}
|
|
// ===== PUBLIC API: SIMILARITY =====
|
|
/**
|
|
* Calculate similarity between any two items (auto-detection)
|
|
* Supports: IDs, text strings, vectors, or mixed types
|
|
*/
|
|
async similar(a, b, options = {}) {
|
|
const startTime = performance.now();
|
|
try {
|
|
// Create cache key
|
|
const cacheKey = this._createSimilarityKey(a, b, options);
|
|
if (this.similarityCache.has(cacheKey)) {
|
|
return this.similarityCache.get(cacheKey);
|
|
}
|
|
let result;
|
|
// Auto-detect input types and route accordingly
|
|
if (this._isId(a) && this._isId(b)) {
|
|
result = await this._similarityById(a, b, options);
|
|
}
|
|
else if (this._isVector(a) && this._isVector(b)) {
|
|
result = await this._similarityByVector(a, b, options);
|
|
}
|
|
else if (typeof a === 'string' && typeof b === 'string') {
|
|
result = await this._similarityByText(a, b, options);
|
|
}
|
|
else {
|
|
// Mixed types - convert to vectors
|
|
const vectorA = await this._convertToVector(a);
|
|
const vectorB = await this._convertToVector(b);
|
|
result = await this._similarityByVector(vectorA, vectorB, options);
|
|
}
|
|
// Cache result
|
|
this._cacheResult(cacheKey, result, this.similarityCache);
|
|
// Track performance
|
|
this._trackPerformance('similarity', startTime, 2, 'mixed');
|
|
return result;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new SimilarityError(`Failed to calculate similarity: ${errorMessage}`, {
|
|
inputA: typeof a === 'object' ? 'vector' : String(a).substring(0, 50),
|
|
inputB: typeof b === 'object' ? 'vector' : String(b).substring(0, 50),
|
|
options
|
|
});
|
|
}
|
|
}
|
|
// ===== PUBLIC API: CLUSTERING =====
|
|
/**
|
|
* Intelligent semantic clustering with auto-routing
|
|
* - No input: Cluster all data
|
|
* - Array: Cluster specific items
|
|
* - String: Find clusters near this item
|
|
* - Options object: Advanced configuration
|
|
*/
|
|
async clusters(input) {
|
|
const startTime = performance.now();
|
|
try {
|
|
let options = {};
|
|
let items;
|
|
// Parse input
|
|
if (!input) {
|
|
// Cluster all data
|
|
items = undefined;
|
|
options = { algorithm: 'auto' };
|
|
}
|
|
else if (Array.isArray(input)) {
|
|
// Cluster specific items
|
|
items = input;
|
|
options = { algorithm: 'auto' };
|
|
}
|
|
else if (typeof input === 'string') {
|
|
// Find clusters near this item
|
|
const nearbyResult = await this.neighbors(input, { limit: 100 });
|
|
items = nearbyResult.neighbors.map(n => n.id);
|
|
options = { algorithm: 'auto' };
|
|
}
|
|
else if (typeof input === 'object') {
|
|
// Configuration object
|
|
options = input;
|
|
items = undefined;
|
|
}
|
|
else {
|
|
throw new ClusteringError('Invalid input for clustering', { input });
|
|
}
|
|
// Check cache
|
|
const cacheKey = this._createClusteringKey(items, options);
|
|
if (this.clusterCache.has(cacheKey)) {
|
|
const cached = this.clusterCache.get(cacheKey);
|
|
return cached.clusters;
|
|
}
|
|
// Route to optimal algorithm
|
|
const result = await this._routeClusteringAlgorithm(items, options);
|
|
// Cache result
|
|
this._cacheResult(cacheKey, result, this.clusterCache);
|
|
// Track performance
|
|
this._trackPerformance('clustering', startTime, items?.length || 0, options.algorithm || 'auto');
|
|
return result.clusters;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new ClusteringError(`Failed to perform clustering: ${errorMessage}`, {
|
|
input: typeof input === 'object' ? JSON.stringify(input) : input,
|
|
});
|
|
}
|
|
}
|
|
/**
|
|
* Fast hierarchical clustering using HNSW levels
|
|
*/
|
|
async clusterFast(options = {}) {
|
|
const fullOptions = {
|
|
algorithm: 'hierarchical',
|
|
maxClusters: options.maxClusters,
|
|
...options
|
|
};
|
|
const result = await this._performHierarchicalClustering(undefined, fullOptions);
|
|
return result.clusters;
|
|
}
|
|
/**
|
|
* Large-scale clustering with intelligent sampling
|
|
*/
|
|
async clusterLarge(options = {}) {
|
|
const fullOptions = {
|
|
algorithm: 'auto',
|
|
sampleSize: options.sampleSize || 1000,
|
|
strategy: options.strategy || 'diverse',
|
|
...options
|
|
};
|
|
const result = await this._performSampledClustering(undefined, fullOptions);
|
|
return result.clusters;
|
|
}
|
|
// ===== PUBLIC API: ADVANCED CLUSTERING =====
|
|
/**
|
|
* Domain-aware clustering based on metadata fields
|
|
*/
|
|
async clusterByDomain(field, options = {}) {
|
|
const startTime = performance.now();
|
|
try {
|
|
// Get all items with the specified field
|
|
const items = await this._getItemsByField(field);
|
|
if (items.length === 0) {
|
|
return [];
|
|
}
|
|
// Group by domain values
|
|
const domainGroups = this._groupByDomain(items, field);
|
|
const domainClusters = [];
|
|
// Cluster within each domain
|
|
for (const [domain, domainItems] of domainGroups) {
|
|
const domainOptions = {
|
|
...options,
|
|
algorithm: 'auto',
|
|
maxClusters: Math.min(options.maxClusters || 10, Math.ceil(domainItems.length / 3))
|
|
};
|
|
const clusters = await this._performClustering(domainItems.map(item => item.id), domainOptions);
|
|
// Convert to domain clusters
|
|
for (const cluster of clusters.clusters) {
|
|
domainClusters.push({
|
|
...cluster,
|
|
domain,
|
|
domainConfidence: this._calculateDomainConfidence(cluster, domainItems),
|
|
crossDomainMembers: options.crossDomainThreshold
|
|
? await this._findCrossDomainMembers(cluster, options.crossDomainThreshold)
|
|
: undefined
|
|
});
|
|
}
|
|
}
|
|
// Handle cross-domain clustering if enabled
|
|
if (!options.preserveDomainBoundaries) {
|
|
const crossDomainClusters = await this._findCrossDomainClusters(domainClusters, options.crossDomainThreshold || 0.8);
|
|
domainClusters.push(...crossDomainClusters);
|
|
}
|
|
this._trackPerformance('domainClustering', startTime, items.length, field);
|
|
return domainClusters;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new ClusteringError(`Failed to cluster by domain: ${errorMessage}`, { field, options });
|
|
}
|
|
}
|
|
/**
|
|
* Temporal clustering based on time windows
|
|
*/
|
|
async clusterByTime(timeField, windows, options = { timeField, windows }) {
|
|
const startTime = performance.now();
|
|
try {
|
|
const temporalClusters = [];
|
|
for (const window of windows) {
|
|
// Get items in this time window
|
|
const windowItems = await this._getItemsByTimeWindow(timeField, window);
|
|
if (windowItems.length === 0)
|
|
continue;
|
|
// Cluster items in this window
|
|
const clusteringOptions = {
|
|
...options,
|
|
algorithm: 'auto'
|
|
};
|
|
const clusters = await this._performClustering(windowItems.map(item => item.id), clusteringOptions);
|
|
// Convert to temporal clusters
|
|
for (const cluster of clusters.clusters) {
|
|
const temporal = await this._calculateTemporalMetrics(cluster, windowItems, timeField);
|
|
temporalClusters.push({
|
|
...cluster,
|
|
timeWindow: window,
|
|
trend: temporal.trend,
|
|
temporal: temporal.metrics
|
|
});
|
|
}
|
|
}
|
|
// Handle overlapping windows
|
|
if (options.overlapStrategy === 'merge') {
|
|
return this._mergeOverlappingTemporalClusters(temporalClusters);
|
|
}
|
|
this._trackPerformance('temporalClustering', startTime, temporalClusters.length, 'temporal');
|
|
return temporalClusters;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new ClusteringError(`Failed to cluster by time: ${errorMessage}`, { timeField, windows, options });
|
|
}
|
|
}
|
|
/**
|
|
* Streaming clustering with real-time updates
|
|
*/
|
|
async *clusterStream(options = {}) {
|
|
const batchSize = options.batchSize || this.config.streamingBatchSize || 100;
|
|
let batchNumber = 0;
|
|
let processedCount = 0;
|
|
try {
|
|
// Get all items for processing
|
|
const allItems = await this._getAllItemIds();
|
|
const totalItems = allItems.length;
|
|
// Process in batches
|
|
for (let i = 0; i < allItems.length; i += batchSize) {
|
|
const startTime = performance.now();
|
|
const batch = allItems.slice(i, i + batchSize);
|
|
// Perform clustering on this batch
|
|
const result = await this._performClustering(batch, {
|
|
...options,
|
|
algorithm: 'auto',
|
|
cacheResults: false // Don't cache streaming results
|
|
});
|
|
processedCount += batch.length;
|
|
const isComplete = processedCount >= totalItems;
|
|
yield {
|
|
clusters: result.clusters,
|
|
batchNumber: ++batchNumber,
|
|
isComplete,
|
|
progress: {
|
|
processed: processedCount,
|
|
total: totalItems,
|
|
percentage: (processedCount / totalItems) * 100
|
|
},
|
|
metrics: {
|
|
...result.metrics,
|
|
executionTime: performance.now() - startTime
|
|
}
|
|
};
|
|
// Adaptive threshold adjustment
|
|
if (options.adaptiveThreshold && batchNumber > 1) {
|
|
options.threshold = this._adjustThresholdAdaptively(result.clusters, options.threshold);
|
|
}
|
|
}
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new ClusteringError(`Failed in streaming clustering: ${errorMessage}`, { options, batchNumber });
|
|
}
|
|
}
|
|
/**
|
|
* Incremental clustering - add new items to existing clusters
|
|
*/
|
|
async updateClusters(newItems, options = {}) {
|
|
const startTime = performance.now();
|
|
try {
|
|
// Get existing clusters
|
|
const existingClusters = await this.clusters({ ...options, algorithm: 'auto' });
|
|
// For each new item, find best cluster or create new one
|
|
const updatedClusters = [...existingClusters];
|
|
const unassignedItems = [];
|
|
for (const itemId of newItems) {
|
|
let bestCluster = null;
|
|
let bestSimilarity = 0;
|
|
// Find most similar existing cluster
|
|
for (const cluster of updatedClusters) {
|
|
const similarity = await this._calculateItemToClusterSimilarity(itemId, cluster);
|
|
if (similarity > bestSimilarity && similarity > (options.threshold || 0.6)) {
|
|
bestSimilarity = similarity;
|
|
bestCluster = cluster;
|
|
}
|
|
}
|
|
if (bestCluster) {
|
|
// Add to existing cluster
|
|
bestCluster.members.push(itemId);
|
|
bestCluster.size = bestCluster.members.length;
|
|
// Recalculate centroid
|
|
bestCluster.centroid = await this._recalculateClusterCentroid(bestCluster);
|
|
}
|
|
else {
|
|
// Item doesn't fit existing clusters
|
|
unassignedItems.push(itemId);
|
|
}
|
|
}
|
|
// Create new clusters for unassigned items
|
|
if (unassignedItems.length > 0) {
|
|
const newClusters = await this._performClustering(unassignedItems, options);
|
|
updatedClusters.push(...newClusters.clusters);
|
|
}
|
|
this._trackPerformance('incrementalClustering', startTime, newItems.length, 'incremental');
|
|
return updatedClusters;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new ClusteringError(`Failed to update clusters: ${errorMessage}`, { newItems, options });
|
|
}
|
|
}
|
|
/**
|
|
* Enhanced clustering with relationship analysis using verbs
|
|
* Returns clusters with intra-cluster and inter-cluster relationship information
|
|
*
|
|
* Scalable for millions of nodes - uses efficient batching and filtering
|
|
*/
|
|
async clustersWithRelationships(input, options) {
|
|
const startTime = performance.now();
|
|
const batchSize = options?.batchSize || 1000;
|
|
const maxRelationships = options?.maxRelationships || 10000;
|
|
let processedCount = 0;
|
|
try {
|
|
// Get basic clusters first
|
|
const basicClusters = await this.clusters(input);
|
|
if (basicClusters.length === 0) {
|
|
return [];
|
|
}
|
|
// Build member lookup for O(1) cluster membership checking
|
|
const memberToClusterMap = new Map();
|
|
const clusterMap = new Map();
|
|
for (const cluster of basicClusters) {
|
|
clusterMap.set(cluster.id, cluster);
|
|
for (const memberId of cluster.members) {
|
|
memberToClusterMap.set(memberId, cluster.id);
|
|
}
|
|
}
|
|
// Initialize cluster edge collections
|
|
const clusterEdges = new Map();
|
|
for (const cluster of basicClusters) {
|
|
clusterEdges.set(cluster.id, {
|
|
intra: [],
|
|
inter: [],
|
|
edgeTypes: {}
|
|
});
|
|
}
|
|
// Process verbs in batches to handle millions of relationships efficiently
|
|
let hasMoreVerbs = true;
|
|
let offset = 0;
|
|
while (hasMoreVerbs && processedCount < maxRelationships) {
|
|
// Get batch of verbs using proper pagination API
|
|
const verbResult = await this.brain.getVerbs({
|
|
pagination: {
|
|
offset: offset,
|
|
limit: batchSize
|
|
}
|
|
});
|
|
const verbBatch = verbResult.data;
|
|
if (verbBatch.length === 0) {
|
|
hasMoreVerbs = false;
|
|
break;
|
|
}
|
|
// Process this batch
|
|
for (const verb of verbBatch) {
|
|
if (processedCount >= maxRelationships)
|
|
break;
|
|
const sourceClusterId = memberToClusterMap.get(verb.sourceId);
|
|
const targetClusterId = memberToClusterMap.get(verb.targetId);
|
|
// Skip verbs that don't involve any clustered nodes
|
|
if (!sourceClusterId && !targetClusterId)
|
|
continue;
|
|
const edgeWeight = this._calculateEdgeWeight(verb);
|
|
const edgeType = verb.verb || verb.type || 'relationship';
|
|
if (sourceClusterId && targetClusterId) {
|
|
if (sourceClusterId === targetClusterId) {
|
|
// Intra-cluster relationship
|
|
const edges = clusterEdges.get(sourceClusterId);
|
|
edges.intra.push({
|
|
id: verb.id,
|
|
source: verb.sourceId,
|
|
target: verb.targetId,
|
|
type: edgeType,
|
|
weight: edgeWeight,
|
|
isInterCluster: false,
|
|
sourceCluster: sourceClusterId,
|
|
targetCluster: sourceClusterId
|
|
});
|
|
edges.edgeTypes[edgeType] = (edges.edgeTypes[edgeType] || 0) + 1;
|
|
}
|
|
else {
|
|
// Inter-cluster relationship
|
|
const sourceEdges = clusterEdges.get(sourceClusterId);
|
|
const targetEdges = clusterEdges.get(targetClusterId);
|
|
const edge = {
|
|
id: verb.id,
|
|
source: verb.sourceId,
|
|
target: verb.targetId,
|
|
type: edgeType,
|
|
weight: edgeWeight,
|
|
isInterCluster: true,
|
|
sourceCluster: sourceClusterId,
|
|
targetCluster: targetClusterId
|
|
};
|
|
sourceEdges.inter.push(edge);
|
|
// Don't duplicate - target cluster will see this as incoming
|
|
sourceEdges.edgeTypes[edgeType] = (sourceEdges.edgeTypes[edgeType] || 0) + 1;
|
|
}
|
|
}
|
|
else {
|
|
// One-way relationship to/from cluster
|
|
const clusterId = sourceClusterId || targetClusterId;
|
|
const edges = clusterEdges.get(clusterId);
|
|
edges.inter.push({
|
|
id: verb.id,
|
|
source: verb.sourceId,
|
|
target: verb.targetId,
|
|
type: edgeType,
|
|
weight: edgeWeight,
|
|
isInterCluster: true,
|
|
sourceCluster: sourceClusterId || 'external',
|
|
targetCluster: targetClusterId || 'external'
|
|
});
|
|
edges.edgeTypes[edgeType] = (edges.edgeTypes[edgeType] || 0) + 1;
|
|
}
|
|
processedCount++;
|
|
}
|
|
offset += batchSize;
|
|
// Memory management: if we have too many edges, break early
|
|
const totalEdges = Array.from(clusterEdges.values())
|
|
.reduce((sum, edges) => sum + edges.intra.length + edges.inter.length, 0);
|
|
if (totalEdges >= maxRelationships) {
|
|
console.warn(`Relationship analysis stopped at ${totalEdges} edges to maintain performance`);
|
|
break;
|
|
}
|
|
// Check if we got fewer verbs than batch size (end of data)
|
|
if (verbBatch.length < batchSize) {
|
|
hasMoreVerbs = false;
|
|
}
|
|
}
|
|
// Build enhanced clusters
|
|
const enhancedClusters = [];
|
|
for (const cluster of basicClusters) {
|
|
const edges = clusterEdges.get(cluster.id);
|
|
enhancedClusters.push({
|
|
...cluster,
|
|
intraClusterEdges: edges.intra,
|
|
interClusterEdges: edges.inter,
|
|
relationshipSummary: {
|
|
totalEdges: edges.intra.length + edges.inter.length,
|
|
intraClusterEdges: edges.intra.length,
|
|
interClusterEdges: edges.inter.length,
|
|
edgeTypes: edges.edgeTypes
|
|
}
|
|
});
|
|
}
|
|
this._trackPerformance('clustersWithRelationships', startTime, processedCount, 'enhanced-scalable');
|
|
return enhancedClusters;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new ClusteringError(`Failed to perform relationship-aware clustering: ${errorMessage}`, {
|
|
input: typeof input === 'object' ? JSON.stringify(input) : input,
|
|
processedCount: processedCount || 0
|
|
});
|
|
}
|
|
}
|
|
// ===== PUBLIC API: NEIGHBORS & HIERARCHY =====
|
|
/**
|
|
* Find K-nearest semantic neighbors
|
|
*/
|
|
async neighbors(id, options = {}) {
|
|
const startTime = performance.now();
|
|
try {
|
|
const cacheKey = `neighbors:${id}:${JSON.stringify(options)}`;
|
|
if (this.neighborsCache.has(cacheKey)) {
|
|
return this.neighborsCache.get(cacheKey);
|
|
}
|
|
const limit = options.limit || 10;
|
|
const minSimilarity = options.minSimilarity || 0.1;
|
|
// Use HNSW index for efficient neighbor search
|
|
const searchResults = await this.brain.search('', {
|
|
...options,
|
|
limit: limit * 2, // Get more than needed for filtering
|
|
metadata: options.includeMetadata ? {} : undefined
|
|
});
|
|
// Filter and sort neighbors
|
|
const neighbors = [];
|
|
for (const result of searchResults) {
|
|
if (result.id === id)
|
|
continue; // Skip self
|
|
const similarity = await this._calculateSimilarity(id, result.id);
|
|
if (similarity >= minSimilarity) {
|
|
neighbors.push({
|
|
id: result.id,
|
|
similarity,
|
|
data: result.content || result.data,
|
|
metadata: options.includeMetadata ? result.metadata : undefined,
|
|
distance: 1 - similarity
|
|
});
|
|
}
|
|
if (neighbors.length >= limit)
|
|
break;
|
|
}
|
|
// Sort by specified criteria
|
|
this._sortNeighbors(neighbors, options.sortBy || 'similarity');
|
|
const result = {
|
|
neighbors: neighbors.slice(0, limit),
|
|
queryId: id,
|
|
totalFound: neighbors.length,
|
|
averageSimilarity: neighbors.reduce((sum, n) => sum + n.similarity, 0) / neighbors.length
|
|
};
|
|
this._cacheResult(cacheKey, result, this.neighborsCache);
|
|
this._trackPerformance('neighbors', startTime, limit, 'knn');
|
|
return result;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new NeuralAPIError(`Failed to find neighbors: ${errorMessage}`, 'NEIGHBORS_ERROR', { id, options });
|
|
}
|
|
}
|
|
/**
|
|
* Build semantic hierarchy around an item
|
|
*/
|
|
async hierarchy(id, options = {}) {
|
|
const startTime = performance.now();
|
|
try {
|
|
const cacheKey = `hierarchy:${id}:${JSON.stringify(options)}`;
|
|
if (this.hierarchyCache.has(cacheKey)) {
|
|
return this.hierarchyCache.get(cacheKey);
|
|
}
|
|
// Get item data
|
|
const item = await this.brain.getNoun(id);
|
|
if (!item) {
|
|
throw new Error(`Item with ID ${id} not found`);
|
|
}
|
|
// Build hierarchy based on strategy
|
|
const hierarchy = await this._buildSemanticHierarchy(item, options);
|
|
this._cacheResult(cacheKey, hierarchy, this.hierarchyCache);
|
|
this._trackPerformance('hierarchy', startTime, 1, 'hierarchy');
|
|
return hierarchy;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new NeuralAPIError(`Failed to build hierarchy: ${errorMessage}`, 'HIERARCHY_ERROR', { id, options });
|
|
}
|
|
}
|
|
// ===== PUBLIC API: ANALYSIS =====
|
|
/**
|
|
* Detect outliers and anomalous items
|
|
*/
|
|
async outliers(options = {}) {
|
|
const startTime = performance.now();
|
|
try {
|
|
const threshold = options.threshold || 0.3;
|
|
const method = options.method || 'cluster-based';
|
|
let outliers = [];
|
|
switch (method) {
|
|
case 'isolation':
|
|
outliers = await this._detectOutliersIsolation(threshold, options);
|
|
break;
|
|
case 'statistical':
|
|
outliers = await this._detectOutliersStatistical(threshold, options);
|
|
break;
|
|
case 'cluster-based':
|
|
default:
|
|
outliers = await this._detectOutliersClusterBased(threshold, options);
|
|
break;
|
|
}
|
|
this._trackPerformance('outlierDetection', startTime, outliers.length, method);
|
|
return outliers;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new NeuralAPIError(`Failed to detect outliers: ${errorMessage}`, 'OUTLIER_ERROR', { options });
|
|
}
|
|
}
|
|
/**
|
|
* Generate visualization data for graph libraries
|
|
*/
|
|
async visualize(options = {}) {
|
|
const startTime = performance.now();
|
|
try {
|
|
const maxNodes = options.maxNodes || 100;
|
|
const dimensions = options.dimensions || 2;
|
|
const algorithm = options.algorithm || 'force';
|
|
// Get data for visualization
|
|
const nodes = await this._generateVisualizationNodes(maxNodes, options);
|
|
const edges = options.includeEdges ? await this._generateVisualizationEdges(nodes, options) : [];
|
|
const clusters = options.clusterColors ? await this._generateVisualizationClusters(nodes) : [];
|
|
// Apply layout algorithm
|
|
const positionedNodes = await this._applyLayoutAlgorithm(nodes, edges, algorithm, dimensions);
|
|
const result = {
|
|
nodes: positionedNodes,
|
|
edges,
|
|
clusters,
|
|
metadata: {
|
|
algorithm,
|
|
dimensions,
|
|
totalNodes: nodes.length,
|
|
totalEdges: edges.length,
|
|
generatedAt: new Date()
|
|
}
|
|
};
|
|
this._trackPerformance('visualization', startTime, nodes.length, algorithm);
|
|
return result;
|
|
}
|
|
catch (error) {
|
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw new NeuralAPIError(`Failed to generate visualization: ${errorMessage}`, 'VISUALIZATION_ERROR', { options });
|
|
}
|
|
}
|
|
// ===== PRIVATE IMPLEMENTATION METHODS =====
|
|
async _routeClusteringAlgorithm(items, options) {
|
|
const algorithm = options.algorithm || 'auto';
|
|
const itemCount = items?.length || await this._getTotalItemCount();
|
|
// Auto-select optimal algorithm based on data size and characteristics
|
|
if (algorithm === 'auto') {
|
|
// Intelligent algorithm selection based on data characteristics
|
|
const itemIds = items || await this._getAllItemIds();
|
|
const dataCharacteristics = await this._analyzeDataCharacteristics(itemIds);
|
|
const hasRichGraph = dataCharacteristics.graphDensity > 0.05;
|
|
const hasSemanticTypes = Object.keys(dataCharacteristics.typeDistribution).length > 3;
|
|
if (hasRichGraph && hasSemanticTypes) {
|
|
// Best of all worlds for rich semantic graphs
|
|
return this._performMultiModalClustering(items, { ...options, algorithm: 'multimodal' });
|
|
}
|
|
else if (hasRichGraph) {
|
|
// Strong relationship network - use graph clustering
|
|
return this._performGraphClustering(items, { ...options, algorithm: 'graph' });
|
|
}
|
|
else if (hasSemanticTypes) {
|
|
// Rich semantic taxonomy - use semantic clustering
|
|
return this._performSemanticClustering(items, { ...options, algorithm: 'semantic' });
|
|
}
|
|
else if (itemCount > 10000) {
|
|
// Large dataset - use sampling
|
|
return this._performSampledClustering(items, { ...options, algorithm: 'sample' });
|
|
}
|
|
else if (itemCount > 1000) {
|
|
// Medium dataset - use hierarchical HNSW
|
|
return this._performHierarchicalClustering(items, { ...options, algorithm: 'hierarchical' });
|
|
}
|
|
else {
|
|
// Small dataset - use k-means for quality
|
|
return this._performKMeansClustering(items, { ...options, algorithm: 'kmeans' });
|
|
}
|
|
}
|
|
// Use specified algorithm
|
|
switch (algorithm) {
|
|
case 'hierarchical':
|
|
return this._performHierarchicalClustering(items, options);
|
|
case 'semantic':
|
|
return this._performSemanticClustering(items, options);
|
|
case 'graph':
|
|
return this._performGraphClustering(items, options);
|
|
case 'multimodal':
|
|
return this._performMultiModalClustering(items, options);
|
|
case 'kmeans':
|
|
return this._performKMeansClustering(items, options);
|
|
case 'dbscan':
|
|
return this._performDBSCANClustering(items, options);
|
|
case 'sample':
|
|
return this._performSampledClustering(items, options);
|
|
default:
|
|
throw new ClusteringError(`Unsupported algorithm: ${algorithm}`);
|
|
}
|
|
}
|
|
async _performClustering(items, options) {
|
|
// This is the main clustering dispatcher - routes to specific algorithms
|
|
return this._routeClusteringAlgorithm(items, options);
|
|
}
|
|
// ===== REAL CLUSTERING IMPLEMENTATIONS =====
|
|
/**
|
|
* SEMANTIC-AWARE CLUSTERING: Uses existing NounType/VerbType taxonomy + HNSW
|
|
*/
|
|
async _performSemanticClustering(items, options) {
|
|
const startTime = performance.now();
|
|
// Get all items if not specified
|
|
const itemIds = items || await this._getAllItemIds();
|
|
if (itemIds.length === 0) {
|
|
return this._createEmptyResult(startTime, 'semantic');
|
|
}
|
|
// 1. Group items by semantic type (NounType) - O(n) operation
|
|
const itemsWithMetadata = await this._getItemsWithMetadata(itemIds);
|
|
const typeGroups = this._groupBySemanticType(itemsWithMetadata);
|
|
const allClusters = [];
|
|
// 2. Cluster within each semantic type using HNSW - parallel processing
|
|
const typeClusteringPromises = Array.from(typeGroups.entries()).map(async ([nounType, groupItems]) => {
|
|
if (groupItems.length < (options.minClusterSize || 2)) {
|
|
// Create single cluster for small groups
|
|
return [{
|
|
id: `semantic-${nounType}`,
|
|
centroid: await this._calculateGroupCentroid(groupItems),
|
|
members: groupItems.map(item => item.id),
|
|
size: groupItems.length,
|
|
confidence: 0.9, // High confidence for type-based clustering
|
|
label: `${nounType} cluster`,
|
|
metadata: { semanticType: nounType, clustering: 'semantic' }
|
|
}];
|
|
}
|
|
// Use HNSW hierarchical clustering within type
|
|
return this._clusterWithinSemanticType(groupItems, options);
|
|
});
|
|
const typeClusterResults = await Promise.all(typeClusteringPromises);
|
|
typeClusterResults.forEach(clusters => allClusters.push(...clusters));
|
|
// 3. Find cross-type relationships using existing verb connections
|
|
const crossTypeConnections = await this._findCrossTypeConnections(typeGroups, options);
|
|
// 4. Merge clusters that have strong cross-type relationships
|
|
const finalClusters = await this._mergeSemanticClusters(allClusters, crossTypeConnections);
|
|
return {
|
|
clusters: finalClusters.slice(0, options.maxClusters || finalClusters.length),
|
|
metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'semantic'),
|
|
metadata: {
|
|
totalItems: itemIds.length,
|
|
clustersFound: finalClusters.length,
|
|
averageClusterSize: finalClusters.reduce((sum, c) => sum + c.size, 0) / finalClusters.length || 0,
|
|
semanticTypes: Array.from(typeGroups.keys()).length,
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
/**
|
|
* HIERARCHICAL CLUSTERING: Uses existing HNSW levels for O(n) clustering
|
|
*/
|
|
async _performHierarchicalClustering(items, options) {
|
|
const startTime = performance.now();
|
|
const itemIds = items || await this._getAllItemIds();
|
|
if (itemIds.length === 0) {
|
|
return this._createEmptyResult(startTime, 'hierarchical');
|
|
}
|
|
// Use existing HNSW level structure for natural clustering
|
|
const level = options.level || this._getOptimalClusteringLevel(itemIds.length);
|
|
const maxClusters = options.maxClusters || Math.min(50, Math.ceil(itemIds.length / 20));
|
|
// Get HNSW level representatives - these are natural cluster centers
|
|
const levelNodes = await this._getHNSWLevelNodes(level);
|
|
const clusterCenters = levelNodes.slice(0, maxClusters);
|
|
const clusters = [];
|
|
// Create clusters around each level representative
|
|
for (let i = 0; i < clusterCenters.length; i++) {
|
|
const center = clusterCenters[i];
|
|
// Find items that belong to this cluster using HNSW neighbors
|
|
const members = await this._findClusterMembers(center, itemIds, 0.5);
|
|
if (members.length > 0) {
|
|
// Get actual node data for creating cluster
|
|
const memberData = await this._getItemsWithMetadata(members);
|
|
const centroid = await this._calculateCentroidFromItems(memberData);
|
|
clusters.push({
|
|
id: `hierarchical-${i}`,
|
|
centroid,
|
|
members,
|
|
size: members.length,
|
|
confidence: await this._calculateHierarchicalConfidence(members),
|
|
label: await this._generateClusterLabel(memberData, 'hierarchical'),
|
|
metadata: { level, clusterCenter: center, clustering: 'hierarchical' }
|
|
});
|
|
}
|
|
}
|
|
// Assign remaining items to nearest clusters
|
|
const assignedItems = new Set(clusters.flatMap(c => c.members));
|
|
const unassignedItems = itemIds.filter(id => !assignedItems.has(id));
|
|
if (unassignedItems.length > 0) {
|
|
await this._assignUnassignedItems(unassignedItems, clusters);
|
|
}
|
|
return {
|
|
clusters,
|
|
metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'hierarchical'),
|
|
metadata: {
|
|
totalItems: itemIds.length,
|
|
clustersFound: clusters.length,
|
|
averageClusterSize: clusters.reduce((sum, c) => sum + c.size, 0) / clusters.length || 0,
|
|
hnswLevel: level,
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
/**
|
|
* K-MEANS CLUSTERING: Real implementation using existing distance functions
|
|
*/
|
|
async _performKMeansClustering(items, options) {
|
|
const startTime = performance.now();
|
|
const itemIds = items || await this._getAllItemIds();
|
|
if (itemIds.length === 0) {
|
|
return this._createEmptyResult(startTime, 'kmeans');
|
|
}
|
|
// Get vectors for all items using existing infrastructure
|
|
const itemsWithVectors = await this._getItemsWithVectors(itemIds);
|
|
// Determine optimal k
|
|
const k = options.maxClusters || Math.min(Math.floor(Math.sqrt(itemsWithVectors.length / 2)), 50 // Maximum clusters for practical use
|
|
);
|
|
if (k <= 1) {
|
|
// Single cluster case
|
|
return {
|
|
clusters: [{
|
|
id: 'kmeans-single',
|
|
centroid: await this._calculateCentroidFromItems(itemsWithVectors),
|
|
members: itemIds,
|
|
size: itemIds.length,
|
|
confidence: 1.0,
|
|
label: 'Single cluster',
|
|
metadata: { clustering: 'kmeans', k: 1 }
|
|
}],
|
|
metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'kmeans'),
|
|
metadata: {
|
|
totalItems: itemIds.length,
|
|
clustersFound: 1,
|
|
averageClusterSize: itemIds.length,
|
|
kValue: 1,
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
// Initialize centroids using k-means++ for better convergence
|
|
const centroids = await this._initializeCentroidsKMeansPlusPlus(itemsWithVectors, k);
|
|
let assignments = new Array(itemsWithVectors.length).fill(0);
|
|
let hasConverged = false;
|
|
const maxIterations = options.maxIterations || 100;
|
|
const tolerance = options.tolerance || 1e-4;
|
|
// K-means iteration loop
|
|
for (let iteration = 0; iteration < maxIterations && !hasConverged; iteration++) {
|
|
// Assignment step: assign each point to nearest centroid
|
|
const newAssignments = await this._assignPointsToCentroids(itemsWithVectors, centroids);
|
|
// Update step: recalculate centroids
|
|
const newCentroids = await this._updateCentroids(itemsWithVectors, newAssignments, k);
|
|
// Check convergence: has assignment changed significantly?
|
|
const changeRate = this._calculateAssignmentChangeRate(assignments, newAssignments);
|
|
hasConverged = changeRate < tolerance;
|
|
assignments = newAssignments;
|
|
// Update centroids for next iteration
|
|
for (let i = 0; i < centroids.length; i++) {
|
|
centroids[i] = newCentroids[i];
|
|
}
|
|
}
|
|
// Create semantic clusters from k-means results
|
|
const clusters = [];
|
|
for (let clusterIndex = 0; clusterIndex < k; clusterIndex++) {
|
|
const clusterMembers = itemsWithVectors.filter((_, i) => assignments[i] === clusterIndex);
|
|
if (clusterMembers.length > 0) {
|
|
const memberIds = clusterMembers.map(item => item.id);
|
|
clusters.push({
|
|
id: `kmeans-${clusterIndex}`,
|
|
centroid: centroids[clusterIndex],
|
|
members: memberIds,
|
|
size: memberIds.length,
|
|
confidence: await this._calculateKMeansClusterConfidence(clusterMembers, centroids[clusterIndex]),
|
|
label: await this._generateClusterLabel(clusterMembers, 'kmeans'),
|
|
metadata: {
|
|
clustering: 'kmeans',
|
|
k,
|
|
clusterIndex,
|
|
convergenceIterations: maxIterations
|
|
}
|
|
});
|
|
}
|
|
}
|
|
return {
|
|
clusters,
|
|
metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'kmeans'),
|
|
metadata: {
|
|
totalItems: itemIds.length,
|
|
clustersFound: clusters.length,
|
|
averageClusterSize: clusters.reduce((sum, c) => sum + c.size, 0) / clusters.length || 0,
|
|
kValue: k,
|
|
hasConverged,
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
/**
|
|
* DBSCAN CLUSTERING: Density-based clustering with adaptive parameters using HNSW
|
|
*/
|
|
async _performDBSCANClustering(items, options) {
|
|
const startTime = performance.now();
|
|
const itemIds = items || await this._getAllItemIds();
|
|
if (itemIds.length === 0) {
|
|
return this._createEmptyResult(startTime, 'dbscan');
|
|
}
|
|
const itemsWithVectors = await this._getItemsWithVectors(itemIds);
|
|
// Adaptive parameter selection using HNSW neighbors
|
|
const minPts = options.minClusterSize || Math.max(4, Math.floor(Math.log2(itemsWithVectors.length)));
|
|
const eps = options.threshold || await this._estimateOptimalEps(itemsWithVectors, minPts);
|
|
// DBSCAN state tracking
|
|
const NOISE = -1;
|
|
const UNVISITED = 0;
|
|
const visited = new Map();
|
|
const clusterAssignments = new Map();
|
|
let currentClusterId = 1;
|
|
// Process each point
|
|
for (const item of itemsWithVectors) {
|
|
if (visited.get(item.id))
|
|
continue;
|
|
visited.set(item.id, true);
|
|
// Find neighbors using existing HNSW infrastructure for efficiency
|
|
const neighbors = await this._findNeighborsWithinEps(item, itemsWithVectors, eps);
|
|
if (neighbors.length < minPts) {
|
|
// Mark as noise (outlier)
|
|
clusterAssignments.set(item.id, NOISE);
|
|
}
|
|
else {
|
|
// Start new cluster
|
|
await this._expandCluster(item, neighbors, currentClusterId, eps, minPts, itemsWithVectors, visited, clusterAssignments);
|
|
currentClusterId++;
|
|
}
|
|
}
|
|
// Convert DBSCAN results to SemanticCluster format
|
|
const clusters = [];
|
|
const clusterGroups = new Map();
|
|
const outliers = [];
|
|
// Group items by cluster assignment
|
|
for (const [itemId, clusterId] of clusterAssignments) {
|
|
if (clusterId === NOISE) {
|
|
outliers.push(itemId);
|
|
}
|
|
else {
|
|
if (!clusterGroups.has(clusterId)) {
|
|
clusterGroups.set(clusterId, []);
|
|
}
|
|
clusterGroups.get(clusterId).push(itemId);
|
|
}
|
|
}
|
|
// Create SemanticCluster objects
|
|
for (const [clusterId, memberIds] of clusterGroups) {
|
|
if (memberIds.length > 0) {
|
|
const members = itemsWithVectors.filter(item => memberIds.includes(item.id));
|
|
clusters.push({
|
|
id: `dbscan-${clusterId}`,
|
|
centroid: await this._calculateCentroidFromItems(members),
|
|
members: memberIds,
|
|
size: memberIds.length,
|
|
confidence: await this._calculateDBSCANClusterConfidence(members, eps),
|
|
label: await this._generateClusterLabel(members, 'dbscan'),
|
|
metadata: {
|
|
clustering: 'dbscan',
|
|
clusterId,
|
|
eps,
|
|
minPts,
|
|
isDensityBased: true
|
|
}
|
|
});
|
|
}
|
|
}
|
|
// Handle outliers - optionally create outlier cluster or assign to nearest
|
|
if (outliers.length > 0 && options.includeOutliers) {
|
|
const outlierMembers = itemsWithVectors.filter(item => outliers.includes(item.id));
|
|
clusters.push({
|
|
id: 'dbscan-outliers',
|
|
centroid: await this._calculateCentroidFromItems(outlierMembers),
|
|
members: outliers,
|
|
size: outliers.length,
|
|
confidence: 0.1, // Low confidence for outliers
|
|
label: 'Outliers',
|
|
metadata: {
|
|
clustering: 'dbscan',
|
|
isOutlierCluster: true,
|
|
eps,
|
|
minPts
|
|
}
|
|
});
|
|
}
|
|
return {
|
|
clusters,
|
|
metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'dbscan'),
|
|
metadata: {
|
|
totalItems: itemIds.length,
|
|
clustersFound: clusters.length,
|
|
averageClusterSize: clusters.reduce((sum, c) => sum + c.size, 0) / clusters.length || 0,
|
|
outlierCount: outliers.length,
|
|
eps,
|
|
minPts,
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
/**
|
|
* GRAPH COMMUNITY DETECTION: Uses existing verb relationships for clustering
|
|
*/
|
|
async _performGraphClustering(items, options) {
|
|
const startTime = performance.now();
|
|
const itemIds = items || await this._getAllItemIds();
|
|
if (itemIds.length === 0) {
|
|
return this._createEmptyResult(startTime, 'graph');
|
|
}
|
|
// Build graph from existing verb relationships
|
|
const graph = await this._buildGraphFromVerbs(itemIds, options);
|
|
// Detect communities using modularity optimization
|
|
const communities = await this._detectCommunities(graph, options);
|
|
// Enhance communities with vector similarity for boundary refinement
|
|
const refinedCommunities = await this._refineCommunitiesWithVectors(communities, options);
|
|
// Convert to SemanticCluster format with Triple Intelligence labeling
|
|
const clusters = [];
|
|
for (let i = 0; i < refinedCommunities.length; i++) {
|
|
const community = refinedCommunities[i];
|
|
if (community.members.length > 0) {
|
|
const members = await this._getItemsWithMetadata(community.members);
|
|
// Use Triple Intelligence for intelligent cluster labeling
|
|
const clusterLabel = await this._generateIntelligentClusterLabel(members, 'graph');
|
|
const clusterCentroid = await this._calculateCentroidFromItems(members);
|
|
clusters.push({
|
|
id: `graph-${i}`,
|
|
centroid: clusterCentroid,
|
|
members: community.members,
|
|
size: community.members.length,
|
|
confidence: community.modularity || 0.7,
|
|
label: clusterLabel,
|
|
metadata: {
|
|
clustering: 'graph',
|
|
communityId: i,
|
|
modularity: community.modularity,
|
|
graphDensity: community.density,
|
|
strongestConnections: community.strongestConnections
|
|
}
|
|
});
|
|
}
|
|
}
|
|
return {
|
|
clusters,
|
|
metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'graph'),
|
|
metadata: {
|
|
totalItems: itemIds.length,
|
|
clustersFound: clusters.length,
|
|
averageClusterSize: clusters.reduce((sum, c) => sum + c.size, 0) / clusters.length || 0,
|
|
averageModularity: clusters.reduce((sum, c) => sum + (c.metadata?.modularity || 0), 0) / clusters.length || 0,
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
/**
|
|
* MULTI-MODAL FUSION: Combines vector + graph + semantic + Triple Intelligence
|
|
*/
|
|
async _performMultiModalClustering(items, options) {
|
|
const startTime = performance.now();
|
|
const itemIds = items || await this._getAllItemIds();
|
|
if (itemIds.length === 0) {
|
|
return this._createEmptyResult(startTime, 'multimodal');
|
|
}
|
|
// Run multiple clustering algorithms in parallel
|
|
const [vectorClusters, graphClusters, semanticClusters] = await Promise.all([
|
|
this._performHierarchicalClustering(itemIds, { ...options, algorithm: 'hierarchical' }),
|
|
this._performGraphClustering(itemIds, { ...options, algorithm: 'graph' }),
|
|
this._performSemanticClustering(itemIds, { ...options, algorithm: 'semantic' })
|
|
]);
|
|
// Fuse results using intelligent consensus with Triple Intelligence
|
|
const fusedClusters = await this._fuseClusteringResultsWithTripleIntelligence([vectorClusters.clusters, graphClusters.clusters, semanticClusters.clusters], options);
|
|
return {
|
|
clusters: fusedClusters,
|
|
metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'multimodal'),
|
|
metadata: {
|
|
totalItems: itemIds.length,
|
|
clustersFound: fusedClusters.length,
|
|
averageClusterSize: fusedClusters.reduce((sum, c) => sum + c.size, 0) / fusedClusters.length || 0,
|
|
fusionMethod: 'triple_intelligence_consensus',
|
|
componentAlgorithms: ['hierarchical', 'graph', 'semantic'],
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
/**
|
|
* SAMPLED CLUSTERING: For very large datasets using intelligent sampling
|
|
*/
|
|
async _performSampledClustering(items, options) {
|
|
const startTime = performance.now();
|
|
const itemIds = items || await this._getAllItemIds();
|
|
if (itemIds.length === 0) {
|
|
return this._createEmptyResult(startTime, 'sampled');
|
|
}
|
|
const sampleSize = Math.min(options.sampleSize || 1000, itemIds.length);
|
|
const strategy = options.strategy || 'diverse';
|
|
// Intelligent sampling using existing infrastructure
|
|
const sample = await this._getSampleUsingStrategy(itemIds, sampleSize, strategy);
|
|
// Cluster the sample using the best algorithm for the sample size
|
|
const sampleResult = await this._performHierarchicalClustering(sample, {
|
|
...options,
|
|
maxClusters: Math.min(options.maxClusters || 50, Math.ceil(sample.length / 10))
|
|
});
|
|
// Project clusters back to full dataset using HNSW neighbors
|
|
const projectedClusters = await this._projectClustersToFullDataset(sampleResult.clusters, itemIds, sample);
|
|
return {
|
|
clusters: projectedClusters,
|
|
metrics: this._createPerformanceMetrics(startTime, itemIds.length, 'sampled'),
|
|
metadata: {
|
|
totalItems: itemIds.length,
|
|
sampleSize: sample.length,
|
|
samplingStrategy: strategy,
|
|
clustersFound: projectedClusters.length,
|
|
averageClusterSize: projectedClusters.reduce((sum, c) => sum + c.size, 0) / projectedClusters.length || 0,
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
// Similarity implementation methods
|
|
async _similarityById(id1, id2, options) {
|
|
// Get vectors for both items
|
|
const item1 = await this.brain.getNoun(id1);
|
|
const item2 = await this.brain.getNoun(id2);
|
|
if (!item1 || !item2) {
|
|
return 0;
|
|
}
|
|
return this._similarityByVector(item1.vector, item2.vector, options);
|
|
}
|
|
async _similarityByVector(v1, v2, options) {
|
|
const metric = options.metric || this.config.similarityMetric || 'cosine';
|
|
let score = 0;
|
|
switch (metric) {
|
|
case 'cosine':
|
|
score = 1 - cosineDistance(v1, v2);
|
|
break;
|
|
case 'euclidean':
|
|
score = 1 / (1 + euclideanDistance(v1, v2));
|
|
break;
|
|
case 'manhattan':
|
|
score = 1 / (1 + this._manhattanDistance(v1, v2));
|
|
break;
|
|
default:
|
|
score = 1 - cosineDistance(v1, v2);
|
|
}
|
|
if (options.detailed) {
|
|
return {
|
|
score: options.normalized !== false ? Math.max(0, Math.min(1, score)) : score,
|
|
confidence: this._calculateConfidence(score, v1, v2),
|
|
explanation: this._generateSimilarityExplanation(score, metric),
|
|
metric
|
|
};
|
|
}
|
|
return options.normalized !== false ? Math.max(0, Math.min(1, score)) : score;
|
|
}
|
|
async _similarityByText(text1, text2, options) {
|
|
// Convert text to vectors using brain's embedding function
|
|
const vector1 = await this.brain.embed(text1);
|
|
const vector2 = await this.brain.embed(text2);
|
|
return this._similarityByVector(vector1, vector2, options);
|
|
}
|
|
// Utility methods for internal operations
|
|
_isId(value) {
|
|
return typeof value === 'string' &&
|
|
(value.length === 36 && value.includes('-')) || // UUID-like
|
|
(value.length > 10 && !value.includes(' ')); // ID-like string
|
|
}
|
|
_isVector(value) {
|
|
return Array.isArray(value) &&
|
|
value.length > 0 &&
|
|
typeof value[0] === 'number';
|
|
}
|
|
async _convertToVector(input) {
|
|
if (this._isVector(input)) {
|
|
return input;
|
|
}
|
|
else if (this._isId(input)) {
|
|
const item = await this.brain.getNoun(input);
|
|
return item?.vector || [];
|
|
}
|
|
else if (typeof input === 'string') {
|
|
return await this.brain.embed(input);
|
|
}
|
|
else {
|
|
throw new Error(`Cannot convert input to vector: ${typeof input}`);
|
|
}
|
|
}
|
|
_createSimilarityKey(a, b, options) {
|
|
const aKey = typeof a === 'object' ? JSON.stringify(a).substring(0, 50) : String(a);
|
|
const bKey = typeof b === 'object' ? JSON.stringify(b).substring(0, 50) : String(b);
|
|
return `${aKey}|${bKey}|${JSON.stringify(options)}`;
|
|
}
|
|
_createClusteringKey(items, options) {
|
|
const itemsKey = items ? [...items].sort().join(',') : 'all';
|
|
return `clustering:${itemsKey}:${JSON.stringify(options)}`;
|
|
}
|
|
_cacheResult(key, result, cache) {
|
|
if (cache.size >= (this.config.cacheSize || 1000)) {
|
|
// Remove oldest entries (simple LRU)
|
|
const firstKey = cache.keys().next().value;
|
|
if (firstKey)
|
|
cache.delete(firstKey);
|
|
}
|
|
cache.set(key, result);
|
|
}
|
|
_trackPerformance(operation, startTime, itemCount, algorithm) {
|
|
if (!this.config.performanceTracking)
|
|
return;
|
|
const metrics = {
|
|
executionTime: performance.now() - startTime,
|
|
memoryUsed: 0, // Would implement actual memory tracking
|
|
itemsProcessed: itemCount,
|
|
cacheHits: 0, // Would track actual cache hits
|
|
cacheMisses: 0, // Would track actual cache misses
|
|
algorithm
|
|
};
|
|
if (!this.performanceMetrics.has(operation)) {
|
|
this.performanceMetrics.set(operation, []);
|
|
}
|
|
this.performanceMetrics.get(operation).push(metrics);
|
|
}
|
|
_createPerformanceMetrics(startTime, itemCount, algorithm) {
|
|
return {
|
|
executionTime: performance.now() - startTime,
|
|
memoryUsed: 0,
|
|
itemsProcessed: itemCount,
|
|
cacheHits: 0,
|
|
cacheMisses: 0,
|
|
algorithm
|
|
};
|
|
}
|
|
_initializeCleanupTimer() {
|
|
// Periodically clean up caches to prevent memory leaks
|
|
setInterval(() => {
|
|
if (this.similarityCache.size > (this.config.cacheSize || 1000)) {
|
|
this.similarityCache.clear();
|
|
}
|
|
if (this.clusterCache.size > (this.config.cacheSize || 1000)) {
|
|
this.clusterCache.clear();
|
|
}
|
|
if (this.hierarchyCache.size > (this.config.cacheSize || 1000)) {
|
|
this.hierarchyCache.clear();
|
|
}
|
|
if (this.neighborsCache.size > (this.config.cacheSize || 1000)) {
|
|
this.neighborsCache.clear();
|
|
}
|
|
}, 300000); // Clean every 5 minutes
|
|
}
|
|
// ===== GRAPH COMMUNITY DETECTION UTILITIES =====
|
|
/**
|
|
* Build graph structure from existing verb relationships
|
|
*/
|
|
async _buildGraphFromVerbs(itemIds, options) {
|
|
const nodes = new Set(itemIds);
|
|
const edges = new Map();
|
|
const verbWeights = new Map();
|
|
// Initialize verb relationship weights
|
|
const relationshipWeights = {
|
|
'creates': 1.0,
|
|
'partOf': 0.9,
|
|
'contains': 0.9,
|
|
'relatedTo': 0.7,
|
|
'references': 0.6,
|
|
'causes': 0.8,
|
|
'dependsOn': 0.8,
|
|
'memberOf': 0.9,
|
|
'worksWith': 0.7,
|
|
'communicates': 0.6
|
|
};
|
|
// Get all verbs connecting the items
|
|
for (const sourceId of itemIds) {
|
|
const sourceVerbs = await this.brain.getVerbsForNoun(sourceId);
|
|
for (const verb of sourceVerbs) {
|
|
const targetId = verb.target;
|
|
if (nodes.has(targetId) && sourceId !== targetId) {
|
|
// Initialize edge map if needed
|
|
if (!edges.has(sourceId)) {
|
|
edges.set(sourceId, new Map());
|
|
}
|
|
// Calculate edge weight from verb type and metadata
|
|
const verbType = verb.verb;
|
|
const baseWeight = relationshipWeights[verbType] || 0.5;
|
|
const confidenceWeight = verb.confidence || 1.0;
|
|
const weight = baseWeight * confidenceWeight;
|
|
// Add or strengthen edge
|
|
const currentWeight = edges.get(sourceId)?.get(targetId) || 0;
|
|
edges.get(sourceId).set(targetId, Math.min(currentWeight + weight, 1.0));
|
|
// Make graph undirected by adding reverse edge
|
|
if (!edges.has(targetId)) {
|
|
edges.set(targetId, new Map());
|
|
}
|
|
const reverseWeight = edges.get(targetId)?.get(sourceId) || 0;
|
|
edges.get(targetId).set(sourceId, Math.min(reverseWeight + weight, 1.0));
|
|
}
|
|
}
|
|
}
|
|
return {
|
|
nodes: Array.from(nodes),
|
|
edges,
|
|
nodeCount: nodes.size,
|
|
edgeCount: Array.from(edges.values()).reduce((sum, edgeMap) => sum + edgeMap.size, 0) / 2 // Undirected
|
|
};
|
|
}
|
|
/**
|
|
* Detect communities using Louvain modularity optimization
|
|
*/
|
|
async _detectCommunities(graph, options) {
|
|
const { nodes, edges } = graph;
|
|
// Initialize each node as its own community
|
|
const communities = new Map();
|
|
nodes.forEach((node, index) => communities.set(node, index));
|
|
const totalWeight = this._calculateTotalWeight(edges);
|
|
let improved = true;
|
|
let iteration = 0;
|
|
const maxIterations = 50;
|
|
// Louvain algorithm: iteratively move nodes to communities that maximize modularity
|
|
while (improved && iteration < maxIterations) {
|
|
improved = false;
|
|
iteration++;
|
|
for (const node of nodes) {
|
|
const currentCommunity = communities.get(node);
|
|
let bestCommunity = currentCommunity;
|
|
let bestGain = 0;
|
|
// Consider neighboring communities
|
|
const neighborCommunities = this._getNeighborCommunities(node, edges, communities);
|
|
for (const neighborCommunity of neighborCommunities) {
|
|
if (neighborCommunity !== currentCommunity) {
|
|
const gain = this._calculateModularityGain(node, currentCommunity, neighborCommunity, edges, communities, totalWeight);
|
|
if (gain > bestGain) {
|
|
bestGain = gain;
|
|
bestCommunity = neighborCommunity;
|
|
}
|
|
}
|
|
}
|
|
// Move node if beneficial
|
|
if (bestCommunity !== currentCommunity) {
|
|
communities.set(node, bestCommunity);
|
|
improved = true;
|
|
}
|
|
}
|
|
}
|
|
// Group nodes by final community assignment
|
|
const communityGroups = new Map();
|
|
for (const [node, communityId] of communities) {
|
|
if (!communityGroups.has(communityId)) {
|
|
communityGroups.set(communityId, []);
|
|
}
|
|
communityGroups.get(communityId).push(node);
|
|
}
|
|
// Convert to Community objects with metadata
|
|
const result = [];
|
|
for (const [communityId, members] of communityGroups) {
|
|
if (members.length >= (options.minClusterSize || 2)) {
|
|
const modularity = this._calculateCommunityModularity(members, edges, totalWeight);
|
|
const density = this._calculateCommunityDensity(members, edges);
|
|
const strongestConnections = this._findStrongestConnections(members, edges, 3);
|
|
result.push({
|
|
id: communityId,
|
|
members,
|
|
modularity,
|
|
density,
|
|
strongestConnections
|
|
});
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
/**
|
|
* Refine community boundaries using vector similarity
|
|
*/
|
|
async _refineCommunitiesWithVectors(communities, options) {
|
|
const refined = [];
|
|
for (const community of communities) {
|
|
const membersWithVectors = await this._getItemsWithVectors(community.members);
|
|
// Check if community is coherent in vector space
|
|
const vectorCoherence = await this._calculateVectorCoherence(membersWithVectors);
|
|
if (vectorCoherence > 0.3) {
|
|
// Community is coherent, keep as is
|
|
refined.push(community);
|
|
}
|
|
else {
|
|
// Split community using vector-based sub-clustering
|
|
const subClusters = await this._performHierarchicalClustering(community.members, { ...options, maxClusters: Math.ceil(community.members.length / 5) });
|
|
// Convert sub-clusters to communities
|
|
for (let i = 0; i < subClusters.clusters.length; i++) {
|
|
const subCluster = subClusters.clusters[i];
|
|
refined.push({
|
|
id: community.id * 1000 + i, // Unique sub-community ID
|
|
members: subCluster.members,
|
|
modularity: community.modularity * 0.8, // Slightly lower modularity for sub-communities
|
|
density: community.density,
|
|
strongestConnections: []
|
|
});
|
|
}
|
|
}
|
|
}
|
|
return refined;
|
|
}
|
|
// ===== SEMANTIC CLUSTERING UTILITIES =====
|
|
/**
|
|
* Get items with their metadata including noun types
|
|
*/
|
|
async _getItemsWithMetadata(itemIds) {
|
|
const items = await Promise.all(itemIds.map(async (id) => {
|
|
const noun = await this.brain.getNoun(id);
|
|
if (!noun) {
|
|
return null;
|
|
}
|
|
return {
|
|
id,
|
|
vector: noun.vector || [],
|
|
metadata: noun.metadata || {},
|
|
nounType: noun.metadata?.noun || noun.metadata?.nounType || 'content',
|
|
label: noun.metadata?.label || noun.metadata?.data || id,
|
|
data: noun.metadata
|
|
};
|
|
}));
|
|
return items.filter((item) => item !== null);
|
|
}
|
|
/**
|
|
* Group items by their semantic noun types
|
|
*/
|
|
_groupBySemanticType(items) {
|
|
const groups = new Map();
|
|
for (const item of items) {
|
|
const type = item.nounType;
|
|
if (!groups.has(type)) {
|
|
groups.set(type, []);
|
|
}
|
|
groups.get(type).push(item);
|
|
}
|
|
return groups;
|
|
}
|
|
// Placeholder implementations for complex operations
|
|
async _getAllItemIds() {
|
|
// Get all noun IDs from the brain
|
|
const stats = await this.brain.getStatistics();
|
|
if (!stats.totalNodes || stats.totalNodes === 0) {
|
|
return [];
|
|
}
|
|
// Get nouns with pagination (limit to 10000 for performance)
|
|
const limit = Math.min(stats.totalNodes, 10000);
|
|
const result = await this.brain.getNouns({
|
|
pagination: { limit }
|
|
});
|
|
return result.map((item) => item.id).filter((id) => id);
|
|
}
|
|
async _getTotalItemCount() {
|
|
const stats = await this.brain.getStatistics();
|
|
return stats.totalNodes || 0;
|
|
}
|
|
// ===== GRAPH ALGORITHM SUPPORTING METHODS =====
|
|
_calculateTotalWeight(edges) {
|
|
let total = 0;
|
|
for (const edgeMap of edges.values()) {
|
|
for (const weight of edgeMap.values()) {
|
|
total += weight;
|
|
}
|
|
}
|
|
return total / 2; // Undirected graph, so divide by 2
|
|
}
|
|
_getNeighborCommunities(node, edges, communities) {
|
|
const neighborCommunities = new Set();
|
|
const nodeEdges = edges.get(node);
|
|
if (nodeEdges) {
|
|
for (const neighbor of nodeEdges.keys()) {
|
|
const neighborCommunity = communities.get(neighbor);
|
|
if (neighborCommunity !== undefined) {
|
|
neighborCommunities.add(neighborCommunity);
|
|
}
|
|
}
|
|
}
|
|
return neighborCommunities;
|
|
}
|
|
_calculateModularityGain(node, oldCommunity, newCommunity, edges, communities, totalWeight) {
|
|
// Calculate the degree of the node
|
|
const nodeDegree = this._getNodeDegree(node, edges);
|
|
// Calculate edges to old and new communities
|
|
const edgesToOld = this._getEdgesToCommunity(node, oldCommunity, edges, communities);
|
|
const edgesToNew = this._getEdgesToCommunity(node, newCommunity, edges, communities);
|
|
// Calculate community weights
|
|
const oldCommunityWeight = this._getCommunityWeight(oldCommunity, edges, communities);
|
|
const newCommunityWeight = this._getCommunityWeight(newCommunity, edges, communities);
|
|
// Modularity gain calculation (simplified)
|
|
const oldContrib = edgesToOld - (nodeDegree * oldCommunityWeight) / (2 * totalWeight);
|
|
const newContrib = edgesToNew - (nodeDegree * newCommunityWeight) / (2 * totalWeight);
|
|
return newContrib - oldContrib;
|
|
}
|
|
_getNodeDegree(node, edges) {
|
|
const nodeEdges = edges.get(node);
|
|
if (!nodeEdges)
|
|
return 0;
|
|
return Array.from(nodeEdges.values()).reduce((sum, weight) => sum + weight, 0);
|
|
}
|
|
_getEdgesToCommunity(node, community, edges, communities) {
|
|
const nodeEdges = edges.get(node);
|
|
if (!nodeEdges)
|
|
return 0;
|
|
let total = 0;
|
|
for (const [neighbor, weight] of nodeEdges) {
|
|
if (communities.get(neighbor) === community) {
|
|
total += weight;
|
|
}
|
|
}
|
|
return total;
|
|
}
|
|
_getCommunityWeight(community, edges, communities) {
|
|
let total = 0;
|
|
for (const [node, nodeCommunity] of communities) {
|
|
if (nodeCommunity === community) {
|
|
total += this._getNodeDegree(node, edges);
|
|
}
|
|
}
|
|
return total;
|
|
}
|
|
_calculateCommunityModularity(members, edges, totalWeight) {
|
|
if (members.length < 2)
|
|
return 0;
|
|
let internalWeight = 0;
|
|
let totalDegree = 0;
|
|
for (const member of members) {
|
|
const memberEdges = edges.get(member);
|
|
if (memberEdges) {
|
|
totalDegree += Array.from(memberEdges.values()).reduce((sum, w) => sum + w, 0);
|
|
// Count internal edges
|
|
for (const [neighbor, weight] of memberEdges) {
|
|
if (members.includes(neighbor)) {
|
|
internalWeight += weight;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
internalWeight /= 2; // Undirected graph
|
|
const expectedInternal = (totalDegree * totalDegree) / (4 * totalWeight);
|
|
return (internalWeight / totalWeight) - expectedInternal / totalWeight;
|
|
}
|
|
_calculateCommunityDensity(members, edges) {
|
|
if (members.length < 2)
|
|
return 0;
|
|
let actualEdges = 0;
|
|
const maxPossibleEdges = (members.length * (members.length - 1)) / 2;
|
|
for (const member of members) {
|
|
const memberEdges = edges.get(member);
|
|
if (memberEdges) {
|
|
for (const neighbor of memberEdges.keys()) {
|
|
if (members.includes(neighbor) && member < neighbor) { // Avoid double counting
|
|
actualEdges++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return actualEdges / maxPossibleEdges;
|
|
}
|
|
_findStrongestConnections(members, edges, limit) {
|
|
const connections = [];
|
|
for (const member of members) {
|
|
const memberEdges = edges.get(member);
|
|
if (memberEdges) {
|
|
for (const [neighbor, weight] of memberEdges) {
|
|
if (members.includes(neighbor) && member < neighbor) { // Avoid duplicates
|
|
connections.push({ from: member, to: neighbor, weight });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return connections
|
|
.sort((a, b) => b.weight - a.weight)
|
|
.slice(0, limit);
|
|
}
|
|
// ===== K-MEANS UTILITIES =====
|
|
/**
|
|
* Get items with their vector representations
|
|
*/
|
|
async _getItemsWithVectors(itemIds) {
|
|
const items = await Promise.all(itemIds.map(async (id) => {
|
|
const noun = await this.brain.getNoun(id);
|
|
return {
|
|
id,
|
|
vector: noun?.vector || []
|
|
};
|
|
}));
|
|
return items.filter((item) => item !== null && item.vector.length > 0);
|
|
}
|
|
/**
|
|
* Calculate centroid from items using existing distance functions
|
|
*/
|
|
async _calculateCentroidFromItems(items) {
|
|
if (items.length === 0)
|
|
return [];
|
|
if (items.length === 1)
|
|
return [...items[0].vector];
|
|
const dimensions = items[0].vector.length;
|
|
const centroid = new Array(dimensions).fill(0);
|
|
for (const item of items) {
|
|
for (let i = 0; i < dimensions; i++) {
|
|
centroid[i] += item.vector[i];
|
|
}
|
|
}
|
|
for (let i = 0; i < dimensions; i++) {
|
|
centroid[i] /= items.length;
|
|
}
|
|
return centroid;
|
|
}
|
|
/**
|
|
* Initialize centroids using k-means++ algorithm for better convergence
|
|
*/
|
|
async _initializeCentroidsKMeansPlusPlus(items, k) {
|
|
const centroids = [];
|
|
// Choose first centroid randomly
|
|
const firstIdx = Math.floor(Math.random() * items.length);
|
|
centroids.push([...items[firstIdx].vector]);
|
|
// Choose remaining centroids using k-means++ probability
|
|
for (let i = 1; i < k; i++) {
|
|
const distances = items.map(item => {
|
|
// Find distance to closest existing centroid
|
|
let minDist = Infinity;
|
|
for (const centroid of centroids) {
|
|
const dist = this._calculateSquaredDistance(item.vector, centroid);
|
|
minDist = Math.min(minDist, dist);
|
|
}
|
|
return minDist;
|
|
});
|
|
// Choose next centroid with probability proportional to squared distance
|
|
const totalDistance = distances.reduce((sum, d) => sum + d, 0);
|
|
const target = Math.random() * totalDistance;
|
|
let cumulative = 0;
|
|
for (let j = 0; j < distances.length; j++) {
|
|
cumulative += distances[j];
|
|
if (cumulative >= target) {
|
|
centroids.push([...items[j].vector]);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return centroids;
|
|
}
|
|
/**
|
|
* Assign points to nearest centroids using existing distance functions
|
|
*/
|
|
async _assignPointsToCentroids(items, centroids) {
|
|
const assignments = [];
|
|
for (const item of items) {
|
|
let bestCentroid = 0;
|
|
let minDistance = Infinity;
|
|
for (let i = 0; i < centroids.length; i++) {
|
|
const distance = this._calculateSquaredDistance(item.vector, centroids[i]);
|
|
if (distance < minDistance) {
|
|
minDistance = distance;
|
|
bestCentroid = i;
|
|
}
|
|
}
|
|
assignments.push(bestCentroid);
|
|
}
|
|
return assignments;
|
|
}
|
|
/**
|
|
* Update centroids based on current assignments
|
|
*/
|
|
async _updateCentroids(items, assignments, k) {
|
|
const newCentroids = [];
|
|
for (let i = 0; i < k; i++) {
|
|
const clusterItems = items.filter((_, idx) => assignments[idx] === i);
|
|
if (clusterItems.length > 0) {
|
|
newCentroids.push(await this._calculateCentroidFromItems(clusterItems));
|
|
}
|
|
else {
|
|
// Keep old centroid if no items assigned
|
|
newCentroids.push(new Array(items[0].vector.length).fill(0));
|
|
}
|
|
}
|
|
return newCentroids;
|
|
}
|
|
/**
|
|
* Calculate how much assignments have changed between iterations
|
|
*/
|
|
_calculateAssignmentChangeRate(oldAssignments, newAssignments) {
|
|
if (oldAssignments.length !== newAssignments.length)
|
|
return 1.0;
|
|
let changes = 0;
|
|
for (let i = 0; i < oldAssignments.length; i++) {
|
|
if (oldAssignments[i] !== newAssignments[i]) {
|
|
changes++;
|
|
}
|
|
}
|
|
return changes / oldAssignments.length;
|
|
}
|
|
/**
|
|
* Calculate cluster confidence for k-means clusters
|
|
*/
|
|
async _calculateKMeansClusterConfidence(clusterItems, centroid) {
|
|
if (clusterItems.length <= 1)
|
|
return 1.0;
|
|
// Calculate average distance to centroid
|
|
const distances = clusterItems.map(item => this._calculateSquaredDistance(item.vector, centroid));
|
|
const avgDistance = distances.reduce((sum, d) => sum + d, 0) / distances.length;
|
|
// Calculate standard deviation
|
|
const variance = distances.reduce((sum, d) => sum + Math.pow(d - avgDistance, 2), 0) / distances.length;
|
|
const stdDev = Math.sqrt(variance);
|
|
// Higher confidence for tighter clusters
|
|
const tightness = avgDistance > 0 ? Math.max(0, 1 - (stdDev / avgDistance)) : 1.0;
|
|
return Math.min(1.0, tightness);
|
|
}
|
|
// ===== DBSCAN UTILITIES =====
|
|
/**
|
|
* Estimate optimal eps parameter using k-nearest neighbor distances
|
|
*/
|
|
async _estimateOptimalEps(items, minPts) {
|
|
if (items.length < minPts)
|
|
return 0.5;
|
|
// Calculate k-nearest neighbor distances for each point
|
|
const kDistances = [];
|
|
for (const item of items) {
|
|
const distances = [];
|
|
for (const otherItem of items) {
|
|
if (item.id !== otherItem.id) {
|
|
const distance = Math.sqrt(this._calculateSquaredDistance(item.vector, otherItem.vector));
|
|
distances.push(distance);
|
|
}
|
|
}
|
|
distances.sort((a, b) => a - b);
|
|
// Get k-th nearest neighbor distance (minPts-1 because we exclude self)
|
|
const kthDistance = distances[Math.min(minPts - 1, distances.length - 1)];
|
|
kDistances.push(kthDistance);
|
|
}
|
|
kDistances.sort((a, b) => a - b);
|
|
// Use knee point detection - find point with maximum curvature
|
|
// Simplified approach: use 90th percentile of k-distances
|
|
const percentileIndex = Math.floor(kDistances.length * 0.9);
|
|
return kDistances[percentileIndex] || 0.5;
|
|
}
|
|
/**
|
|
* Find neighbors within epsilon distance using efficient vector operations
|
|
*/
|
|
async _findNeighborsWithinEps(item, allItems, eps) {
|
|
const neighbors = [];
|
|
const epsSquared = eps * eps;
|
|
for (const otherItem of allItems) {
|
|
if (item.id !== otherItem.id) {
|
|
const distanceSquared = this._calculateSquaredDistance(item.vector, otherItem.vector);
|
|
if (distanceSquared <= epsSquared) {
|
|
neighbors.push(otherItem);
|
|
}
|
|
}
|
|
}
|
|
return neighbors;
|
|
}
|
|
/**
|
|
* Expand DBSCAN cluster by adding density-reachable points
|
|
*/
|
|
async _expandCluster(seedPoint, neighbors, clusterId, eps, minPts, allItems, visited, clusterAssignments) {
|
|
clusterAssignments.set(seedPoint.id, clusterId);
|
|
let i = 0;
|
|
while (i < neighbors.length) {
|
|
const neighbor = neighbors[i];
|
|
if (!visited.get(neighbor.id)) {
|
|
visited.set(neighbor.id, true);
|
|
const neighborNeighbors = await this._findNeighborsWithinEps(neighbor, allItems, eps);
|
|
if (neighborNeighbors.length >= minPts) {
|
|
// Add new neighbors to the list (union operation)
|
|
for (const newNeighbor of neighborNeighbors) {
|
|
if (!neighbors.some(n => n.id === newNeighbor.id)) {
|
|
neighbors.push(newNeighbor);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// If neighbor is not assigned to any cluster, assign to current cluster
|
|
if (!clusterAssignments.has(neighbor.id)) {
|
|
clusterAssignments.set(neighbor.id, clusterId);
|
|
}
|
|
i++;
|
|
}
|
|
}
|
|
/**
|
|
* Calculate DBSCAN cluster confidence based on density
|
|
*/
|
|
async _calculateDBSCANClusterConfidence(clusterItems, eps) {
|
|
if (clusterItems.length <= 1)
|
|
return 1.0;
|
|
// Calculate average density within the cluster
|
|
let totalNeighborCount = 0;
|
|
const epsSquared = eps * eps;
|
|
for (const item of clusterItems) {
|
|
let neighborCount = 0;
|
|
for (const otherItem of clusterItems) {
|
|
if (item !== otherItem) {
|
|
const distanceSquared = this._calculateSquaredDistance(item.vector, otherItem.vector);
|
|
if (distanceSquared <= epsSquared) {
|
|
neighborCount++;
|
|
}
|
|
}
|
|
}
|
|
totalNeighborCount += neighborCount;
|
|
}
|
|
const avgDensity = totalNeighborCount / clusterItems.length;
|
|
const maxPossibleDensity = clusterItems.length - 1;
|
|
return maxPossibleDensity > 0 ? avgDensity / maxPossibleDensity : 1.0;
|
|
}
|
|
// ===== VECTOR UTILITIES =====
|
|
/**
|
|
* Calculate squared Euclidean distance (more efficient than sqrt)
|
|
*/
|
|
_calculateSquaredDistance(vec1, vec2) {
|
|
if (vec1.length !== vec2.length)
|
|
return Infinity;
|
|
let sum = 0;
|
|
for (let i = 0; i < vec1.length; i++) {
|
|
const diff = vec1[i] - vec2[i];
|
|
sum += diff * diff;
|
|
}
|
|
return sum;
|
|
}
|
|
/**
|
|
* Calculate vector coherence for community refinement
|
|
*/
|
|
async _calculateVectorCoherence(items) {
|
|
if (items.length <= 1)
|
|
return 1.0;
|
|
const centroid = await this._calculateCentroidFromItems(items);
|
|
// Calculate average distance to centroid
|
|
const distances = items.map(item => Math.sqrt(this._calculateSquaredDistance(item.vector, centroid)));
|
|
const avgDistance = distances.reduce((sum, d) => sum + d, 0) / distances.length;
|
|
// Calculate cohesion as inverse of average distance (normalized)
|
|
const maxDistance = Math.sqrt(centroid.length); // Rough normalization
|
|
return Math.max(0, 1 - (avgDistance / maxDistance));
|
|
}
|
|
async _getItemsByField(field) {
|
|
// Implementation would query items by metadata field
|
|
return [];
|
|
}
|
|
// ===== TRIPLE INTELLIGENCE INTEGRATION =====
|
|
/**
|
|
* Generate intelligent cluster labels using Triple Intelligence
|
|
*/
|
|
async _generateIntelligentClusterLabel(members, algorithm) {
|
|
if (members.length === 0)
|
|
return `${algorithm}-cluster`;
|
|
try {
|
|
// Lazy load Triple Intelligence if available
|
|
const TripleIntelligenceEngine = await import('../triple/TripleIntelligence.js')
|
|
.then(m => m.TripleIntelligenceEngine)
|
|
.catch(() => null);
|
|
if (!TripleIntelligenceEngine) {
|
|
return this._generateClusterLabel(members, algorithm);
|
|
}
|
|
const intelligence = new TripleIntelligenceEngine(this.brain);
|
|
// Extract key features from cluster members
|
|
const memberData = members.map(m => ({
|
|
id: m.id,
|
|
type: m.nounType,
|
|
label: m.label,
|
|
data: m.data
|
|
}));
|
|
// Use Triple Intelligence to analyze the cluster and generate label
|
|
const prompt = `Analyze this cluster of ${memberData.length} related items and provide a concise, descriptive label (2-4 words):
|
|
|
|
Items:
|
|
${memberData.map(item => `- ${item.label || item.id} (${item.type})`).join('\n')}
|
|
|
|
The items were grouped using ${algorithm} clustering. What is the most appropriate label that captures their common theme or relationship?`;
|
|
const response = await intelligence.find({
|
|
like: prompt,
|
|
limit: 1
|
|
});
|
|
// Extract clean label from response
|
|
const firstResult = response[0];
|
|
const label = (firstResult?.metadata?.content || firstResult?.id || `${algorithm}-cluster`)
|
|
.toString()
|
|
.replace(/^(Label:|Cluster:|Theme:)/i, '')
|
|
.trim()
|
|
.replace(/['"]/g, '')
|
|
.slice(0, 50);
|
|
return label || `${algorithm}-cluster`;
|
|
}
|
|
catch (error) {
|
|
// Fallback to simple labeling
|
|
return this._generateClusterLabel(members, algorithm);
|
|
}
|
|
}
|
|
/**
|
|
* Generate simple cluster labels based on semantic analysis
|
|
*/
|
|
async _generateClusterLabel(members, algorithm) {
|
|
if (members.length === 0)
|
|
return `${algorithm}-cluster`;
|
|
// Analyze member types and create descriptive label
|
|
const typeCount = new Map();
|
|
for (const member of members) {
|
|
const type = member.nounType || 'unknown';
|
|
typeCount.set(type, (typeCount.get(type) || 0) + 1);
|
|
}
|
|
// Find most common type
|
|
let dominantType = 'mixed';
|
|
let maxCount = 0;
|
|
for (const [type, count] of typeCount) {
|
|
if (count > maxCount) {
|
|
maxCount = count;
|
|
dominantType = type;
|
|
}
|
|
}
|
|
// Generate label based on dominant type and size
|
|
const size = members.length;
|
|
const typePercent = Math.round((maxCount / size) * 100);
|
|
if (typePercent >= 80) {
|
|
return `${dominantType} group (${size})`;
|
|
}
|
|
else if (typePercent >= 60) {
|
|
return `mostly ${dominantType} (${size})`;
|
|
}
|
|
else {
|
|
const topTypes = Array.from(typeCount.entries())
|
|
.sort((a, b) => b[1] - a[1])
|
|
.slice(0, 2)
|
|
.map(([type]) => type)
|
|
.join(' & ');
|
|
return `${topTypes} cluster (${size})`;
|
|
}
|
|
}
|
|
/**
|
|
* Fuse clustering results using Triple Intelligence consensus
|
|
*/
|
|
async _fuseClusteringResultsWithTripleIntelligence(clusterSets, options) {
|
|
if (clusterSets.length === 0)
|
|
return [];
|
|
if (clusterSets.length === 1)
|
|
return clusterSets[0];
|
|
// Simple weighted fusion if Triple Intelligence is not available
|
|
const [vectorClusters, graphClusters, semanticClusters] = clusterSets;
|
|
// Create consensus mapping of items to clusters
|
|
const itemClusterMapping = new Map();
|
|
// Collect all cluster assignments
|
|
const allAlgorithms = ['vector', 'graph', 'semantic'];
|
|
const algorithmClusters = [vectorClusters, graphClusters, semanticClusters];
|
|
for (let i = 0; i < algorithmClusters.length; i++) {
|
|
const algorithm = allAlgorithms[i];
|
|
const clusters = algorithmClusters[i] || [];
|
|
for (const cluster of clusters) {
|
|
for (const memberId of cluster.members) {
|
|
if (!itemClusterMapping.has(memberId)) {
|
|
itemClusterMapping.set(memberId, []);
|
|
}
|
|
itemClusterMapping.get(memberId).push({
|
|
algorithm,
|
|
clusterId: cluster.id,
|
|
confidence: cluster.confidence
|
|
});
|
|
}
|
|
}
|
|
}
|
|
// Find consensus clusters - items that appear together in multiple algorithms
|
|
const consensusClusters = new Map();
|
|
const processedItems = new Set();
|
|
for (const [itemId, assignments] of itemClusterMapping) {
|
|
if (processedItems.has(itemId))
|
|
continue;
|
|
// Find all items that consistently cluster with this item
|
|
const consensusGroup = new Set([itemId]);
|
|
// Look for items that share clusters with this item across algorithms
|
|
for (const assignment of assignments) {
|
|
const sameClusterItems = this._getItemsInCluster(assignment.clusterId, clusterSets);
|
|
for (const otherItem of sameClusterItems) {
|
|
if (!processedItems.has(otherItem) && otherItem !== itemId) {
|
|
const otherAssignments = itemClusterMapping.get(otherItem) || [];
|
|
// Check if items co-occur in multiple algorithms
|
|
const coOccurrences = this._countCoOccurrences(assignments, otherAssignments);
|
|
if (coOccurrences >= 2) { // Must appear together in at least 2 algorithms
|
|
consensusGroup.add(otherItem);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Mark all items in this consensus group as processed
|
|
for (const groupItem of consensusGroup) {
|
|
processedItems.add(groupItem);
|
|
}
|
|
if (consensusGroup.size >= (options.minClusterSize || 2)) {
|
|
const consensusId = `fusion-${consensusClusters.size}`;
|
|
consensusClusters.set(consensusId, consensusGroup);
|
|
}
|
|
}
|
|
// Convert consensus groups to SemanticCluster objects
|
|
const fusedClusters = [];
|
|
for (const [clusterId, memberSet] of consensusClusters) {
|
|
const members = Array.from(memberSet);
|
|
const membersWithMetadata = await this._getItemsWithMetadata(members);
|
|
if (membersWithMetadata.length > 0) {
|
|
const centroid = await this._calculateCentroidFromItems(membersWithMetadata);
|
|
const label = await this._generateIntelligentClusterLabel(membersWithMetadata, 'multimodal');
|
|
// Calculate fusion confidence based on algorithm agreement
|
|
const avgConfidence = this._calculateFusionConfidence(members, itemClusterMapping);
|
|
fusedClusters.push({
|
|
id: clusterId,
|
|
centroid,
|
|
members,
|
|
size: members.length,
|
|
confidence: avgConfidence,
|
|
label,
|
|
metadata: {
|
|
clustering: 'multimodal_fusion',
|
|
algorithms: allAlgorithms,
|
|
fusionMethod: 'consensus',
|
|
agreementLevel: avgConfidence
|
|
}
|
|
});
|
|
}
|
|
}
|
|
return fusedClusters;
|
|
}
|
|
/**
|
|
* Get items in a specific cluster from cluster sets
|
|
*/
|
|
_getItemsInCluster(clusterId, clusterSets) {
|
|
for (const clusterSet of clusterSets) {
|
|
for (const cluster of clusterSet) {
|
|
if (cluster.id === clusterId) {
|
|
return cluster.members;
|
|
}
|
|
}
|
|
}
|
|
return [];
|
|
}
|
|
/**
|
|
* Count co-occurrences between two sets of assignments
|
|
*/
|
|
_countCoOccurrences(assignments1, assignments2) {
|
|
let count = 0;
|
|
for (const assignment1 of assignments1) {
|
|
for (const assignment2 of assignments2) {
|
|
if (assignment1.algorithm === assignment2.algorithm &&
|
|
assignment1.clusterId === assignment2.clusterId) {
|
|
count++;
|
|
}
|
|
}
|
|
}
|
|
return count;
|
|
}
|
|
/**
|
|
* Calculate fusion confidence based on algorithm agreement
|
|
*/
|
|
_calculateFusionConfidence(members, itemClusterMapping) {
|
|
let totalConfidence = 0;
|
|
let totalAssignments = 0;
|
|
for (const member of members) {
|
|
const assignments = itemClusterMapping.get(member) || [];
|
|
for (const assignment of assignments) {
|
|
totalConfidence += assignment.confidence;
|
|
totalAssignments++;
|
|
}
|
|
}
|
|
return totalAssignments > 0 ? totalConfidence / totalAssignments : 0.5;
|
|
}
|
|
// ===== ADDITIONAL UTILITIES =====
|
|
/**
|
|
* Generate empty clustering result for edge cases
|
|
*/
|
|
_createEmptyResult(startTime, algorithm) {
|
|
return {
|
|
clusters: [],
|
|
metrics: this._createPerformanceMetrics(startTime, 0, algorithm),
|
|
metadata: {
|
|
totalItems: 0,
|
|
clustersFound: 0,
|
|
averageClusterSize: 0,
|
|
timestamp: new Date()
|
|
}
|
|
};
|
|
}
|
|
// ===== SAMPLING AND PROJECTION UTILITIES =====
|
|
/**
|
|
* Get sample using specified strategy for large dataset clustering
|
|
*/
|
|
async _getSampleUsingStrategy(itemIds, sampleSize, strategy) {
|
|
if (itemIds.length <= sampleSize)
|
|
return itemIds;
|
|
switch (strategy) {
|
|
case 'random':
|
|
return this._getRandomSample(itemIds, sampleSize);
|
|
case 'diverse':
|
|
return await this._getDiverseSample(itemIds, sampleSize);
|
|
case 'recent':
|
|
return await this._getRecentSample(itemIds, sampleSize);
|
|
case 'important':
|
|
return await this._getImportantSample(itemIds, sampleSize);
|
|
default:
|
|
return this._getRandomSample(itemIds, sampleSize);
|
|
}
|
|
}
|
|
/**
|
|
* Random sampling
|
|
*/
|
|
_getRandomSample(itemIds, sampleSize) {
|
|
const shuffled = [...itemIds].sort(() => Math.random() - 0.5);
|
|
return shuffled.slice(0, sampleSize);
|
|
}
|
|
/**
|
|
* Diverse sampling using vector space distribution
|
|
*/
|
|
async _getDiverseSample(itemIds, sampleSize) {
|
|
// Get vectors for all items
|
|
const itemsWithVectors = await this._getItemsWithVectors(itemIds);
|
|
if (itemsWithVectors.length <= sampleSize) {
|
|
return itemIds;
|
|
}
|
|
// Use k-means++ style selection for diversity
|
|
const sample = [];
|
|
// Select first item randomly
|
|
let remainingItems = [...itemsWithVectors];
|
|
const firstIdx = Math.floor(Math.random() * remainingItems.length);
|
|
sample.push(remainingItems[firstIdx].id);
|
|
remainingItems.splice(firstIdx, 1);
|
|
// Select remaining items based on maximum distance to already selected items
|
|
while (sample.length < sampleSize && remainingItems.length > 0) {
|
|
let maxDistance = -1;
|
|
let bestIdx = 0;
|
|
for (let i = 0; i < remainingItems.length; i++) {
|
|
const item = remainingItems[i];
|
|
// Find minimum distance to any selected item
|
|
let minDistanceToSelected = Infinity;
|
|
for (const selectedId of sample) {
|
|
const selectedItem = itemsWithVectors.find(it => it.id === selectedId);
|
|
if (selectedItem) {
|
|
const distance = Math.sqrt(this._calculateSquaredDistance(item.vector, selectedItem.vector));
|
|
minDistanceToSelected = Math.min(minDistanceToSelected, distance);
|
|
}
|
|
}
|
|
// Select item with maximum minimum distance (most diverse)
|
|
if (minDistanceToSelected > maxDistance) {
|
|
maxDistance = minDistanceToSelected;
|
|
bestIdx = i;
|
|
}
|
|
}
|
|
sample.push(remainingItems[bestIdx].id);
|
|
remainingItems.splice(bestIdx, 1);
|
|
}
|
|
return sample;
|
|
}
|
|
/**
|
|
* Recent sampling based on creation time
|
|
*/
|
|
async _getRecentSample(itemIds, sampleSize) {
|
|
const items = await Promise.all(itemIds.map(async (id) => {
|
|
const noun = await this.brain.getNoun(id);
|
|
return {
|
|
id,
|
|
createdAt: noun?.createdAt || new Date(0)
|
|
};
|
|
}));
|
|
// Sort by creation time (most recent first)
|
|
items.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
|
|
return items.slice(0, sampleSize).map(item => item.id);
|
|
}
|
|
/**
|
|
* Important sampling based on connection count and metadata
|
|
*/
|
|
async _getImportantSample(itemIds, sampleSize) {
|
|
const items = await Promise.all(itemIds.map(async (id) => {
|
|
const verbs = await this.brain.getVerbsForNoun(id);
|
|
const noun = await this.brain.getNoun(id);
|
|
// Calculate importance score
|
|
const connectionScore = verbs.length;
|
|
const dataScore = noun?.data ? Object.keys(noun.data).length : 0;
|
|
const importanceScore = connectionScore * 2 + dataScore;
|
|
return {
|
|
id,
|
|
importance: importanceScore
|
|
};
|
|
}));
|
|
// Sort by importance (highest first)
|
|
items.sort((a, b) => b.importance - a.importance);
|
|
return items.slice(0, sampleSize).map(item => item.id);
|
|
}
|
|
/**
|
|
* Project clusters back to full dataset using HNSW neighbors
|
|
*/
|
|
async _projectClustersToFullDataset(sampleClusters, fullItemIds, sampleIds) {
|
|
const projectedClusters = [];
|
|
// Create mapping of items not in sample
|
|
const remainingItems = fullItemIds.filter(id => !sampleIds.includes(id));
|
|
// For each sample cluster, find which remaining items should belong to it
|
|
for (const sampleCluster of sampleClusters) {
|
|
const projectedMembers = [...sampleCluster.members];
|
|
// For each remaining item, find its nearest neighbors in the sample
|
|
for (const itemId of remainingItems) {
|
|
try {
|
|
const neighbors = await this.brain.neural.neighbors(itemId, {
|
|
limit: 3,
|
|
includeMetadata: false
|
|
});
|
|
// Check if any of the nearest neighbors belong to this cluster
|
|
let belongsToCluster = false;
|
|
for (const neighbor of neighbors.neighbors) {
|
|
if (sampleCluster.members.includes(neighbor.id) && neighbor.similarity > 0.7) {
|
|
belongsToCluster = true;
|
|
break;
|
|
}
|
|
}
|
|
if (belongsToCluster) {
|
|
projectedMembers.push(itemId);
|
|
}
|
|
}
|
|
catch (error) {
|
|
// Skip items that can't be processed
|
|
continue;
|
|
}
|
|
}
|
|
// Create projected cluster
|
|
if (projectedMembers.length > 0) {
|
|
const membersWithVectors = await this._getItemsWithVectors(projectedMembers);
|
|
projectedClusters.push({
|
|
...sampleCluster,
|
|
id: `projected-${sampleCluster.id}`,
|
|
members: projectedMembers,
|
|
size: projectedMembers.length,
|
|
centroid: await this._calculateCentroidFromItems(membersWithVectors),
|
|
confidence: sampleCluster.confidence * 0.9, // Slightly lower confidence for projection
|
|
metadata: {
|
|
...sampleCluster.metadata,
|
|
isProjected: true,
|
|
originalSampleSize: sampleCluster.size,
|
|
projectedSize: projectedMembers.length
|
|
}
|
|
});
|
|
}
|
|
}
|
|
return projectedClusters;
|
|
}
|
|
_groupByDomain(items, field) {
|
|
const groups = new Map();
|
|
for (const item of items) {
|
|
const domain = item.metadata?.[field] || 'unknown';
|
|
if (!groups.has(domain)) {
|
|
groups.set(domain, []);
|
|
}
|
|
groups.get(domain).push(item);
|
|
}
|
|
return groups;
|
|
}
|
|
_calculateDomainConfidence(cluster, domainItems) {
|
|
// Calculate how well this cluster represents the domain
|
|
// Based on cluster density and coherence
|
|
const density = cluster.members.length / (cluster.members.length + 10); // Normalize
|
|
const coherence = cluster.cohesion || 0.5; // Use cluster's cohesion if available
|
|
// Domain relevance: what fraction of cluster members are from this domain
|
|
const domainMemberCount = cluster.members.filter(id => domainItems.some(item => item.id === id)).length;
|
|
const domainRelevance = cluster.members.length > 0
|
|
? domainMemberCount / cluster.members.length
|
|
: 0;
|
|
return (density * 0.3 + coherence * 0.3 + domainRelevance * 0.4); // Weighted average
|
|
}
|
|
async _findCrossDomainMembers(cluster, threshold) {
|
|
// Find members that might belong to multiple domains
|
|
return [];
|
|
}
|
|
async _findCrossDomainClusters(clusters, threshold) {
|
|
// Find clusters that span multiple domains
|
|
return [];
|
|
}
|
|
async _getItemsByTimeWindow(timeField, window) {
|
|
// Implementation would query items within time window
|
|
return [];
|
|
}
|
|
async _calculateTemporalMetrics(cluster, items, timeField) {
|
|
// Calculate temporal characteristics of the cluster
|
|
return {
|
|
trend: 'stable',
|
|
metrics: {
|
|
startTime: new Date(),
|
|
endTime: new Date(),
|
|
peakTime: new Date(),
|
|
frequency: 1
|
|
}
|
|
};
|
|
}
|
|
_mergeOverlappingTemporalClusters(clusters) {
|
|
// Merge clusters from overlapping time windows
|
|
return clusters;
|
|
}
|
|
_adjustThresholdAdaptively(clusters, currentThreshold) {
|
|
// Adjust clustering threshold based on results
|
|
return currentThreshold || 0.6;
|
|
}
|
|
async _calculateItemToClusterSimilarity(itemId, cluster) {
|
|
// Calculate similarity between an item and a cluster centroid
|
|
return 0.5; // Placeholder
|
|
}
|
|
async _recalculateClusterCentroid(cluster) {
|
|
// Recalculate centroid after adding new members
|
|
return cluster.centroid;
|
|
}
|
|
async _calculateSimilarity(id1, id2) {
|
|
return await this.similar(id1, id2);
|
|
}
|
|
_calculateEdgeWeight(verb) {
|
|
// Calculate edge weight based on verb properties
|
|
let weight = 1.0;
|
|
// Factor in connection strength if available
|
|
if (verb.connections && verb.connections instanceof Map) {
|
|
const connectionCount = verb.connections.size;
|
|
weight += Math.log(connectionCount + 1) * 0.1;
|
|
}
|
|
// Factor in verb type significance
|
|
const significantVerbs = ['caused', 'created', 'contains', 'implements', 'extends'];
|
|
if (verb.verb && significantVerbs.includes(verb.verb.toLowerCase())) {
|
|
weight += 0.3;
|
|
}
|
|
// Factor in recency if available
|
|
if (verb.metadata?.createdAt) {
|
|
const now = Date.now();
|
|
const created = new Date(verb.metadata.createdAt).getTime();
|
|
const daysSinceCreated = (now - created) / (1000 * 60 * 60 * 24);
|
|
// Newer relationships get slight boost
|
|
weight += Math.max(0, (30 - daysSinceCreated) / 100);
|
|
}
|
|
return Math.min(weight, 3.0); // Cap at 3.0
|
|
}
|
|
_sortNeighbors(neighbors, sortBy) {
|
|
switch (sortBy) {
|
|
case 'similarity':
|
|
neighbors.sort((a, b) => b.similarity - a.similarity);
|
|
break;
|
|
case 'importance':
|
|
neighbors.sort((a, b) => (b.metadata?.importance || 0) - (a.metadata?.importance || 0));
|
|
break;
|
|
case 'recency':
|
|
neighbors.sort((a, b) => {
|
|
const aTime = new Date(a.metadata?.createdAt || 0).getTime();
|
|
const bTime = new Date(b.metadata?.createdAt || 0).getTime();
|
|
return bTime - aTime;
|
|
});
|
|
break;
|
|
}
|
|
}
|
|
async _buildSemanticHierarchy(item, options) {
|
|
// Build semantic hierarchy around an item
|
|
return {
|
|
self: { id: item.id, vector: item.vector, metadata: item.metadata }
|
|
};
|
|
}
|
|
async _detectOutliersClusterBased(threshold, options) {
|
|
// Detect outliers using cluster-based method
|
|
return [];
|
|
}
|
|
async _detectOutliersIsolation(threshold, options) {
|
|
// Detect outliers using isolation forest method
|
|
return [];
|
|
}
|
|
async _detectOutliersStatistical(threshold, options) {
|
|
// Detect outliers using statistical methods
|
|
return [];
|
|
}
|
|
async _generateVisualizationNodes(maxNodes, options) {
|
|
// Generate nodes for visualization
|
|
return [];
|
|
}
|
|
async _generateVisualizationEdges(nodes, options) {
|
|
// Generate edges for visualization
|
|
return [];
|
|
}
|
|
async _generateVisualizationClusters(nodes) {
|
|
// Generate cluster information for visualization
|
|
return [];
|
|
}
|
|
async _applyLayoutAlgorithm(nodes, edges, algorithm, dimensions) {
|
|
// Apply layout algorithm to position nodes
|
|
return nodes.map((node, i) => ({
|
|
...node,
|
|
x: Math.random() * 100,
|
|
y: Math.random() * 100,
|
|
z: dimensions === 3 ? Math.random() * 100 : undefined
|
|
}));
|
|
}
|
|
_manhattanDistance(v1, v2) {
|
|
let sum = 0;
|
|
for (let i = 0; i < v1.length; i++) {
|
|
sum += Math.abs(v1[i] - v2[i]);
|
|
}
|
|
return sum;
|
|
}
|
|
_calculateConfidence(score, v1, v2) {
|
|
// Calculate confidence based on vector magnitudes and score
|
|
return Math.min(1, score + 0.1);
|
|
}
|
|
_generateSimilarityExplanation(score, metric) {
|
|
if (score > 0.9)
|
|
return `Very high similarity using ${metric} distance`;
|
|
if (score > 0.7)
|
|
return `High similarity using ${metric} distance`;
|
|
if (score > 0.5)
|
|
return `Moderate similarity using ${metric} distance`;
|
|
if (score > 0.3)
|
|
return `Low similarity using ${metric} distance`;
|
|
return `Very low similarity using ${metric} distance`;
|
|
}
|
|
// ===== PUBLIC API: UTILITY & STATUS =====
|
|
/**
|
|
* Get performance metrics for monitoring
|
|
*/
|
|
getPerformanceMetrics(operation) {
|
|
if (operation) {
|
|
return this.performanceMetrics.get(operation) || [];
|
|
}
|
|
return this.performanceMetrics;
|
|
}
|
|
/**
|
|
* Clear all caches
|
|
*/
|
|
clearCaches() {
|
|
this.similarityCache.clear();
|
|
this.clusterCache.clear();
|
|
this.hierarchyCache.clear();
|
|
this.neighborsCache.clear();
|
|
}
|
|
/**
|
|
* Get cache statistics
|
|
*/
|
|
getCacheStats() {
|
|
const maxSize = this.config.cacheSize || 1000;
|
|
return {
|
|
similarity: { size: this.similarityCache.size, maxSize },
|
|
clustering: { size: this.clusterCache.size, maxSize },
|
|
hierarchy: { size: this.hierarchyCache.size, maxSize },
|
|
neighbors: { size: this.neighborsCache.size, maxSize }
|
|
};
|
|
}
|
|
// ===== MISSING HELPER METHODS =====
|
|
/**
|
|
* Analyze data characteristics for algorithm selection
|
|
*/
|
|
async _analyzeDataCharacteristics(itemIds) {
|
|
const size = itemIds.length;
|
|
const items = await this._getItemsWithMetadata(itemIds.slice(0, Math.min(100, size)));
|
|
const dimensionality = items.length > 0 ? items[0].vector.length : 0;
|
|
// Calculate graph density by sampling verb relationships
|
|
let connectionCount = 0;
|
|
const sampleSize = Math.min(50, itemIds.length);
|
|
for (let i = 0; i < sampleSize; i++) {
|
|
try {
|
|
const verbs = await this.brain.getVerbsForNoun(itemIds[i]);
|
|
connectionCount += verbs.length;
|
|
}
|
|
catch (error) {
|
|
// Skip items that can't be processed
|
|
continue;
|
|
}
|
|
}
|
|
const graphDensity = sampleSize > 0 ? connectionCount / (sampleSize * sampleSize) : 0;
|
|
// Calculate type distribution
|
|
const typeDistribution = {};
|
|
for (const item of items) {
|
|
const type = item.nounType;
|
|
typeDistribution[type] = (typeDistribution[type] || 0) + 1;
|
|
}
|
|
return { size, dimensionality, graphDensity, typeDistribution };
|
|
}
|
|
/**
|
|
* Calculate centroid for a group of items
|
|
*/
|
|
async _calculateGroupCentroid(items) {
|
|
return this._calculateCentroidFromItems(items);
|
|
}
|
|
/**
|
|
* Cluster within semantic type using vector similarity
|
|
*/
|
|
async _clusterWithinSemanticType(items, options) {
|
|
if (items.length <= 2) {
|
|
return [{
|
|
id: `semantic-single-${items[0]?.nounType || 'unknown'}`,
|
|
centroid: await this._calculateCentroidFromItems(items),
|
|
members: items.map(item => item.id),
|
|
size: items.length,
|
|
confidence: 1.0,
|
|
label: `${items[0]?.nounType || 'unknown'} group`,
|
|
metadata: { clustering: 'semantic', nounType: items[0]?.nounType }
|
|
}];
|
|
}
|
|
// Use hierarchical clustering for within-type clustering
|
|
const result = await this._performHierarchicalClustering(items.map(item => item.id), { ...options, maxClusters: Math.min(Math.ceil(items.length / 3), 10) });
|
|
return result.clusters;
|
|
}
|
|
/**
|
|
* Find cross-type connections via verbs
|
|
*/
|
|
async _findCrossTypeConnections(typeGroups, _options) {
|
|
const connections = [];
|
|
// Convert Map to array for compatibility
|
|
const typeGroupsArray = Array.from(typeGroups.entries());
|
|
for (const [fromType, fromItems] of typeGroupsArray) {
|
|
for (const [toType, toItems] of typeGroupsArray) {
|
|
if (fromType !== toType) {
|
|
for (const fromItem of fromItems.slice(0, 10)) { // Sample to avoid N^2
|
|
try {
|
|
const verbs = await this.brain.getVerbsForNoun(fromItem.id);
|
|
for (const verb of verbs) {
|
|
const toItem = toItems.find(item => item.id === verb.target);
|
|
if (toItem) {
|
|
connections.push({
|
|
from: fromItem.id,
|
|
to: toItem.id,
|
|
strength: verb.confidence || 0.7
|
|
});
|
|
}
|
|
}
|
|
}
|
|
catch (error) {
|
|
// Skip items that can't be processed
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return connections.filter(conn => conn.strength > 0.5);
|
|
}
|
|
/**
|
|
* Merge semantic clusters based on connections
|
|
*/
|
|
async _mergeSemanticClusters(clusters, connections) {
|
|
// Simple merging based on strong connections
|
|
const merged = [...clusters];
|
|
for (const connection of connections) {
|
|
if (connection.strength > 0.8) {
|
|
const fromCluster = merged.find(c => c.members.includes(connection.from));
|
|
const toCluster = merged.find(c => c.members.includes(connection.to));
|
|
if (fromCluster && toCluster && fromCluster !== toCluster) {
|
|
// Merge clusters
|
|
fromCluster.members = [...fromCluster.members, ...toCluster.members];
|
|
fromCluster.size = fromCluster.members.length;
|
|
fromCluster.label = `merged ${fromCluster.label}`;
|
|
// Remove merged cluster
|
|
const index = merged.indexOf(toCluster);
|
|
if (index > -1)
|
|
merged.splice(index, 1);
|
|
}
|
|
}
|
|
}
|
|
return merged;
|
|
}
|
|
/**
|
|
* Get optimal clustering level for HNSW
|
|
*/
|
|
_getOptimalClusteringLevel(totalItems) {
|
|
if (totalItems < 100)
|
|
return 0;
|
|
if (totalItems < 1000)
|
|
return 1;
|
|
if (totalItems < 10000)
|
|
return 2;
|
|
return 3;
|
|
}
|
|
/**
|
|
* Get nodes at HNSW level
|
|
*/
|
|
async _getHNSWLevelNodes(level) {
|
|
// This would use the HNSW index to get nodes at specified level
|
|
// For now, return a sample of all items
|
|
const allItems = await this._getAllItemIds();
|
|
const sampleSize = Math.max(10, Math.floor(allItems.length / Math.pow(2, level + 1)));
|
|
return this._getRandomSample(allItems, sampleSize);
|
|
}
|
|
/**
|
|
* Find cluster members using HNSW neighbors
|
|
*/
|
|
async _findClusterMembers(levelNode, _allItems, threshold) {
|
|
try {
|
|
const neighbors = await this.brain.neural.neighbors(levelNode, {
|
|
limit: Math.min(50, Math.floor(_allItems.length / 10)),
|
|
minSimilarity: threshold
|
|
});
|
|
return [levelNode, ...neighbors.neighbors.map((n) => n.id)];
|
|
}
|
|
catch (error) {
|
|
return [levelNode];
|
|
}
|
|
}
|
|
/**
|
|
* Calculate hierarchical clustering confidence
|
|
*/
|
|
async _calculateHierarchicalConfidence(members) {
|
|
if (members.length <= 1)
|
|
return 1.0;
|
|
const items = await this._getItemsWithVectors(members);
|
|
const coherence = await this._calculateVectorCoherence(items);
|
|
return coherence;
|
|
}
|
|
/**
|
|
* Assign unassigned items to nearest clusters
|
|
*/
|
|
async _assignUnassignedItems(unassigned, clusters) {
|
|
for (const itemId of unassigned) {
|
|
if (clusters.length === 0)
|
|
break;
|
|
try {
|
|
const noun = await this.brain.getNoun(itemId);
|
|
const itemVector = noun?.vector || [];
|
|
if (itemVector.length === 0)
|
|
continue;
|
|
let bestCluster = clusters[0];
|
|
let minDistance = Infinity;
|
|
for (const cluster of clusters) {
|
|
const distance = Math.sqrt(this._calculateSquaredDistance(itemVector, cluster.centroid));
|
|
if (distance < minDistance) {
|
|
minDistance = distance;
|
|
bestCluster = cluster;
|
|
}
|
|
}
|
|
bestCluster.members.push(itemId);
|
|
bestCluster.size++;
|
|
}
|
|
catch (error) {
|
|
// Skip items that can't be processed
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
//# sourceMappingURL=improvedNeuralAPI.js.map
|