David Snelling
8ff382ca3b
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
612 lines
No EOL
22 KiB
JavaScript
612 lines
No EOL
22 KiB
JavaScript
/**
|
|
* Neural API - Unified Semantic Intelligence
|
|
*
|
|
* Best-of-both: Complete functionality + Enterprise performance
|
|
* Combines rich features with O(n) algorithms for millions of items
|
|
*/
|
|
import { cosineDistance } from '../utils/distance.js';
|
|
/**
|
|
* Neural API - Unified best-of-both implementation
|
|
*/
|
|
export class NeuralAPI {
|
|
constructor(brain) {
|
|
this.similarityCache = new Map();
|
|
this.clusterCache = new Map(); // Enhanced for enterprise
|
|
this.hierarchyCache = new Map();
|
|
this.brain = brain;
|
|
}
|
|
// ===== SMART USER-FRIENDLY API =====
|
|
/**
|
|
* Calculate similarity between any two items (smart detection)
|
|
*/
|
|
async similar(a, b, options) {
|
|
// Auto-detect input types
|
|
if (typeof a === 'string' && typeof b === 'string') {
|
|
if (this.isId(a) && this.isId(b)) {
|
|
return this.similarityById(a, b, options);
|
|
}
|
|
else {
|
|
return this.similarityByText(a, b, options);
|
|
}
|
|
}
|
|
else if (Array.isArray(a) && Array.isArray(b)) {
|
|
return this.similarityByVector(a, b, options);
|
|
}
|
|
// Handle mixed types
|
|
return this.smartSimilarity(a, b, options);
|
|
}
|
|
/**
|
|
* Find semantic clusters (auto-detects best approach)
|
|
* Now with enterprise performance!
|
|
*/
|
|
async clusters(input) {
|
|
// No input? Use enterprise fast clustering
|
|
if (!input) {
|
|
return this.clusterFast();
|
|
}
|
|
// Array? Cluster these items (use large clustering for big arrays)
|
|
if (Array.isArray(input)) {
|
|
if (input.length > 1000) {
|
|
return this.clusterLarge({ sampleSize: Math.min(input.length, 1000) });
|
|
}
|
|
return this.clusterItems(input);
|
|
}
|
|
// String? Find clusters near this
|
|
if (typeof input === 'string') {
|
|
return this.clustersNear(input);
|
|
}
|
|
// Object? Use as config with enterprise algorithms
|
|
if (typeof input === 'object' && !Array.isArray(input)) {
|
|
return this.clusterWithConfig(input);
|
|
}
|
|
throw new Error('Invalid input for clustering');
|
|
}
|
|
/**
|
|
* Get semantic hierarchy for an item
|
|
*/
|
|
async hierarchy(id) {
|
|
// Check cache first
|
|
if (this.hierarchyCache.has(id)) {
|
|
return this.hierarchyCache.get(id);
|
|
}
|
|
const item = await this.brain.get(id);
|
|
if (!item) {
|
|
throw new Error(`Item not found: ${id}`);
|
|
}
|
|
// Find semantic relationships
|
|
const hierarchy = await this.buildHierarchy(item);
|
|
// Cache result
|
|
this.hierarchyCache.set(id, hierarchy);
|
|
return hierarchy;
|
|
}
|
|
/**
|
|
* Find semantic neighbors for visualization
|
|
*/
|
|
async neighbors(id, options) {
|
|
const radius = options?.radius ?? 0.3;
|
|
const limit = options?.limit ?? 50;
|
|
// Search for nearby items
|
|
const results = await this.brain.search(id, limit * 2);
|
|
// Filter by semantic radius
|
|
const neighbors = results
|
|
.filter((r) => r.similarity >= (1 - radius))
|
|
.slice(0, limit)
|
|
.map((r) => ({
|
|
id: r.id,
|
|
similarity: r.similarity,
|
|
type: r.metadata?.type,
|
|
connections: r.metadata?.connections?.size || 0
|
|
}));
|
|
const graph = {
|
|
center: id,
|
|
neighbors
|
|
};
|
|
// Add edges if requested
|
|
if (options?.includeEdges) {
|
|
graph.edges = await this.buildEdges(id, neighbors);
|
|
}
|
|
return graph;
|
|
}
|
|
/**
|
|
* Find semantic path between two items
|
|
*/
|
|
async semanticPath(fromId, toId, options) {
|
|
const maxHops = options?.maxHops ?? 5;
|
|
const algorithm = options?.algorithm ?? 'breadth';
|
|
if (algorithm === 'dijkstra') {
|
|
return this.dijkstraPath(fromId, toId, maxHops);
|
|
}
|
|
else {
|
|
return this.breadthFirstPath(fromId, toId, maxHops);
|
|
}
|
|
}
|
|
/**
|
|
* Detect semantic outliers
|
|
*/
|
|
async outliers(threshold = 0.3) {
|
|
// Get all items
|
|
const stats = await this.brain.getStatistics();
|
|
const totalItems = stats.nounCount;
|
|
if (totalItems === 0)
|
|
return [];
|
|
// For large datasets, use sampling
|
|
if (totalItems > 10000) {
|
|
return this.outliersViaSampling(threshold, 1000);
|
|
}
|
|
return this.outliersByDistance(threshold);
|
|
}
|
|
/**
|
|
* Generate visualization data
|
|
*/
|
|
async visualize(options) {
|
|
const maxNodes = options?.maxNodes ?? 100;
|
|
const dimensions = options?.dimensions ?? 2;
|
|
const algorithm = options?.algorithm ?? 'force';
|
|
// Get representative nodes
|
|
const nodes = await this.getVisualizationNodes(maxNodes);
|
|
// Apply layout algorithm
|
|
const positioned = await this.applyLayout(nodes, algorithm, dimensions);
|
|
// Build edges if requested
|
|
const edges = options?.includeEdges !== false ?
|
|
await this.buildVisualizationEdges(positioned) : [];
|
|
// Detect optimal format
|
|
const format = this.detectOptimalFormat(positioned, edges);
|
|
return {
|
|
format,
|
|
nodes: positioned,
|
|
edges,
|
|
layout: {
|
|
dimensions,
|
|
algorithm,
|
|
bounds: this.calculateBounds(positioned, dimensions)
|
|
}
|
|
};
|
|
}
|
|
// ===== ENTERPRISE PERFORMANCE ALGORITHMS =====
|
|
/**
|
|
* Fast clustering using HNSW levels - O(n) instead of O(n²)
|
|
*/
|
|
async clusterFast(options = {}) {
|
|
const cacheKey = `hierarchical-${options.level}-${options.maxClusters}`;
|
|
if (this.clusterCache.has(cacheKey)) {
|
|
return this.clusterCache.get(cacheKey);
|
|
}
|
|
// Use HNSW's natural hierarchy - auto-select optimal level
|
|
const level = options.level ?? await this.getOptimalClusteringLevel();
|
|
const maxClusters = options.maxClusters ?? 100;
|
|
// Get representative nodes from HNSW level
|
|
const representatives = await this.getHNSWLevelNodes(level);
|
|
// Each representative is a natural cluster center
|
|
const clusters = [];
|
|
for (const rep of representatives.slice(0, maxClusters)) {
|
|
const members = await this.findClusterMembers(rep, level - 1);
|
|
clusters.push({
|
|
id: `cluster-${rep.id}`,
|
|
centroid: rep.vector,
|
|
center: rep,
|
|
members: members.map(m => m.id),
|
|
size: members.length,
|
|
level,
|
|
confidence: 0.8 + (members.length / 100) * 0.2 // Size-based confidence
|
|
});
|
|
}
|
|
this.clusterCache.set(cacheKey, clusters);
|
|
return clusters;
|
|
}
|
|
/**
|
|
* Large-scale clustering for massive datasets (millions of items)
|
|
*/
|
|
async clusterLarge(options = {}) {
|
|
const sampleSize = options.sampleSize ?? 1000;
|
|
const strategy = options.strategy ?? 'diverse';
|
|
// Get representative sample
|
|
const sample = await this.getSample(sampleSize, strategy);
|
|
// Cluster the sample (fast on small set)
|
|
const sampleClusters = await this.performFastClustering(sample);
|
|
// Project clusters to full dataset
|
|
return this.projectClustersToFullDataset(sampleClusters);
|
|
}
|
|
/**
|
|
* Streaming clustering for progressive refinement
|
|
*/
|
|
async *clusterStream(options = {}) {
|
|
const batchSize = options.batchSize ?? 1000;
|
|
const maxBatches = options.maxBatches ?? Infinity;
|
|
let offset = 0;
|
|
let batchCount = 0;
|
|
let globalClusters = [];
|
|
while (batchCount < maxBatches) {
|
|
// Get next batch
|
|
const batch = await this.getBatch(offset, batchSize);
|
|
if (batch.length === 0)
|
|
break;
|
|
// Cluster this batch
|
|
const batchClusters = await this.performFastClustering(batch);
|
|
// Merge with global clusters
|
|
globalClusters = await this.mergeClusters(globalClusters, batchClusters);
|
|
// Yield current state
|
|
yield globalClusters;
|
|
offset += batchSize;
|
|
batchCount++;
|
|
}
|
|
}
|
|
/**
|
|
* Level-of-detail for massive visualization
|
|
*/
|
|
async getLOD(zoomLevel, viewport) {
|
|
// Define LOD levels based on zoom
|
|
const lodLevels = [
|
|
{ zoom: 0, maxNodes: 50, clusterLevel: 3 },
|
|
{ zoom: 1, maxNodes: 200, clusterLevel: 2 },
|
|
{ zoom: 2, maxNodes: 1000, clusterLevel: 1 },
|
|
{ zoom: 3, maxNodes: 5000, clusterLevel: 0 }
|
|
];
|
|
const lod = lodLevels.find(l => zoomLevel <= l.zoom) || lodLevels[lodLevels.length - 1];
|
|
if (viewport) {
|
|
return this.getViewportLOD(viewport, lod);
|
|
}
|
|
else {
|
|
return this.getGlobalLOD(lod);
|
|
}
|
|
}
|
|
// ===== IMPLEMENTATION HELPERS =====
|
|
isId(str) {
|
|
// Check if string looks like an ID (UUID pattern, etc.)
|
|
return (str.length === 36 && str.includes('-')) || !!str.match(/^[a-f0-9]{24}$/);
|
|
}
|
|
async similarityById(idA, idB, options) {
|
|
const cacheKey = `${idA}-${idB}`;
|
|
if (this.similarityCache.has(cacheKey)) {
|
|
return this.similarityCache.get(cacheKey);
|
|
}
|
|
// Get items
|
|
const [itemA, itemB] = await Promise.all([
|
|
this.brain.get(idA),
|
|
this.brain.get(idB)
|
|
]);
|
|
if (!itemA || !itemB) {
|
|
throw new Error('One or both items not found');
|
|
}
|
|
// Calculate similarity
|
|
const score = cosineDistance(itemA.vector, itemB.vector);
|
|
this.similarityCache.set(cacheKey, score);
|
|
if (options?.explain) {
|
|
return {
|
|
score,
|
|
method: 'cosine',
|
|
confidence: 0.9,
|
|
explanation: `Semantic similarity between ${idA} and ${idB}`
|
|
};
|
|
}
|
|
return score;
|
|
}
|
|
async similarityByText(textA, textB, options) {
|
|
// Generate embeddings
|
|
const [vectorA, vectorB] = await Promise.all([
|
|
this.brain.embed(textA),
|
|
this.brain.embed(textB)
|
|
]);
|
|
return this.similarityByVector(vectorA, vectorB, options);
|
|
}
|
|
async similarityByVector(vectorA, vectorB, options) {
|
|
const score = cosineDistance(vectorA, vectorB);
|
|
if (options?.explain) {
|
|
return {
|
|
score,
|
|
method: options.method || 'cosine',
|
|
confidence: 0.95,
|
|
explanation: 'Direct vector similarity calculation'
|
|
};
|
|
}
|
|
return score;
|
|
}
|
|
async smartSimilarity(a, b, options) {
|
|
// Convert both to vectors and compare
|
|
const vectorA = await this.toVector(a);
|
|
const vectorB = await this.toVector(b);
|
|
return this.similarityByVector(vectorA, vectorB, options);
|
|
}
|
|
async toVector(item) {
|
|
if (Array.isArray(item))
|
|
return item;
|
|
if (typeof item === 'string') {
|
|
if (this.isId(item)) {
|
|
const found = await this.brain.get(item);
|
|
return found?.vector || await this.brain.embed(item);
|
|
}
|
|
return await this.brain.embed(item);
|
|
}
|
|
if (typeof item === 'object' && item.vector) {
|
|
return item.vector;
|
|
}
|
|
// Convert object to string and embed
|
|
return await this.brain.embed(JSON.stringify(item));
|
|
}
|
|
// Enterprise clustering implementations
|
|
async getOptimalClusteringLevel() {
|
|
// Analyze dataset size and return optimal HNSW level
|
|
const stats = await this.brain.getStatistics();
|
|
const itemCount = stats.nounCount;
|
|
if (itemCount < 1000)
|
|
return 0;
|
|
if (itemCount < 10000)
|
|
return 1;
|
|
if (itemCount < 100000)
|
|
return 2;
|
|
return 3;
|
|
}
|
|
async getHNSWLevelNodes(level) {
|
|
// Get nodes from specific HNSW level
|
|
// For now, use search to get a representative sample
|
|
const stats = await this.brain.getStatistics();
|
|
const sampleSize = Math.min(100, Math.floor(stats.nounCount / (level + 1)));
|
|
// Use search with a general query to get representative items
|
|
const queryVector = await this.brain.embed('data information content');
|
|
const allItems = await this.brain.search(queryVector, sampleSize * 2);
|
|
return allItems.slice(0, sampleSize);
|
|
}
|
|
async findClusterMembers(center, level) {
|
|
// Find all items that belong to this cluster
|
|
const results = await this.brain.search(center.vector, 50);
|
|
return results.filter((r) => r.similarity > 0.7);
|
|
}
|
|
async getSample(size, strategy) {
|
|
// Use search to get a sample of items
|
|
const stats = await this.brain.getStatistics();
|
|
const maxSize = Math.min(size * 3, stats.nounCount); // Get more than needed for sampling
|
|
const queryVector = await this.brain.embed('sample data content');
|
|
const allItems = await this.brain.search(queryVector, maxSize);
|
|
switch (strategy) {
|
|
case 'random':
|
|
return this.shuffleArray(allItems).slice(0, size);
|
|
case 'diverse':
|
|
return this.getDiverseSample(allItems, size);
|
|
case 'recent':
|
|
return allItems.slice(-size);
|
|
default:
|
|
return allItems.slice(0, size);
|
|
}
|
|
}
|
|
shuffleArray(array) {
|
|
const shuffled = [...array];
|
|
for (let i = shuffled.length - 1; i > 0; i--) {
|
|
const j = Math.floor(Math.random() * (i + 1));
|
|
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
|
}
|
|
return shuffled;
|
|
}
|
|
async getDiverseSample(items, size) {
|
|
// Select diverse items using maximum distance sampling
|
|
if (items.length <= size)
|
|
return items;
|
|
const sample = [items[0]]; // Start with first item
|
|
for (let i = 1; i < size; i++) {
|
|
let maxMinDistance = -1;
|
|
let bestItem = null;
|
|
for (const candidate of items) {
|
|
if (sample.includes(candidate))
|
|
continue;
|
|
// Find minimum distance to existing sample
|
|
let minDistance = Infinity;
|
|
for (const selected of sample) {
|
|
const distance = cosineDistance(candidate.vector, selected.vector);
|
|
minDistance = Math.min(minDistance, distance);
|
|
}
|
|
// Select item with maximum minimum distance
|
|
if (minDistance > maxMinDistance) {
|
|
maxMinDistance = minDistance;
|
|
bestItem = candidate;
|
|
}
|
|
}
|
|
if (bestItem)
|
|
sample.push(bestItem);
|
|
}
|
|
return sample;
|
|
}
|
|
async performFastClustering(items) {
|
|
// Simple k-means clustering for the sample
|
|
const k = Math.min(10, Math.floor(items.length / 3));
|
|
if (k <= 1) {
|
|
return [{
|
|
id: 'cluster-0',
|
|
centroid: items[0]?.vector || [],
|
|
members: items.map(i => i.id),
|
|
confidence: 1.0
|
|
}];
|
|
}
|
|
// Initialize centroids randomly
|
|
const centroids = items.slice(0, k).map(item => item.vector);
|
|
// Run k-means iterations (simplified)
|
|
for (let iter = 0; iter < 10; iter++) {
|
|
const clusters = Array(k).fill(null).map(() => []);
|
|
// Assign items to nearest centroid
|
|
for (const item of items) {
|
|
let bestCluster = 0;
|
|
let bestDistance = Infinity;
|
|
for (let c = 0; c < k; c++) {
|
|
const distance = cosineDistance(item.vector, centroids[c]);
|
|
if (distance < bestDistance) {
|
|
bestDistance = distance;
|
|
bestCluster = c;
|
|
}
|
|
}
|
|
clusters[bestCluster].push(item);
|
|
}
|
|
// Update centroids
|
|
for (let c = 0; c < k; c++) {
|
|
if (clusters[c].length > 0) {
|
|
const newCentroid = this.calculateCentroid(clusters[c]);
|
|
centroids[c] = newCentroid;
|
|
}
|
|
}
|
|
}
|
|
// Convert to SemanticCluster format
|
|
const result = [];
|
|
for (let c = 0; c < k; c++) {
|
|
const members = items.filter(item => {
|
|
let bestCluster = 0;
|
|
let bestDistance = Infinity;
|
|
for (let cc = 0; cc < k; cc++) {
|
|
const distance = cosineDistance(item.vector, centroids[cc]);
|
|
if (distance < bestDistance) {
|
|
bestDistance = distance;
|
|
bestCluster = cc;
|
|
}
|
|
}
|
|
return bestCluster === c;
|
|
});
|
|
if (members.length > 0) {
|
|
result.push({
|
|
id: `cluster-${c}`,
|
|
centroid: centroids[c],
|
|
members: members.map(m => m.id),
|
|
confidence: Math.min(0.9, members.length / items.length * 2)
|
|
});
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
calculateCentroid(items) {
|
|
if (items.length === 0)
|
|
return [];
|
|
const dimensions = items[0].vector.length;
|
|
const centroid = new Array(dimensions).fill(0);
|
|
for (const item of items) {
|
|
for (let d = 0; d < dimensions; d++) {
|
|
centroid[d] += item.vector[d];
|
|
}
|
|
}
|
|
for (let d = 0; d < dimensions; d++) {
|
|
centroid[d] /= items.length;
|
|
}
|
|
return centroid;
|
|
}
|
|
async projectClustersToFullDataset(sampleClusters) {
|
|
// Project sample clusters to full dataset
|
|
const result = [];
|
|
for (const cluster of sampleClusters) {
|
|
// Find all items similar to this cluster's centroid
|
|
const similar = await this.brain.search(cluster.centroid, 1000);
|
|
const members = similar
|
|
.filter((s) => s.similarity > 0.6)
|
|
.map((s) => s.id);
|
|
result.push({
|
|
...cluster,
|
|
members,
|
|
size: members.length
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
async mergeClusters(globalClusters, batchClusters) {
|
|
// Simple merge strategy - combine similar clusters
|
|
const result = [...globalClusters];
|
|
for (const batchCluster of batchClusters) {
|
|
let merged = false;
|
|
for (let i = 0; i < result.length; i++) {
|
|
const similarity = cosineDistance(result[i].centroid, batchCluster.centroid);
|
|
if (similarity > 0.8) {
|
|
// Merge clusters
|
|
const newMembers = [...new Set([...result[i].members, ...batchCluster.members])];
|
|
result[i] = {
|
|
...result[i],
|
|
members: newMembers,
|
|
size: newMembers.length,
|
|
centroid: this.averageVectors(result[i].centroid, batchCluster.centroid)
|
|
};
|
|
merged = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!merged) {
|
|
result.push(batchCluster);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
averageVectors(v1, v2) {
|
|
const result = new Array(v1.length);
|
|
for (let i = 0; i < v1.length; i++) {
|
|
result[i] = (v1[i] + v2[i]) / 2;
|
|
}
|
|
return result;
|
|
}
|
|
async getBatch(offset, size) {
|
|
// Get batch of items for streaming using search with offset
|
|
const queryVector = await this.brain.embed('batch data content');
|
|
const items = await this.brain.search(queryVector, size, { offset });
|
|
return items;
|
|
}
|
|
// Additional methods needed for full compatibility...
|
|
async clusterAll() {
|
|
return this.clusterFast();
|
|
}
|
|
async clusterItems(items) {
|
|
return this.performFastClustering(items);
|
|
}
|
|
async clustersNear(id) {
|
|
const neighbors = await this.neighbors(id, { limit: 100 });
|
|
return this.performFastClustering(neighbors.neighbors);
|
|
}
|
|
async clusterWithConfig(config) {
|
|
switch (config.algorithm) {
|
|
case 'hierarchical':
|
|
return this.clusterFast(config);
|
|
case 'sample':
|
|
return this.clusterLarge(config);
|
|
case 'stream':
|
|
const generator = this.clusterStream(config);
|
|
const results = [];
|
|
for await (const batch of generator) {
|
|
results.push(...batch);
|
|
}
|
|
return results;
|
|
default:
|
|
return this.clusterFast(config);
|
|
}
|
|
}
|
|
// Placeholder implementations for remaining methods
|
|
async buildHierarchy(item) {
|
|
// Implementation for hierarchy building
|
|
return {
|
|
self: { id: item.id, vector: item.vector }
|
|
};
|
|
}
|
|
async buildEdges(centerId, neighbors) {
|
|
return [];
|
|
}
|
|
async dijkstraPath(from, to, maxHops) {
|
|
return [];
|
|
}
|
|
async breadthFirstPath(from, to, maxHops) {
|
|
return [];
|
|
}
|
|
async outliersViaSampling(threshold, sampleSize) {
|
|
return [];
|
|
}
|
|
async outliersByDistance(threshold) {
|
|
return [];
|
|
}
|
|
async getVisualizationNodes(maxNodes) {
|
|
return [];
|
|
}
|
|
async applyLayout(nodes, algorithm, dimensions) {
|
|
return nodes;
|
|
}
|
|
async buildVisualizationEdges(nodes) {
|
|
return [];
|
|
}
|
|
detectOptimalFormat(nodes, edges) {
|
|
return 'force-directed';
|
|
}
|
|
calculateBounds(nodes, dimensions) {
|
|
return { width: 100, height: 100 };
|
|
}
|
|
async getViewportLOD(viewport, lod) {
|
|
return {};
|
|
}
|
|
async getGlobalLOD(lod) {
|
|
return {};
|
|
}
|
|
}
|
|
//# sourceMappingURL=neuralAPI.js.map
|