chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
41
.recovery-workspace/dist-backup-20250910-141917/neural/embeddedPatterns.d.ts
vendored
Normal file
41
.recovery-workspace/dist-backup-20250910-141917/neural/embeddedPatterns.d.ts
vendored
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* 🧠 BRAINY EMBEDDED PATTERNS
|
||||
*
|
||||
* AUTO-GENERATED - DO NOT EDIT
|
||||
* Generated: 2025-09-09T17:55:32.179Z
|
||||
* Patterns: 220
|
||||
* Coverage: 94-98% of all queries
|
||||
*
|
||||
* This file contains ALL patterns and embeddings compiled into Brainy.
|
||||
* No external files needed, no runtime loading, instant availability!
|
||||
*/
|
||||
import type { Pattern } from './patternLibrary.js';
|
||||
export declare const EMBEDDED_PATTERNS: Pattern[];
|
||||
/**
|
||||
* Get pattern embeddings as a Map for fast lookup
|
||||
* This is called once at startup and cached
|
||||
*/
|
||||
export declare function getPatternEmbeddings(): Map<string, Float32Array>;
|
||||
export declare const PATTERNS_METADATA: {
|
||||
version: string;
|
||||
totalPatterns: number;
|
||||
categories: string[];
|
||||
domains: string[];
|
||||
embeddingDimensions: number;
|
||||
averageConfidence: number;
|
||||
coverage: {
|
||||
general: string;
|
||||
programming: string;
|
||||
ai_ml: string;
|
||||
social: string;
|
||||
medical_legal: string;
|
||||
financial_academic: string;
|
||||
ecommerce: string;
|
||||
overall: string;
|
||||
};
|
||||
sizeBytes: {
|
||||
patterns: number;
|
||||
embeddings: number;
|
||||
total: number;
|
||||
};
|
||||
};
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
65
.recovery-workspace/dist-backup-20250910-141917/neural/entityExtractor.d.ts
vendored
Normal file
65
.recovery-workspace/dist-backup-20250910-141917/neural/entityExtractor.d.ts
vendored
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
/**
|
||||
* Neural Entity Extractor using Brainy's NounTypes
|
||||
* Uses embeddings and similarity matching for accurate type detection
|
||||
*/
|
||||
import { NounType } from '../types/graphTypes.js';
|
||||
import { Vector } from '../coreTypes.js';
|
||||
import type { Brainy } from '../brainy.js';
|
||||
export interface ExtractedEntity {
|
||||
text: string;
|
||||
type: NounType;
|
||||
position: {
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
confidence: number;
|
||||
vector?: Vector;
|
||||
metadata?: any;
|
||||
}
|
||||
export declare class NeuralEntityExtractor {
|
||||
private brain;
|
||||
private typeEmbeddings;
|
||||
private initialized;
|
||||
constructor(brain: Brainy | Brainy<any>);
|
||||
/**
|
||||
* Initialize type embeddings for neural matching
|
||||
*/
|
||||
private initializeTypeEmbeddings;
|
||||
/**
|
||||
* Extract entities from text using neural matching
|
||||
*/
|
||||
extract(text: string, options?: {
|
||||
types?: NounType[];
|
||||
confidence?: number;
|
||||
includeVectors?: boolean;
|
||||
neuralMatching?: boolean;
|
||||
}): Promise<ExtractedEntity[]>;
|
||||
/**
|
||||
* Extract candidate entities using patterns
|
||||
*/
|
||||
private extractCandidates;
|
||||
/**
|
||||
* Get context-based confidence boost for type matching
|
||||
*/
|
||||
private getContextBoost;
|
||||
/**
|
||||
* Rule-based classification fallback
|
||||
*/
|
||||
private classifyByRules;
|
||||
/**
|
||||
* Get embedding for text
|
||||
*/
|
||||
private getEmbedding;
|
||||
/**
|
||||
* Calculate cosine similarity between vectors
|
||||
*/
|
||||
private cosineSimilarity;
|
||||
/**
|
||||
* Simple hash function for fallback
|
||||
*/
|
||||
private simpleHash;
|
||||
/**
|
||||
* Remove duplicate and overlapping entities
|
||||
*/
|
||||
private deduplicateEntities;
|
||||
}
|
||||
|
|
@ -0,0 +1,316 @@
|
|||
/**
|
||||
* Neural Entity Extractor using Brainy's NounTypes
|
||||
* Uses embeddings and similarity matching for accurate type detection
|
||||
*/
|
||||
import { NounType } from '../types/graphTypes.js';
|
||||
export class NeuralEntityExtractor {
|
||||
constructor(brain) {
|
||||
// Type embeddings for similarity matching
|
||||
this.typeEmbeddings = new Map();
|
||||
this.initialized = false;
|
||||
this.brain = brain;
|
||||
}
|
||||
/**
|
||||
* Initialize type embeddings for neural matching
|
||||
*/
|
||||
async initializeTypeEmbeddings() {
|
||||
if (this.initialized)
|
||||
return;
|
||||
// Create representative embeddings for each NounType
|
||||
const typeExamples = {
|
||||
[NounType.Person]: ['John Smith', 'Jane Doe', 'person', 'individual', 'human'],
|
||||
[NounType.Organization]: ['Microsoft Corporation', 'company', 'organization', 'business', 'enterprise'],
|
||||
[NounType.Location]: ['New York City', 'location', 'place', 'address', 'geography'],
|
||||
[NounType.Document]: ['document', 'file', 'report', 'paper', 'text'],
|
||||
[NounType.Event]: ['conference', 'meeting', 'event', 'occurrence', 'happening'],
|
||||
[NounType.Product]: ['iPhone', 'product', 'item', 'merchandise', 'goods'],
|
||||
[NounType.Service]: ['consulting', 'service', 'offering', 'provision'],
|
||||
[NounType.Concept]: ['idea', 'concept', 'theory', 'principle', 'notion'],
|
||||
[NounType.Media]: ['image', 'video', 'audio', 'media', 'content'],
|
||||
[NounType.Message]: ['email', 'message', 'communication', 'note'],
|
||||
[NounType.Task]: ['task', 'todo', 'assignment', 'job', 'work'],
|
||||
[NounType.Project]: ['project', 'initiative', 'program', 'endeavor'],
|
||||
[NounType.Process]: ['workflow', 'process', 'procedure', 'method'],
|
||||
[NounType.User]: ['user', 'account', 'profile', 'member'],
|
||||
[NounType.Role]: ['manager', 'role', 'position', 'title', 'responsibility'],
|
||||
[NounType.Topic]: ['subject', 'topic', 'theme', 'matter'],
|
||||
[NounType.Language]: ['English', 'language', 'tongue', 'dialect'],
|
||||
[NounType.Currency]: ['dollar', 'currency', 'money', 'USD', 'EUR'],
|
||||
[NounType.Measurement]: ['meter', 'measurement', 'unit', 'quantity'],
|
||||
[NounType.Contract]: ['agreement', 'contract', 'deal', 'treaty'],
|
||||
[NounType.Regulation]: ['law', 'regulation', 'rule', 'policy'],
|
||||
[NounType.Resource]: ['resource', 'asset', 'material', 'supply'],
|
||||
[NounType.Dataset]: ['database', 'dataset', 'data', 'records'],
|
||||
[NounType.Interface]: ['API', 'interface', 'endpoint', 'connection'],
|
||||
[NounType.Thing]: ['thing', 'object', 'item', 'entity'],
|
||||
[NounType.Content]: ['content', 'material', 'information'],
|
||||
[NounType.Collection]: ['collection', 'group', 'set', 'list'],
|
||||
[NounType.File]: ['file', 'document', 'archive'],
|
||||
[NounType.State]: ['state', 'status', 'condition'],
|
||||
[NounType.Hypothesis]: ['hypothesis', 'theory', 'assumption'],
|
||||
[NounType.Experiment]: ['experiment', 'test', 'trial', 'study']
|
||||
};
|
||||
// Generate embeddings for each type
|
||||
for (const [type, examples] of Object.entries(typeExamples)) {
|
||||
const combinedText = examples.join(' ');
|
||||
const embedding = await this.getEmbedding(combinedText);
|
||||
this.typeEmbeddings.set(type, embedding);
|
||||
}
|
||||
this.initialized = true;
|
||||
}
|
||||
/**
|
||||
* Extract entities from text using neural matching
|
||||
*/
|
||||
async extract(text, options) {
|
||||
await this.initializeTypeEmbeddings();
|
||||
const entities = [];
|
||||
const minConfidence = options?.confidence || 0.6;
|
||||
const targetTypes = options?.types || Object.values(NounType);
|
||||
const useNeuralMatching = options?.neuralMatching !== false; // Default true
|
||||
// Step 1: Extract potential entities using patterns
|
||||
const candidates = await this.extractCandidates(text);
|
||||
// Step 2: Classify each candidate using neural matching
|
||||
for (const candidate of candidates) {
|
||||
let bestType = NounType.Thing;
|
||||
let bestConfidence = 0;
|
||||
if (useNeuralMatching) {
|
||||
// Get embedding for the candidate
|
||||
const candidateVector = await this.getEmbedding(candidate.text);
|
||||
// Find best matching NounType
|
||||
for (const type of targetTypes) {
|
||||
const typeVector = this.typeEmbeddings.get(type);
|
||||
if (!typeVector)
|
||||
continue;
|
||||
const similarity = this.cosineSimilarity(candidateVector, typeVector);
|
||||
// Apply context-based boosting
|
||||
const contextBoost = this.getContextBoost(candidate.text, candidate.context, type);
|
||||
const adjustedConfidence = similarity * (1 + contextBoost);
|
||||
if (adjustedConfidence > bestConfidence) {
|
||||
bestConfidence = adjustedConfidence;
|
||||
bestType = type;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Fallback to rule-based classification
|
||||
const classification = this.classifyByRules(candidate);
|
||||
bestType = classification.type;
|
||||
bestConfidence = classification.confidence;
|
||||
}
|
||||
if (bestConfidence >= minConfidence) {
|
||||
const entity = {
|
||||
text: candidate.text,
|
||||
type: bestType,
|
||||
position: candidate.position,
|
||||
confidence: bestConfidence
|
||||
};
|
||||
if (options?.includeVectors) {
|
||||
entity.vector = await this.getEmbedding(candidate.text);
|
||||
}
|
||||
entities.push(entity);
|
||||
}
|
||||
}
|
||||
// Remove duplicates and overlaps
|
||||
return this.deduplicateEntities(entities);
|
||||
}
|
||||
/**
|
||||
* Extract candidate entities using patterns
|
||||
*/
|
||||
async extractCandidates(text) {
|
||||
const candidates = [];
|
||||
// Enhanced patterns for entity detection
|
||||
const patterns = [
|
||||
// Capitalized words (potential names, places, organizations)
|
||||
/\b([A-Z][a-zA-Z]+(?:\s+[A-Z][a-zA-Z]+)*)\b/g,
|
||||
// Email addresses
|
||||
/\b([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})\b/g,
|
||||
// URLs
|
||||
/\b(https?:\/\/[^\s]+|www\.[^\s]+)\b/g,
|
||||
// Phone numbers
|
||||
/\b(\+?\d{1,3}?[- .]?\(?\d{1,4}\)?[- .]?\d{1,4}[- .]?\d{1,4})\b/g,
|
||||
// Dates
|
||||
/\b(\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4}|\d{4}[\/\-]\d{1,2}[\/\-]\d{1,2})\b/g,
|
||||
// Money amounts
|
||||
/\b(\$[\d,]+(?:\.\d{2})?|[\d,]+(?:\.\d{2})?\s*(?:USD|EUR|GBP|JPY|CNY))\b/gi,
|
||||
// Percentages
|
||||
/\b(\d+(?:\.\d+)?%)\b/g,
|
||||
// Hashtags and mentions
|
||||
/([#@][a-zA-Z0-9_]+)/g,
|
||||
// Product versions
|
||||
/\b([A-Z][a-zA-Z0-9]+\s+v?\d+(?:\.\d+)*)\b/g,
|
||||
// Quoted strings (potential names, titles)
|
||||
/"([^"]+)"/g,
|
||||
/'([^']+)'/g
|
||||
];
|
||||
for (const pattern of patterns) {
|
||||
let match;
|
||||
while ((match = pattern.exec(text)) !== null) {
|
||||
const extractedText = match[1] || match[0];
|
||||
// Skip too short or too long
|
||||
if (extractedText.length < 2 || extractedText.length > 100)
|
||||
continue;
|
||||
// Get context (surrounding text)
|
||||
const contextStart = Math.max(0, match.index - 30);
|
||||
const contextEnd = Math.min(text.length, match.index + match[0].length + 30);
|
||||
const context = text.substring(contextStart, contextEnd);
|
||||
candidates.push({
|
||||
text: extractedText,
|
||||
position: {
|
||||
start: match.index,
|
||||
end: match.index + match[0].length
|
||||
},
|
||||
context
|
||||
});
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
/**
|
||||
* Get context-based confidence boost for type matching
|
||||
*/
|
||||
getContextBoost(text, context, type) {
|
||||
const contextLower = context.toLowerCase();
|
||||
let boost = 0;
|
||||
// Context clues for each type
|
||||
const contextClues = {
|
||||
[NounType.Person]: ['mr', 'ms', 'mrs', 'dr', 'prof', 'said', 'told', 'wrote'],
|
||||
[NounType.Organization]: ['inc', 'corp', 'llc', 'ltd', 'company', 'announced'],
|
||||
[NounType.Location]: ['in', 'at', 'from', 'to', 'near', 'located', 'city', 'country'],
|
||||
[NounType.Document]: ['file', 'document', 'report', 'paper', 'pdf', 'doc'],
|
||||
[NounType.Event]: ['event', 'conference', 'meeting', 'summit', 'on', 'at'],
|
||||
[NounType.Product]: ['product', 'version', 'release', 'model', 'buy', 'sell'],
|
||||
[NounType.Currency]: ['$', '€', '£', '¥', 'usd', 'eur', 'price', 'cost'],
|
||||
[NounType.Message]: ['email', 'message', 'sent', 'received', 'wrote', 'reply'],
|
||||
// Add more context clues as needed
|
||||
};
|
||||
const clues = contextClues[type] || [];
|
||||
for (const clue of clues) {
|
||||
if (contextLower.includes(clue)) {
|
||||
boost += 0.1;
|
||||
}
|
||||
}
|
||||
return Math.min(boost, 0.3); // Cap boost at 0.3
|
||||
}
|
||||
/**
|
||||
* Rule-based classification fallback
|
||||
*/
|
||||
classifyByRules(candidate) {
|
||||
const text = candidate.text;
|
||||
// Email
|
||||
if (text.includes('@')) {
|
||||
return { type: NounType.Message, confidence: 0.9 };
|
||||
}
|
||||
// URL
|
||||
if (text.startsWith('http') || text.startsWith('www.')) {
|
||||
return { type: NounType.Resource, confidence: 0.9 };
|
||||
}
|
||||
// Money
|
||||
if (text.startsWith('$') || /\d+\.\d{2}/.test(text)) {
|
||||
return { type: NounType.Currency, confidence: 0.85 };
|
||||
}
|
||||
// Percentage
|
||||
if (text.endsWith('%')) {
|
||||
return { type: NounType.Measurement, confidence: 0.85 };
|
||||
}
|
||||
// Date pattern
|
||||
if (/\d{1,2}[\/\-]\d{1,2}/.test(text)) {
|
||||
return { type: NounType.Event, confidence: 0.7 };
|
||||
}
|
||||
// Hashtag
|
||||
if (text.startsWith('#')) {
|
||||
return { type: NounType.Topic, confidence: 0.8 };
|
||||
}
|
||||
// Mention
|
||||
if (text.startsWith('@')) {
|
||||
return { type: NounType.User, confidence: 0.8 };
|
||||
}
|
||||
// Capitalized words (likely proper nouns)
|
||||
if (/^[A-Z]/.test(text)) {
|
||||
// Multiple words - likely organization or person
|
||||
const words = text.split(/\s+/);
|
||||
if (words.length > 1) {
|
||||
// Check for organization suffixes
|
||||
if (/\b(Inc|Corp|LLC|Ltd|Co|Group|Foundation|University)\b/i.test(text)) {
|
||||
return { type: NounType.Organization, confidence: 0.75 };
|
||||
}
|
||||
// Likely a person's name
|
||||
return { type: NounType.Person, confidence: 0.65 };
|
||||
}
|
||||
// Single capitalized word - could be location
|
||||
return { type: NounType.Location, confidence: 0.5 };
|
||||
}
|
||||
// Default to Thing with low confidence
|
||||
return { type: NounType.Thing, confidence: 0.3 };
|
||||
}
|
||||
/**
|
||||
* Get embedding for text
|
||||
*/
|
||||
async getEmbedding(text) {
|
||||
if ('embed' in this.brain && typeof this.brain.embed === 'function') {
|
||||
return await this.brain.embed(text);
|
||||
}
|
||||
else {
|
||||
// Fallback - create simple hash-based vector
|
||||
const vector = new Array(384).fill(0);
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
vector[i % 384] += text.charCodeAt(i) / 255;
|
||||
}
|
||||
return vector.map(v => v / text.length);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Calculate cosine similarity between vectors
|
||||
*/
|
||||
cosineSimilarity(a, b) {
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
normA = Math.sqrt(normA);
|
||||
normB = Math.sqrt(normB);
|
||||
if (normA === 0 || normB === 0)
|
||||
return 0;
|
||||
return dotProduct / (normA * normB);
|
||||
}
|
||||
/**
|
||||
* Simple hash function for fallback
|
||||
*/
|
||||
simpleHash(text) {
|
||||
let hash = 0;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const char = text.charCodeAt(i);
|
||||
hash = ((hash << 5) - hash) + char;
|
||||
hash = hash & hash; // Convert to 32-bit integer
|
||||
}
|
||||
return Math.abs(hash);
|
||||
}
|
||||
/**
|
||||
* Remove duplicate and overlapping entities
|
||||
*/
|
||||
deduplicateEntities(entities) {
|
||||
// Sort by position and confidence
|
||||
entities.sort((a, b) => {
|
||||
if (a.position.start !== b.position.start) {
|
||||
return a.position.start - b.position.start;
|
||||
}
|
||||
return b.confidence - a.confidence; // Higher confidence first
|
||||
});
|
||||
const result = [];
|
||||
for (const entity of entities) {
|
||||
// Check for overlap with already added entities
|
||||
const hasOverlap = result.some(existing => (entity.position.start >= existing.position.start &&
|
||||
entity.position.start < existing.position.end) ||
|
||||
(entity.position.end > existing.position.start &&
|
||||
entity.position.end <= existing.position.end));
|
||||
if (!hasOverlap) {
|
||||
result.push(entity);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=entityExtractor.js.map
|
||||
File diff suppressed because one or more lines are too long
357
.recovery-workspace/dist-backup-20250910-141917/neural/improvedNeuralAPI.d.ts
vendored
Normal file
357
.recovery-workspace/dist-backup-20250910-141917/neural/improvedNeuralAPI.d.ts
vendored
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
/**
|
||||
* Improved Neural API - Clean, Consistent, Performant
|
||||
*
|
||||
* Public API Surface:
|
||||
* - brain.neural.similar(a, b, options?) // Similarity calculation
|
||||
* - brain.neural.clusters(items?, options?) // Semantic clustering
|
||||
* - brain.neural.neighbors(id, options?) // K-nearest neighbors
|
||||
* - brain.neural.hierarchy(id, options?) // Semantic hierarchy
|
||||
* - brain.neural.outliers(options?) // Anomaly detection
|
||||
* - brain.neural.visualize(options?) // Visualization data
|
||||
*
|
||||
* Advanced Clustering:
|
||||
* - brain.neural.clusterByDomain(field, options?) // Domain-aware clustering
|
||||
* - brain.neural.clusterByTime(field, windows, options?) // Temporal clustering
|
||||
* - brain.neural.clusterStream(options?) // AsyncIterator for streaming
|
||||
* - brain.neural.updateClusters(items, options?) // Incremental clustering
|
||||
*
|
||||
* Private methods are prefixed with _ and not exposed in public API
|
||||
*/
|
||||
import { Vector } from '../coreTypes.js';
|
||||
import { SemanticCluster, DomainCluster, TemporalCluster, EnhancedSemanticCluster, SimilarityOptions, SimilarityResult, NeighborOptions, NeighborsResult, SemanticHierarchy, HierarchyOptions, ClusteringOptions, DomainClusteringOptions, TemporalClusteringOptions, StreamClusteringOptions, VisualizationOptions, VisualizationResult, OutlierOptions, Outlier, StreamingBatch, TimeWindow, PerformanceMetrics, NeuralAPIConfig } from './types.js';
|
||||
export declare class ImprovedNeuralAPI {
|
||||
private brain;
|
||||
private config;
|
||||
private similarityCache;
|
||||
private clusterCache;
|
||||
private hierarchyCache;
|
||||
private neighborsCache;
|
||||
private performanceMetrics;
|
||||
constructor(brain: any, config?: NeuralAPIConfig);
|
||||
/**
|
||||
* Calculate similarity between any two items (auto-detection)
|
||||
* Supports: IDs, text strings, vectors, or mixed types
|
||||
*/
|
||||
similar(a: string | Vector | any, b: string | Vector | any, options?: SimilarityOptions): Promise<number | SimilarityResult>;
|
||||
/**
|
||||
* Intelligent semantic clustering with auto-routing
|
||||
* - No input: Cluster all data
|
||||
* - Array: Cluster specific items
|
||||
* - String: Find clusters near this item
|
||||
* - Options object: Advanced configuration
|
||||
*/
|
||||
clusters(input?: string | string[] | ClusteringOptions): Promise<SemanticCluster[]>;
|
||||
/**
|
||||
* Fast hierarchical clustering using HNSW levels
|
||||
*/
|
||||
clusterFast(options?: {
|
||||
level?: number;
|
||||
maxClusters?: number;
|
||||
}): Promise<SemanticCluster[]>;
|
||||
/**
|
||||
* Large-scale clustering with intelligent sampling
|
||||
*/
|
||||
clusterLarge(options?: {
|
||||
sampleSize?: number;
|
||||
strategy?: 'random' | 'diverse' | 'recent';
|
||||
}): Promise<SemanticCluster[]>;
|
||||
/**
|
||||
* Domain-aware clustering based on metadata fields
|
||||
*/
|
||||
clusterByDomain(field: string, options?: DomainClusteringOptions): Promise<DomainCluster[]>;
|
||||
/**
|
||||
* Temporal clustering based on time windows
|
||||
*/
|
||||
clusterByTime(timeField: string, windows: TimeWindow[], options?: TemporalClusteringOptions): Promise<TemporalCluster[]>;
|
||||
/**
|
||||
* Streaming clustering with real-time updates
|
||||
*/
|
||||
clusterStream(options?: StreamClusteringOptions): AsyncIterableIterator<StreamingBatch>;
|
||||
/**
|
||||
* Incremental clustering - add new items to existing clusters
|
||||
*/
|
||||
updateClusters(newItems: string[], options?: ClusteringOptions): Promise<SemanticCluster[]>;
|
||||
/**
|
||||
* Enhanced clustering with relationship analysis using verbs
|
||||
* Returns clusters with intra-cluster and inter-cluster relationship information
|
||||
*
|
||||
* Scalable for millions of nodes - uses efficient batching and filtering
|
||||
*/
|
||||
clustersWithRelationships(input?: string | string[] | ClusteringOptions, options?: {
|
||||
batchSize?: number;
|
||||
maxRelationships?: number;
|
||||
}): Promise<EnhancedSemanticCluster[]>;
|
||||
/**
|
||||
* Find K-nearest semantic neighbors
|
||||
*/
|
||||
neighbors(id: string, options?: NeighborOptions): Promise<NeighborsResult>;
|
||||
/**
|
||||
* Build semantic hierarchy around an item
|
||||
*/
|
||||
hierarchy(id: string, options?: HierarchyOptions): Promise<SemanticHierarchy>;
|
||||
/**
|
||||
* Detect outliers and anomalous items
|
||||
*/
|
||||
outliers(options?: OutlierOptions): Promise<Outlier[]>;
|
||||
/**
|
||||
* Generate visualization data for graph libraries
|
||||
*/
|
||||
visualize(options?: VisualizationOptions): Promise<VisualizationResult>;
|
||||
private _routeClusteringAlgorithm;
|
||||
private _performClustering;
|
||||
/**
|
||||
* SEMANTIC-AWARE CLUSTERING: Uses existing NounType/VerbType taxonomy + HNSW
|
||||
*/
|
||||
private _performSemanticClustering;
|
||||
/**
|
||||
* HIERARCHICAL CLUSTERING: Uses existing HNSW levels for O(n) clustering
|
||||
*/
|
||||
private _performHierarchicalClustering;
|
||||
/**
|
||||
* K-MEANS CLUSTERING: Real implementation using existing distance functions
|
||||
*/
|
||||
private _performKMeansClustering;
|
||||
/**
|
||||
* DBSCAN CLUSTERING: Density-based clustering with adaptive parameters using HNSW
|
||||
*/
|
||||
private _performDBSCANClustering;
|
||||
/**
|
||||
* GRAPH COMMUNITY DETECTION: Uses existing verb relationships for clustering
|
||||
*/
|
||||
private _performGraphClustering;
|
||||
/**
|
||||
* MULTI-MODAL FUSION: Combines vector + graph + semantic + Triple Intelligence
|
||||
*/
|
||||
private _performMultiModalClustering;
|
||||
/**
|
||||
* SAMPLED CLUSTERING: For very large datasets using intelligent sampling
|
||||
*/
|
||||
private _performSampledClustering;
|
||||
private _similarityById;
|
||||
private _similarityByVector;
|
||||
private _similarityByText;
|
||||
private _isId;
|
||||
private _isVector;
|
||||
private _convertToVector;
|
||||
private _createSimilarityKey;
|
||||
private _createClusteringKey;
|
||||
private _cacheResult;
|
||||
private _trackPerformance;
|
||||
private _createPerformanceMetrics;
|
||||
private _initializeCleanupTimer;
|
||||
/**
|
||||
* Build graph structure from existing verb relationships
|
||||
*/
|
||||
private _buildGraphFromVerbs;
|
||||
/**
|
||||
* Detect communities using Louvain modularity optimization
|
||||
*/
|
||||
private _detectCommunities;
|
||||
/**
|
||||
* Refine community boundaries using vector similarity
|
||||
*/
|
||||
private _refineCommunitiesWithVectors;
|
||||
/**
|
||||
* Get items with their metadata including noun types
|
||||
*/
|
||||
private _getItemsWithMetadata;
|
||||
/**
|
||||
* Group items by their semantic noun types
|
||||
*/
|
||||
private _groupBySemanticType;
|
||||
private _getAllItemIds;
|
||||
private _getTotalItemCount;
|
||||
private _calculateTotalWeight;
|
||||
private _getNeighborCommunities;
|
||||
private _calculateModularityGain;
|
||||
private _getNodeDegree;
|
||||
private _getEdgesToCommunity;
|
||||
private _getCommunityWeight;
|
||||
private _calculateCommunityModularity;
|
||||
private _calculateCommunityDensity;
|
||||
private _findStrongestConnections;
|
||||
/**
|
||||
* Get items with their vector representations
|
||||
*/
|
||||
private _getItemsWithVectors;
|
||||
/**
|
||||
* Calculate centroid from items using existing distance functions
|
||||
*/
|
||||
private _calculateCentroidFromItems;
|
||||
/**
|
||||
* Initialize centroids using k-means++ algorithm for better convergence
|
||||
*/
|
||||
private _initializeCentroidsKMeansPlusPlus;
|
||||
/**
|
||||
* Assign points to nearest centroids using existing distance functions
|
||||
*/
|
||||
private _assignPointsToCentroids;
|
||||
/**
|
||||
* Update centroids based on current assignments
|
||||
*/
|
||||
private _updateCentroids;
|
||||
/**
|
||||
* Calculate how much assignments have changed between iterations
|
||||
*/
|
||||
private _calculateAssignmentChangeRate;
|
||||
/**
|
||||
* Calculate cluster confidence for k-means clusters
|
||||
*/
|
||||
private _calculateKMeansClusterConfidence;
|
||||
/**
|
||||
* Estimate optimal eps parameter using k-nearest neighbor distances
|
||||
*/
|
||||
private _estimateOptimalEps;
|
||||
/**
|
||||
* Find neighbors within epsilon distance using efficient vector operations
|
||||
*/
|
||||
private _findNeighborsWithinEps;
|
||||
/**
|
||||
* Expand DBSCAN cluster by adding density-reachable points
|
||||
*/
|
||||
private _expandCluster;
|
||||
/**
|
||||
* Calculate DBSCAN cluster confidence based on density
|
||||
*/
|
||||
private _calculateDBSCANClusterConfidence;
|
||||
/**
|
||||
* Calculate squared Euclidean distance (more efficient than sqrt)
|
||||
*/
|
||||
private _calculateSquaredDistance;
|
||||
/**
|
||||
* Calculate vector coherence for community refinement
|
||||
*/
|
||||
private _calculateVectorCoherence;
|
||||
private _getItemsByField;
|
||||
/**
|
||||
* Generate intelligent cluster labels using Triple Intelligence
|
||||
*/
|
||||
private _generateIntelligentClusterLabel;
|
||||
/**
|
||||
* Generate simple cluster labels based on semantic analysis
|
||||
*/
|
||||
private _generateClusterLabel;
|
||||
/**
|
||||
* Fuse clustering results using Triple Intelligence consensus
|
||||
*/
|
||||
private _fuseClusteringResultsWithTripleIntelligence;
|
||||
/**
|
||||
* Get items in a specific cluster from cluster sets
|
||||
*/
|
||||
private _getItemsInCluster;
|
||||
/**
|
||||
* Count co-occurrences between two sets of assignments
|
||||
*/
|
||||
private _countCoOccurrences;
|
||||
/**
|
||||
* Calculate fusion confidence based on algorithm agreement
|
||||
*/
|
||||
private _calculateFusionConfidence;
|
||||
/**
|
||||
* Generate empty clustering result for edge cases
|
||||
*/
|
||||
private _createEmptyResult;
|
||||
/**
|
||||
* Get sample using specified strategy for large dataset clustering
|
||||
*/
|
||||
private _getSampleUsingStrategy;
|
||||
/**
|
||||
* Random sampling
|
||||
*/
|
||||
private _getRandomSample;
|
||||
/**
|
||||
* Diverse sampling using vector space distribution
|
||||
*/
|
||||
private _getDiverseSample;
|
||||
/**
|
||||
* Recent sampling based on creation time
|
||||
*/
|
||||
private _getRecentSample;
|
||||
/**
|
||||
* Important sampling based on connection count and metadata
|
||||
*/
|
||||
private _getImportantSample;
|
||||
/**
|
||||
* Project clusters back to full dataset using HNSW neighbors
|
||||
*/
|
||||
private _projectClustersToFullDataset;
|
||||
private _groupByDomain;
|
||||
private _calculateDomainConfidence;
|
||||
private _findCrossDomainMembers;
|
||||
private _findCrossDomainClusters;
|
||||
private _getItemsByTimeWindow;
|
||||
private _calculateTemporalMetrics;
|
||||
private _mergeOverlappingTemporalClusters;
|
||||
private _adjustThresholdAdaptively;
|
||||
private _calculateItemToClusterSimilarity;
|
||||
private _recalculateClusterCentroid;
|
||||
private _calculateSimilarity;
|
||||
private _calculateEdgeWeight;
|
||||
private _sortNeighbors;
|
||||
private _buildSemanticHierarchy;
|
||||
private _detectOutliersClusterBased;
|
||||
private _detectOutliersIsolation;
|
||||
private _detectOutliersStatistical;
|
||||
private _generateVisualizationNodes;
|
||||
private _generateVisualizationEdges;
|
||||
private _generateVisualizationClusters;
|
||||
private _applyLayoutAlgorithm;
|
||||
private _manhattanDistance;
|
||||
private _calculateConfidence;
|
||||
private _generateSimilarityExplanation;
|
||||
/**
|
||||
* Get performance metrics for monitoring
|
||||
*/
|
||||
getPerformanceMetrics(operation?: string): Map<string, PerformanceMetrics[]> | PerformanceMetrics[];
|
||||
/**
|
||||
* Clear all caches
|
||||
*/
|
||||
clearCaches(): void;
|
||||
/**
|
||||
* Get cache statistics
|
||||
*/
|
||||
getCacheStats(): Record<string, {
|
||||
size: number;
|
||||
maxSize: number;
|
||||
}>;
|
||||
/**
|
||||
* Analyze data characteristics for algorithm selection
|
||||
*/
|
||||
private _analyzeDataCharacteristics;
|
||||
/**
|
||||
* Calculate centroid for a group of items
|
||||
*/
|
||||
private _calculateGroupCentroid;
|
||||
/**
|
||||
* Cluster within semantic type using vector similarity
|
||||
*/
|
||||
private _clusterWithinSemanticType;
|
||||
/**
|
||||
* Find cross-type connections via verbs
|
||||
*/
|
||||
private _findCrossTypeConnections;
|
||||
/**
|
||||
* Merge semantic clusters based on connections
|
||||
*/
|
||||
private _mergeSemanticClusters;
|
||||
/**
|
||||
* Get optimal clustering level for HNSW
|
||||
*/
|
||||
private _getOptimalClusteringLevel;
|
||||
/**
|
||||
* Get nodes at HNSW level
|
||||
*/
|
||||
private _getHNSWLevelNodes;
|
||||
/**
|
||||
* Find cluster members using HNSW neighbors
|
||||
*/
|
||||
private _findClusterMembers;
|
||||
/**
|
||||
* Calculate hierarchical clustering confidence
|
||||
*/
|
||||
private _calculateHierarchicalConfidence;
|
||||
/**
|
||||
* Assign unassigned items to nearest clusters
|
||||
*/
|
||||
private _assignUnassignedItems;
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
File diff suppressed because one or more lines are too long
185
.recovery-workspace/dist-backup-20250910-141917/neural/naturalLanguageProcessor.d.ts
vendored
Normal file
185
.recovery-workspace/dist-backup-20250910-141917/neural/naturalLanguageProcessor.d.ts
vendored
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
/**
|
||||
* 🧠 Natural Language Query Processor
|
||||
* Auto-breaks down natural language into structured Triple Intelligence queries
|
||||
*
|
||||
* Uses all of Brainy's sophisticated features:
|
||||
* - Embedding model for semantic understanding
|
||||
* - Pattern library with 100+ research-based patterns
|
||||
* - Entity Registry for concept mapping
|
||||
* - Progressive learning from usage
|
||||
*/
|
||||
import { TripleQuery } from '../triple/TripleIntelligence.js';
|
||||
import { Brainy } from '../brainy.js';
|
||||
export interface NaturalQueryIntent {
|
||||
type: 'vector' | 'field' | 'graph' | 'combined';
|
||||
primaryIntent: 'search' | 'filter' | 'aggregate' | 'navigate' | 'compare' | 'explain';
|
||||
confidence: number;
|
||||
extractedTerms: {
|
||||
searchTerms?: string[];
|
||||
fields?: Record<string, any>;
|
||||
connections?: {
|
||||
entities: string[];
|
||||
relationships: string[];
|
||||
};
|
||||
filters?: Record<string, any>;
|
||||
modifiers?: {
|
||||
recent?: boolean;
|
||||
popular?: boolean;
|
||||
limit?: number;
|
||||
boost?: string;
|
||||
sortBy?: string;
|
||||
groupBy?: string;
|
||||
};
|
||||
};
|
||||
context?: {
|
||||
domain?: string;
|
||||
temporalScope?: 'past' | 'present' | 'future' | 'all';
|
||||
complexity?: 'simple' | 'moderate' | 'complex';
|
||||
};
|
||||
}
|
||||
export declare class NaturalLanguageProcessor {
|
||||
private brain;
|
||||
private patternLibrary;
|
||||
private queryHistory;
|
||||
private initialized;
|
||||
private embeddingCache;
|
||||
constructor(brain: Brainy);
|
||||
/**
|
||||
* Get embedding using add/get/delete pattern
|
||||
*/
|
||||
private getEmbedding;
|
||||
/**
|
||||
* Initialize the pattern library (lazy loading)
|
||||
*/
|
||||
private ensureInitialized;
|
||||
/**
|
||||
* 🎯 MAIN METHOD: Convert natural language to Triple Intelligence query
|
||||
*/
|
||||
processNaturalQuery(naturalQuery: string): Promise<TripleQuery>;
|
||||
/**
|
||||
* Hybrid parse when pattern matching fails
|
||||
*/
|
||||
private hybridParse;
|
||||
/**
|
||||
* Analyze intent using keywords and structure with enhanced classification
|
||||
*/
|
||||
private analyzeIntent;
|
||||
/**
|
||||
* Detect the domain of the query
|
||||
*/
|
||||
private detectDomain;
|
||||
/**
|
||||
* Detect temporal scope in query
|
||||
*/
|
||||
private detectTemporalScope;
|
||||
/**
|
||||
* Assess query complexity
|
||||
*/
|
||||
private assessComplexity;
|
||||
/**
|
||||
* Step 2: Use neural analysis to decompose complex queries
|
||||
*/
|
||||
private decomposeQuery;
|
||||
/**
|
||||
* Step 3: Map concepts using Entity Registry and taxonomy
|
||||
*/
|
||||
private mapConcepts;
|
||||
/**
|
||||
* Step 4: Construct final Triple Intelligence query
|
||||
*/
|
||||
private constructTripleQuery;
|
||||
/**
|
||||
* Initialize pattern recognition for common query types
|
||||
*/
|
||||
private initializePatterns;
|
||||
/**
|
||||
* Detect field query patterns
|
||||
*/
|
||||
private hasFieldPatterns;
|
||||
/**
|
||||
* Detect connection query patterns
|
||||
*/
|
||||
private hasConnectionPatterns;
|
||||
/**
|
||||
* Extract terms and modifiers from query
|
||||
*/
|
||||
private extractTerms;
|
||||
/**
|
||||
* Find entity matches using Brainy's search capabilities
|
||||
*/
|
||||
private findEntityMatches;
|
||||
/**
|
||||
* Check if term is a known field name
|
||||
*/
|
||||
private isKnownField;
|
||||
/**
|
||||
* Map colloquial terms to actual field names
|
||||
*/
|
||||
private mapToFieldName;
|
||||
/**
|
||||
* Find similar successful queries from history
|
||||
* Uses Brainy's vector search to find semantically similar previous queries
|
||||
*/
|
||||
private findSimilarQueries;
|
||||
/**
|
||||
* Extract entities from query using Brainy's semantic search
|
||||
* Identifies known entities, concepts, and relationships in the query text
|
||||
*/
|
||||
private extractEntities;
|
||||
/**
|
||||
* Build final TripleQuery based on intent, entities, and query analysis
|
||||
* Constructs optimized query combining vector, graph, and field searches
|
||||
*/
|
||||
private buildQuery;
|
||||
/**
|
||||
* Extract entities from text using NEURAL matching to strict NounTypes
|
||||
* ALWAYS uses neural matching, NEVER falls back to patterns
|
||||
*/
|
||||
extract(text: string, options?: {
|
||||
types?: string[];
|
||||
includeMetadata?: boolean;
|
||||
confidence?: number;
|
||||
}): Promise<Array<{
|
||||
text: string;
|
||||
type: string;
|
||||
position: {
|
||||
start: number;
|
||||
end: number;
|
||||
};
|
||||
confidence: number;
|
||||
metadata?: any;
|
||||
}>>;
|
||||
/**
|
||||
* DEPRECATED - Old pattern-based extraction
|
||||
* This should NEVER be used - kept only for reference
|
||||
*/
|
||||
private extractWithPatterns_DEPRECATED;
|
||||
/**
|
||||
* Analyze sentiment of text
|
||||
*/
|
||||
sentiment(text: string, options?: {
|
||||
granularity?: 'document' | 'sentence' | 'aspect';
|
||||
aspects?: string[];
|
||||
}): Promise<{
|
||||
overall: {
|
||||
score: number;
|
||||
magnitude: number;
|
||||
label: 'positive' | 'negative' | 'neutral' | 'mixed';
|
||||
};
|
||||
sentences?: Array<{
|
||||
text: string;
|
||||
score: number;
|
||||
magnitude: number;
|
||||
label: string;
|
||||
}>;
|
||||
aspects?: Record<string, {
|
||||
score: number;
|
||||
magnitude: number;
|
||||
mentions: number;
|
||||
}>;
|
||||
}>;
|
||||
/**
|
||||
* Calculate confidence for entity extraction
|
||||
*/
|
||||
private calculateConfidence;
|
||||
}
|
||||
|
|
@ -0,0 +1,795 @@
|
|||
/**
|
||||
* 🧠 Natural Language Query Processor
|
||||
* Auto-breaks down natural language into structured Triple Intelligence queries
|
||||
*
|
||||
* Uses all of Brainy's sophisticated features:
|
||||
* - Embedding model for semantic understanding
|
||||
* - Pattern library with 100+ research-based patterns
|
||||
* - Entity Registry for concept mapping
|
||||
* - Progressive learning from usage
|
||||
*/
|
||||
import { PatternLibrary } from './patternLibrary.js';
|
||||
export class NaturalLanguageProcessor {
|
||||
constructor(brain) {
|
||||
this.initialized = false;
|
||||
this.embeddingCache = new Map();
|
||||
this.brain = brain;
|
||||
this.patternLibrary = new PatternLibrary(brain);
|
||||
this.queryHistory = [];
|
||||
}
|
||||
/**
|
||||
* Get embedding using add/get/delete pattern
|
||||
*/
|
||||
async getEmbedding(text) {
|
||||
// Check cache first
|
||||
if (this.embeddingCache.has(text)) {
|
||||
return this.embeddingCache.get(text);
|
||||
}
|
||||
// Use add/get/delete pattern to get embedding
|
||||
const id = await this.brain.add({
|
||||
data: text,
|
||||
type: 'document'
|
||||
});
|
||||
const entity = await this.brain.get(id);
|
||||
const embedding = entity?.vector || [];
|
||||
// Clean up temporary entity
|
||||
await this.brain.delete(id);
|
||||
// Cache the embedding
|
||||
this.embeddingCache.set(text, embedding);
|
||||
return embedding;
|
||||
}
|
||||
/**
|
||||
* Initialize the pattern library (lazy loading)
|
||||
*/
|
||||
async ensureInitialized() {
|
||||
if (!this.initialized) {
|
||||
await this.patternLibrary.init();
|
||||
this.initialized = true;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 🎯 MAIN METHOD: Convert natural language to Triple Intelligence query
|
||||
*/
|
||||
async processNaturalQuery(naturalQuery) {
|
||||
await this.ensureInitialized();
|
||||
// Step 1: Get embedding via add/get/delete pattern
|
||||
const queryEmbedding = await this.getEmbedding(naturalQuery);
|
||||
// Step 2: Find best matching patterns from our library
|
||||
const matches = await this.patternLibrary.findBestPatterns(queryEmbedding, 3);
|
||||
// Step 3: Try each pattern until we get a good match
|
||||
for (const { pattern, similarity } of matches) {
|
||||
if (similarity < 0.5)
|
||||
break; // Too low similarity, skip
|
||||
// Extract slots from the query based on pattern
|
||||
const extraction = this.patternLibrary.extractSlots(naturalQuery, pattern);
|
||||
if (extraction.confidence > 0.6) {
|
||||
// Fill the template with extracted slots
|
||||
const query = this.patternLibrary.fillTemplate(pattern.template, extraction.slots);
|
||||
// Track this query for learning
|
||||
this.queryHistory.push({
|
||||
query: naturalQuery,
|
||||
result: query,
|
||||
success: true // Will be updated based on user behavior
|
||||
});
|
||||
// Update pattern success metric
|
||||
this.patternLibrary.updateSuccessMetric(pattern.id, true);
|
||||
return query;
|
||||
}
|
||||
}
|
||||
// Step 4: Fall back to hybrid approach if no pattern matches well
|
||||
return this.hybridParse(naturalQuery, queryEmbedding);
|
||||
}
|
||||
/**
|
||||
* Hybrid parse when pattern matching fails
|
||||
*/
|
||||
async hybridParse(query, queryEmbedding) {
|
||||
// Analyze intent using embeddings and keywords
|
||||
const intent = await this.analyzeIntent(query);
|
||||
// Find similar successful queries from history
|
||||
const similar = await this.findSimilarQueries(queryEmbedding);
|
||||
if (similar.length > 0 && similar[0].similarity > 0.9) {
|
||||
// Adapt a very similar previous query (for future implementation)
|
||||
// return this.adaptQuery(query, similar[0].result)
|
||||
}
|
||||
// Extract entities using Brainy's search
|
||||
const entities = await this.extractEntities(query);
|
||||
// Build query based on intent and entities
|
||||
return this.buildQuery(query, intent, entities);
|
||||
}
|
||||
/**
|
||||
* Analyze intent using keywords and structure with enhanced classification
|
||||
*/
|
||||
async analyzeIntent(query) {
|
||||
// Analyze query structure patterns
|
||||
const lowerQuery = query.toLowerCase();
|
||||
// Determine primary intent
|
||||
let primaryIntent = 'search';
|
||||
let confidence = 0.7; // Base confidence
|
||||
let type = 'vector'; // Default
|
||||
// Intent detection patterns
|
||||
if (lowerQuery.match(/\b(filter|where|with|having)\b/)) {
|
||||
primaryIntent = 'filter';
|
||||
confidence += 0.15;
|
||||
}
|
||||
else if (lowerQuery.match(/\b(count|sum|average|total|group by)\b/)) {
|
||||
primaryIntent = 'aggregate';
|
||||
confidence += 0.2;
|
||||
}
|
||||
else if (lowerQuery.match(/\b(compare|versus|vs|difference|between)\b/)) {
|
||||
primaryIntent = 'compare';
|
||||
confidence += 0.15;
|
||||
}
|
||||
else if (lowerQuery.match(/\b(explain|why|how|what causes)\b/)) {
|
||||
primaryIntent = 'explain';
|
||||
confidence += 0.1;
|
||||
}
|
||||
else if (lowerQuery.match(/\b(connected|related|linked|from.*to)\b/)) {
|
||||
primaryIntent = 'navigate';
|
||||
type = 'graph';
|
||||
confidence += 0.15;
|
||||
}
|
||||
// Detect field queries
|
||||
if (this.hasFieldPatterns(lowerQuery)) {
|
||||
type = type === 'graph' ? 'combined' : 'field';
|
||||
confidence += 0.1;
|
||||
}
|
||||
// Detect connection queries
|
||||
if (this.hasConnectionPatterns(lowerQuery)) {
|
||||
type = type === 'field' ? 'combined' : 'graph';
|
||||
confidence += 0.1;
|
||||
}
|
||||
// Extract context
|
||||
const context = {
|
||||
domain: this.detectDomain(query),
|
||||
temporalScope: this.detectTemporalScope(query),
|
||||
complexity: this.assessComplexity(query)
|
||||
};
|
||||
// Extract basic terms with enhanced modifiers
|
||||
const extractedTerms = this.extractTerms(query);
|
||||
return {
|
||||
type,
|
||||
primaryIntent,
|
||||
confidence,
|
||||
extractedTerms,
|
||||
context
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Detect the domain of the query
|
||||
*/
|
||||
detectDomain(query) {
|
||||
const lowerQuery = query.toLowerCase();
|
||||
if (lowerQuery.match(/\b(code|function|api|bug|error|debug)\b/)) {
|
||||
return 'technical';
|
||||
}
|
||||
else if (lowerQuery.match(/\b(revenue|sales|profit|customer|market)\b/)) {
|
||||
return 'business';
|
||||
}
|
||||
else if (lowerQuery.match(/\b(research|study|paper|theory|hypothesis)\b/)) {
|
||||
return 'academic';
|
||||
}
|
||||
return 'general';
|
||||
}
|
||||
/**
|
||||
* Detect temporal scope in query
|
||||
*/
|
||||
detectTemporalScope(query) {
|
||||
const lowerQuery = query.toLowerCase();
|
||||
if (lowerQuery.match(/\b(was|were|did|had|yesterday|last|previous|ago)\b/)) {
|
||||
return 'past';
|
||||
}
|
||||
else if (lowerQuery.match(/\b(will|going to|tomorrow|next|future|upcoming)\b/)) {
|
||||
return 'future';
|
||||
}
|
||||
else if (lowerQuery.match(/\b(is|are|currently|now|today|present)\b/)) {
|
||||
return 'present';
|
||||
}
|
||||
return 'all';
|
||||
}
|
||||
/**
|
||||
* Assess query complexity
|
||||
*/
|
||||
assessComplexity(query) {
|
||||
const words = query.split(/\s+/).length;
|
||||
const hasMultipleClauses = query.match(/\b(and|or|but|with|where)\b/g)?.length || 0;
|
||||
const hasNesting = query.includes('(') || query.includes('[');
|
||||
if (words < 5 && hasMultipleClauses === 0) {
|
||||
return 'simple';
|
||||
}
|
||||
else if (words > 15 || hasMultipleClauses > 2 || hasNesting) {
|
||||
return 'complex';
|
||||
}
|
||||
return 'moderate';
|
||||
}
|
||||
/**
|
||||
* Step 2: Use neural analysis to decompose complex queries
|
||||
*/
|
||||
async decomposeQuery(query, intent) {
|
||||
// Use Brainy's neural clustering to find similar patterns
|
||||
const queryTerms = query.split(/\\s+/).filter(term => term.length > 2);
|
||||
// Try to find existing entities that match query terms
|
||||
const entityMatches = await this.findEntityMatches(queryTerms);
|
||||
return {
|
||||
originalQuery: query,
|
||||
intent,
|
||||
entityMatches,
|
||||
queryTerms
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Step 3: Map concepts using Entity Registry and taxonomy
|
||||
*/
|
||||
async mapConcepts(decomposition) {
|
||||
const mappedFields = {};
|
||||
const searchTerms = [];
|
||||
const connections = {};
|
||||
// Use Entity Registry to map known entities
|
||||
for (const term of decomposition.queryTerms) {
|
||||
const entityMatch = decomposition.entityMatches.find((m) => m.term.toLowerCase() === term.toLowerCase());
|
||||
if (entityMatch) {
|
||||
if (entityMatch.type === 'field') {
|
||||
mappedFields[entityMatch.field] = entityMatch.value;
|
||||
}
|
||||
else if (entityMatch.type === 'entity') {
|
||||
connections[entityMatch.id] = entityMatch;
|
||||
}
|
||||
}
|
||||
else {
|
||||
searchTerms.push(term);
|
||||
}
|
||||
}
|
||||
return {
|
||||
searchTerms,
|
||||
mappedFields,
|
||||
connections
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Step 4: Construct final Triple Intelligence query
|
||||
*/
|
||||
constructTripleQuery(originalQuery, intent, mapped) {
|
||||
const query = {};
|
||||
// Set vector search if we have search terms
|
||||
if (mapped.searchTerms.length > 0) {
|
||||
query.like = mapped.searchTerms.join(' ');
|
||||
}
|
||||
else if (intent.type === 'vector') {
|
||||
query.like = originalQuery;
|
||||
}
|
||||
// Set field filters if we found field mappings
|
||||
if (Object.keys(mapped.mappedFields).length > 0) {
|
||||
query.where = mapped.mappedFields;
|
||||
}
|
||||
// Set connection searches if we found entity connections
|
||||
if (Object.keys(mapped.connections).length > 0) {
|
||||
const entities = Object.keys(mapped.connections);
|
||||
if (entities.length > 0) {
|
||||
query.connected = { to: entities };
|
||||
}
|
||||
}
|
||||
// Apply extracted modifiers
|
||||
if (intent.extractedTerms.modifiers) {
|
||||
const mods = intent.extractedTerms.modifiers;
|
||||
if (mods.limit)
|
||||
query.limit = mods.limit;
|
||||
if (mods.boost)
|
||||
query.boost = mods.boost;
|
||||
}
|
||||
return query;
|
||||
}
|
||||
/**
|
||||
* Initialize pattern recognition for common query types
|
||||
*/
|
||||
initializePatterns() {
|
||||
const patterns = new Map();
|
||||
// "Find papers about AI from 2023"
|
||||
patterns.set(/find\\s+(.+?)\\s+about\\s+(.+?)\\s+from\\s+(\\d{4})/i, (match) => ({
|
||||
like: match[2],
|
||||
where: { year: parseInt(match[3]) }
|
||||
}));
|
||||
// "Show me recent posts by John"
|
||||
patterns.set(/show\\s+me\\s+recent\\s+(.+?)\\s+by\\s+(.+)/i, (match) => ({
|
||||
like: match[1],
|
||||
boost: 'recent',
|
||||
connected: { from: match[2] }
|
||||
}));
|
||||
// "Papers with more than 100 citations"
|
||||
patterns.set(/(.+?)\\s+with\\s+more\\s+than\\s+(\\d+)\\s+(.+)/i, (match) => ({
|
||||
like: match[1],
|
||||
where: { [match[3]]: { greaterThan: parseInt(match[2]) } }
|
||||
}));
|
||||
// "Documents related to Stanford"
|
||||
patterns.set(/(.+?)\\s+related\\s+to\\s+(.+)/i, (match) => ({
|
||||
like: match[1],
|
||||
connected: { to: match[2] }
|
||||
}));
|
||||
return patterns;
|
||||
}
|
||||
/**
|
||||
* Detect field query patterns
|
||||
*/
|
||||
hasFieldPatterns(query) {
|
||||
const fieldIndicators = [
|
||||
'from', 'after', 'before', 'with more than', 'with less than',
|
||||
'published', 'created', 'year', 'date', 'citations', 'score'
|
||||
];
|
||||
return fieldIndicators.some(indicator => query.includes(indicator));
|
||||
}
|
||||
/**
|
||||
* Detect connection query patterns
|
||||
*/
|
||||
hasConnectionPatterns(query) {
|
||||
const connectionIndicators = [
|
||||
'by', 'from', 'connected to', 'related to', 'authored by',
|
||||
'created by', 'associated with', 'linked to'
|
||||
];
|
||||
return connectionIndicators.some(indicator => query.includes(indicator));
|
||||
}
|
||||
/**
|
||||
* Extract terms and modifiers from query
|
||||
*/
|
||||
extractTerms(query) {
|
||||
const extracted = {};
|
||||
// Extract limit numbers
|
||||
const limitMatch = query.match(/(?:top|first|limit)\\s+(\\d+)/i);
|
||||
if (limitMatch) {
|
||||
extracted.modifiers = { limit: parseInt(limitMatch[1]) };
|
||||
}
|
||||
// Extract boost indicators
|
||||
if (query.toLowerCase().includes('recent')) {
|
||||
extracted.modifiers = { ...extracted.modifiers, boost: 'recent' };
|
||||
}
|
||||
if (query.toLowerCase().includes('popular')) {
|
||||
extracted.modifiers = { ...extracted.modifiers, boost: 'popular' };
|
||||
}
|
||||
return extracted;
|
||||
}
|
||||
/**
|
||||
* Find entity matches using Brainy's search capabilities
|
||||
*/
|
||||
async findEntityMatches(terms) {
|
||||
const matches = [];
|
||||
for (const term of terms) {
|
||||
try {
|
||||
// Search for similar entities in the knowledge base
|
||||
const results = await this.brain.search(term, 5);
|
||||
for (const result of results) {
|
||||
if (result.score > 0.8) { // High similarity threshold
|
||||
matches.push({
|
||||
term,
|
||||
id: result.id,
|
||||
type: 'entity',
|
||||
confidence: result.score,
|
||||
metadata: result.entity?.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
// Check if term matches known field names
|
||||
if (this.isKnownField(term)) {
|
||||
matches.push({
|
||||
term,
|
||||
type: 'field',
|
||||
field: this.mapToFieldName(term),
|
||||
confidence: 0.9
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
// If search fails, continue with other terms
|
||||
console.debug(`Failed to search for term: ${term}`, error);
|
||||
}
|
||||
}
|
||||
return matches;
|
||||
}
|
||||
/**
|
||||
* Check if term is a known field name
|
||||
*/
|
||||
isKnownField(term) {
|
||||
const knownFields = [
|
||||
'year', 'date', 'created', 'published', 'author', 'title',
|
||||
'citations', 'views', 'score', 'rating', 'category', 'type'
|
||||
];
|
||||
return knownFields.includes(term.toLowerCase());
|
||||
}
|
||||
/**
|
||||
* Map colloquial terms to actual field names
|
||||
*/
|
||||
mapToFieldName(term) {
|
||||
const fieldMappings = {
|
||||
'published': 'publishDate',
|
||||
'created': 'createdAt',
|
||||
'author': 'authorId',
|
||||
'citations': 'citationCount'
|
||||
};
|
||||
return fieldMappings[term.toLowerCase()] || term.toLowerCase();
|
||||
}
|
||||
/**
|
||||
* Find similar successful queries from history
|
||||
* Uses Brainy's vector search to find semantically similar previous queries
|
||||
*/
|
||||
async findSimilarQueries(queryEmbedding) {
|
||||
try {
|
||||
// Search for similar queries in a hypothetical query history
|
||||
// For now, return empty array since we don't have query history storage yet
|
||||
// This would integrate with Brainy's search to find similar query patterns
|
||||
// Future implementation could search a query_history noun type:
|
||||
// const similarQueries = await this.brainy.search(queryEmbedding, {
|
||||
// limit: 5,
|
||||
// metadata: { type: 'successful_query' },
|
||||
// nounTypes: ['query_history']
|
||||
// })
|
||||
return [];
|
||||
}
|
||||
catch (error) {
|
||||
console.debug('Failed to find similar queries:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Extract entities from query using Brainy's semantic search
|
||||
* Identifies known entities, concepts, and relationships in the query text
|
||||
*/
|
||||
async extractEntities(query) {
|
||||
try {
|
||||
// Split query into potential entity terms
|
||||
const terms = query.toLowerCase()
|
||||
.split(/[\s,\.;!?]+/)
|
||||
.filter(term => term.length > 2);
|
||||
const entities = [];
|
||||
// Search for each term in Brainy to see if it matches known entities
|
||||
for (const term of terms) {
|
||||
try {
|
||||
const results = await this.brain.search(term, 3);
|
||||
if (results && results.length > 0) {
|
||||
// Found matching entities
|
||||
entities.push({
|
||||
term,
|
||||
matches: results,
|
||||
confidence: results[0].score || 0.7
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (searchError) {
|
||||
// Continue if individual term search fails
|
||||
console.debug(`Entity search failed for term: ${term}`, searchError);
|
||||
}
|
||||
}
|
||||
return entities;
|
||||
}
|
||||
catch (error) {
|
||||
console.debug('Failed to extract entities:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Build final TripleQuery based on intent, entities, and query analysis
|
||||
* Constructs optimized query combining vector, graph, and field searches
|
||||
*/
|
||||
async buildQuery(query, intent, entities) {
|
||||
try {
|
||||
const tripleQuery = {
|
||||
like: query, // Default to semantic search
|
||||
limit: 10
|
||||
};
|
||||
// Add field filters based on intent
|
||||
if (intent.hasFieldPatterns) {
|
||||
// Extract field-based constraints from the query
|
||||
const whereClause = {};
|
||||
// Look for date/year patterns
|
||||
const yearMatch = query.match(/(\d{4})/g);
|
||||
if (yearMatch) {
|
||||
whereClause.year = parseInt(yearMatch[0]);
|
||||
}
|
||||
// Look for numeric constraints
|
||||
const moreThanMatch = query.match(/more than (\d+)/i);
|
||||
if (moreThanMatch) {
|
||||
whereClause.count = { greaterThan: parseInt(moreThanMatch[1]) };
|
||||
}
|
||||
if (Object.keys(whereClause).length > 0) {
|
||||
tripleQuery.where = whereClause;
|
||||
}
|
||||
}
|
||||
// Add connection-based searches
|
||||
if (intent.hasConnectionPatterns) {
|
||||
// Look for relationship patterns in the query
|
||||
const connectedMatch = query.match(/connected to (.+?)$/i) ||
|
||||
query.match(/related to (.+?)$/i);
|
||||
if (connectedMatch) {
|
||||
tripleQuery.connected = {
|
||||
to: connectedMatch[1].trim()
|
||||
};
|
||||
}
|
||||
}
|
||||
// Add entity-specific filters
|
||||
if (entities && entities.length > 0) {
|
||||
const highConfidenceEntities = entities.filter(e => e.confidence > 0.8);
|
||||
if (highConfidenceEntities.length > 0) {
|
||||
// Use the highest confidence entity to refine search
|
||||
const topEntity = highConfidenceEntities[0];
|
||||
if (topEntity.matches && topEntity.matches.length > 0) {
|
||||
// Add entity-specific metadata or connection
|
||||
const entityData = topEntity.matches[0].metadata;
|
||||
if (entityData && entityData.category) {
|
||||
tripleQuery.where = {
|
||||
...tripleQuery.where,
|
||||
category: entityData.category
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return tripleQuery;
|
||||
}
|
||||
catch (error) {
|
||||
console.debug('Failed to build query:', error);
|
||||
// Return simple query as fallback
|
||||
return {
|
||||
like: query,
|
||||
limit: 10
|
||||
};
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Extract entities from text using NEURAL matching to strict NounTypes
|
||||
* ALWAYS uses neural matching, NEVER falls back to patterns
|
||||
*/
|
||||
async extract(text, options) {
|
||||
await this.ensureInitialized();
|
||||
// ALWAYS use NeuralEntityExtractor for proper type matching
|
||||
const { NeuralEntityExtractor } = await import('./entityExtractor.js');
|
||||
const extractor = new NeuralEntityExtractor(this.brain);
|
||||
// Convert string types to NounTypes if provided
|
||||
const nounTypes = options?.types ?
|
||||
options.types.map(t => t) :
|
||||
undefined;
|
||||
// Extract using neural matching
|
||||
const entities = await extractor.extract(text, {
|
||||
types: nounTypes,
|
||||
confidence: options?.confidence || 0.0, // Accept ALL matches
|
||||
includeVectors: false,
|
||||
neuralMatching: true // ALWAYS use neural matching
|
||||
});
|
||||
// Convert to expected format
|
||||
return entities.map(entity => ({
|
||||
text: entity.text,
|
||||
type: entity.type,
|
||||
position: entity.position,
|
||||
confidence: entity.confidence,
|
||||
metadata: options?.includeMetadata ? {
|
||||
...entity.metadata,
|
||||
neuralMatch: true,
|
||||
extractedAt: Date.now()
|
||||
} : undefined
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* DEPRECATED - Old pattern-based extraction
|
||||
* This should NEVER be used - kept only for reference
|
||||
*/
|
||||
async extractWithPatterns_DEPRECATED(text, options) {
|
||||
const extracted = [];
|
||||
// Common entity patterns
|
||||
const patterns = {
|
||||
// People (names with capitals)
|
||||
person: /\b([A-Z][a-z]+ [A-Z][a-z]+)\b/g,
|
||||
// Organizations (capitals, Inc, LLC, etc)
|
||||
organization: /\b([A-Z][a-zA-Z&]+(?: [A-Z][a-zA-Z&]+)*(?:,? (?:Inc|LLC|Corp|Ltd|Co|Group|Foundation|Institute|University|College|School|Hospital|Bank|Agency)\.?))\b/g,
|
||||
// Locations (capitals, common place words)
|
||||
location: /\b([A-Z][a-z]+(?: [A-Z][a-z]+)*(?:,? (?:[A-Z][a-z]+))?)(?= (?:City|County|State|Country|Street|Road|Avenue|Boulevard|Drive|Park|Square|Place|Island|Mountain|River|Lake|Ocean|Sea))\b/g,
|
||||
// Dates
|
||||
date: /\b(\d{1,2}[\/\-]\d{1,2}[\/\-]\d{2,4}|\d{4}[\/\-]\d{1,2}[\/\-]\d{1,2}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{1,2},? \d{4}|\d{1,2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]* \d{4})\b/gi,
|
||||
// Times
|
||||
time: /\b(\d{1,2}:\d{2}(?::\d{2})?(?:\s?[AP]M)?)\b/gi,
|
||||
// Emails
|
||||
email: /\b([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})\b/g,
|
||||
// URLs
|
||||
url: /\b(https?:\/\/[^\s]+)\b/g,
|
||||
// Phone numbers
|
||||
phone: /\b(\+?\d{1,3}?[- .]?\(?\d{1,4}\)?[- .]?\d{1,4}[- .]?\d{1,4})\b/g,
|
||||
// Money
|
||||
money: /\b(\$[\d,]+(?:\.\d{2})?|[\d,]+(?:\.\d{2})?\s*(?:USD|EUR|GBP|JPY|CNY))\b/gi,
|
||||
// Percentages
|
||||
percentage: /\b(\d+(?:\.\d+)?%)\b/g,
|
||||
// Products/versions
|
||||
product: /\b([A-Z][a-zA-Z0-9]*(?: [A-Z][a-zA-Z0-9]*)*\s+v?\d+(?:\.\d+)*)\b/g,
|
||||
// Hashtags
|
||||
hashtag: /#[a-zA-Z0-9_]+/g,
|
||||
// Mentions
|
||||
mention: /@[a-zA-Z0-9_]+/g
|
||||
};
|
||||
const minConfidence = options?.confidence || 0.5;
|
||||
const targetTypes = options?.types || Object.keys(patterns);
|
||||
// Apply each pattern
|
||||
for (const [type, pattern] of Object.entries(patterns)) {
|
||||
if (!targetTypes.includes(type))
|
||||
continue;
|
||||
let match;
|
||||
while ((match = pattern.exec(text)) !== null) {
|
||||
const extractedText = match[1] || match[0];
|
||||
const confidence = this.calculateConfidence(extractedText, type);
|
||||
if (confidence >= minConfidence) {
|
||||
const entity = {
|
||||
text: extractedText,
|
||||
type,
|
||||
position: {
|
||||
start: match.index,
|
||||
end: match.index + match[0].length
|
||||
},
|
||||
confidence
|
||||
};
|
||||
if (options?.includeMetadata) {
|
||||
;
|
||||
entity.metadata = {
|
||||
pattern: pattern.source,
|
||||
contextBefore: text.substring(Math.max(0, match.index - 20), match.index),
|
||||
contextAfter: text.substring(match.index + match[0].length, Math.min(text.length, match.index + match[0].length + 20))
|
||||
};
|
||||
}
|
||||
extracted.push(entity);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Sort by position
|
||||
extracted.sort((a, b) => a.position.start - b.position.start);
|
||||
// Remove overlapping entities (keep higher confidence)
|
||||
const filtered = [];
|
||||
for (const entity of extracted) {
|
||||
const overlapping = filtered.find(e => (entity.position.start >= e.position.start && entity.position.start < e.position.end) ||
|
||||
(entity.position.end > e.position.start && entity.position.end <= e.position.end));
|
||||
if (!overlapping) {
|
||||
filtered.push(entity);
|
||||
}
|
||||
else if (entity.confidence > overlapping.confidence) {
|
||||
const index = filtered.indexOf(overlapping);
|
||||
filtered[index] = entity;
|
||||
}
|
||||
}
|
||||
return filtered;
|
||||
}
|
||||
/**
|
||||
* Analyze sentiment of text
|
||||
*/
|
||||
async sentiment(text, options) {
|
||||
// Sentiment words with scores
|
||||
const positiveWords = new Set(['good', 'great', 'excellent', 'amazing', 'wonderful', 'fantastic', 'love', 'like', 'best', 'happy', 'joy', 'brilliant', 'outstanding', 'perfect', 'beautiful', 'awesome', 'super', 'nice', 'fun', 'exciting', 'impressive', 'incredible', 'remarkable', 'delightful', 'pleased', 'satisfied', 'successful', 'effective', 'helpful']);
|
||||
const negativeWords = new Set(['bad', 'terrible', 'awful', 'horrible', 'hate', 'dislike', 'worst', 'sad', 'angry', 'poor', 'disappointing', 'failed', 'broken', 'useless', 'waste', 'sucks', 'disgusting', 'ugly', 'boring', 'annoying', 'frustrating', 'difficult', 'complicated', 'confusing', 'slow', 'expensive', 'unfair', 'wrong', 'mistake', 'problem', 'issue']);
|
||||
const intensifiers = new Set(['very', 'extremely', 'really', 'absolutely', 'completely', 'totally', 'quite', 'rather', 'so']);
|
||||
const negations = new Set(['not', 'no', 'never', 'neither', 'none', 'nobody', 'nothing', 'nowhere', 'hardly', 'barely', 'scarcely']);
|
||||
const normalizedText = text.toLowerCase();
|
||||
const words = normalizedText.split(/\s+/);
|
||||
// Calculate overall sentiment
|
||||
let positiveCount = 0;
|
||||
let negativeCount = 0;
|
||||
let intensifierBoost = 1;
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
const word = words[i].replace(/[^a-z]/g, '');
|
||||
const prevWord = i > 0 ? words[i - 1].replace(/[^a-z]/g, '') : '';
|
||||
// Check for intensifiers
|
||||
if (intensifiers.has(prevWord)) {
|
||||
intensifierBoost = 1.5;
|
||||
}
|
||||
else {
|
||||
intensifierBoost = 1;
|
||||
}
|
||||
// Check for negation
|
||||
const isNegated = negations.has(prevWord);
|
||||
if (positiveWords.has(word)) {
|
||||
if (isNegated) {
|
||||
negativeCount += intensifierBoost;
|
||||
}
|
||||
else {
|
||||
positiveCount += intensifierBoost;
|
||||
}
|
||||
}
|
||||
else if (negativeWords.has(word)) {
|
||||
if (isNegated) {
|
||||
positiveCount += intensifierBoost;
|
||||
}
|
||||
else {
|
||||
negativeCount += intensifierBoost;
|
||||
}
|
||||
}
|
||||
}
|
||||
const total = positiveCount + negativeCount;
|
||||
const score = total > 0 ? (positiveCount - negativeCount) / total : 0;
|
||||
const magnitude = Math.min(1, total / words.length);
|
||||
let label;
|
||||
if (score > 0.2)
|
||||
label = 'positive';
|
||||
else if (score < -0.2)
|
||||
label = 'negative';
|
||||
else if (magnitude > 0.3)
|
||||
label = 'mixed';
|
||||
else
|
||||
label = 'neutral';
|
||||
const result = {
|
||||
overall: {
|
||||
score,
|
||||
magnitude,
|
||||
label
|
||||
}
|
||||
};
|
||||
// Sentence-level analysis
|
||||
if (options?.granularity === 'sentence' || options?.granularity === 'aspect') {
|
||||
const sentences = text.match(/[^.!?]+[.!?]+/g) || [text];
|
||||
result.sentences = [];
|
||||
for (const sentence of sentences) {
|
||||
const sentenceResult = await this.sentiment(sentence);
|
||||
result.sentences.push({
|
||||
text: sentence.trim(),
|
||||
score: sentenceResult.overall.score,
|
||||
magnitude: sentenceResult.overall.magnitude,
|
||||
label: sentenceResult.overall.label
|
||||
});
|
||||
}
|
||||
}
|
||||
// Aspect-based analysis
|
||||
if (options?.granularity === 'aspect' && options?.aspects) {
|
||||
result.aspects = {};
|
||||
for (const aspect of options.aspects) {
|
||||
const aspectRegex = new RegExp(`[^.!?]*\\b${aspect}\\b[^.!?]*[.!?]?`, 'gi');
|
||||
const aspectSentences = text.match(aspectRegex) || [];
|
||||
if (aspectSentences.length > 0) {
|
||||
let aspectScore = 0;
|
||||
let aspectMagnitude = 0;
|
||||
for (const sentence of aspectSentences) {
|
||||
const sentimentResult = await this.sentiment(sentence);
|
||||
aspectScore += sentimentResult.overall.score;
|
||||
aspectMagnitude += sentimentResult.overall.magnitude;
|
||||
}
|
||||
result.aspects[aspect] = {
|
||||
score: aspectScore / aspectSentences.length,
|
||||
magnitude: aspectMagnitude / aspectSentences.length,
|
||||
mentions: aspectSentences.length
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
/**
|
||||
* Calculate confidence for entity extraction
|
||||
*/
|
||||
calculateConfidence(text, type) {
|
||||
let confidence = 0.5; // Base confidence
|
||||
// Adjust based on type-specific rules
|
||||
switch (type) {
|
||||
case 'person':
|
||||
// Names with 2-3 capitalized words are more confident
|
||||
const nameWords = text.split(' ');
|
||||
if (nameWords.length >= 2 && nameWords.length <= 3) {
|
||||
confidence += 0.3;
|
||||
}
|
||||
if (nameWords.every(w => /^[A-Z]/.test(w))) {
|
||||
confidence += 0.2;
|
||||
}
|
||||
break;
|
||||
case 'organization':
|
||||
// Presence of corporate suffixes increases confidence
|
||||
if (/\b(Inc|LLC|Corp|Ltd|Co|Group)\.?$/.test(text)) {
|
||||
confidence += 0.4;
|
||||
}
|
||||
break;
|
||||
case 'email':
|
||||
case 'url':
|
||||
// These patterns are very specific, high confidence
|
||||
confidence = 0.95;
|
||||
break;
|
||||
case 'date':
|
||||
case 'time':
|
||||
case 'money':
|
||||
case 'percentage':
|
||||
// Numeric patterns are reliable
|
||||
confidence = 0.9;
|
||||
break;
|
||||
case 'location':
|
||||
// Geographic terms increase confidence
|
||||
if (/\b(City|State|Country|Street|Road|Avenue)$/.test(text)) {
|
||||
confidence += 0.3;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return Math.min(1, confidence);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=naturalLanguageProcessor.js.map
|
||||
File diff suppressed because one or more lines are too long
64
.recovery-workspace/dist-backup-20250910-141917/neural/naturalLanguageProcessorStatic.d.ts
vendored
Normal file
64
.recovery-workspace/dist-backup-20250910-141917/neural/naturalLanguageProcessorStatic.d.ts
vendored
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
/**
|
||||
* 🧠 Natural Language Query Processor - STATIC VERSION
|
||||
* No runtime initialization, no memory leaks, patterns pre-built at compile time
|
||||
*
|
||||
* Uses static pattern matching with 220 pre-built patterns
|
||||
*/
|
||||
import { Vector } from '../coreTypes.js';
|
||||
import { TripleQuery } from '../triple/TripleIntelligence.js';
|
||||
export interface NaturalQueryIntent {
|
||||
type: 'vector' | 'field' | 'graph' | 'combined';
|
||||
confidence: number;
|
||||
extractedTerms: {
|
||||
entities?: string[];
|
||||
fields?: string[];
|
||||
relationships?: string[];
|
||||
modifiers?: string[];
|
||||
};
|
||||
}
|
||||
export declare class NaturalLanguageProcessor {
|
||||
private queryHistory;
|
||||
constructor();
|
||||
/**
|
||||
* No initialization needed - patterns are pre-built!
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Process natural language query into structured Triple Intelligence query
|
||||
* @param naturalQuery The natural language query string
|
||||
* @param queryEmbedding Pre-computed embedding from Brainy (passed in to avoid circular dependency)
|
||||
*/
|
||||
processNaturalQuery(naturalQuery: string, queryEmbedding?: Vector): Promise<TripleQuery>;
|
||||
/**
|
||||
* Analyze query intent using keywords
|
||||
*/
|
||||
private analyzeIntent;
|
||||
/**
|
||||
* Extract field terms from query
|
||||
*/
|
||||
private extractFieldTerms;
|
||||
/**
|
||||
* Extract relationship terms
|
||||
*/
|
||||
private extractRelationshipTerms;
|
||||
/**
|
||||
* Build field constraints from extracted terms
|
||||
*/
|
||||
private buildFieldConstraints;
|
||||
/**
|
||||
* Find similar queries from history (without using Brainy)
|
||||
*/
|
||||
private findSimilarQueries;
|
||||
/**
|
||||
* Adapt a previous query for new input
|
||||
*/
|
||||
private adaptQuery;
|
||||
/**
|
||||
* Extract entities from query
|
||||
*/
|
||||
private extractEntities;
|
||||
/**
|
||||
* Build query from components
|
||||
*/
|
||||
private buildQuery;
|
||||
}
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* 🧠 Natural Language Query Processor - STATIC VERSION
|
||||
* No runtime initialization, no memory leaks, patterns pre-built at compile time
|
||||
*
|
||||
* Uses static pattern matching with 220 pre-built patterns
|
||||
*/
|
||||
import { patternMatchQuery } from './staticPatternMatcher.js';
|
||||
export class NaturalLanguageProcessor {
|
||||
constructor() {
|
||||
this.queryHistory = [];
|
||||
// Patterns are static - no initialization needed!
|
||||
}
|
||||
/**
|
||||
* No initialization needed - patterns are pre-built!
|
||||
*/
|
||||
async init() {
|
||||
// Nothing to do - patterns are compiled into the code
|
||||
return Promise.resolve();
|
||||
}
|
||||
/**
|
||||
* Process natural language query into structured Triple Intelligence query
|
||||
* @param naturalQuery The natural language query string
|
||||
* @param queryEmbedding Pre-computed embedding from Brainy (passed in to avoid circular dependency)
|
||||
*/
|
||||
async processNaturalQuery(naturalQuery, queryEmbedding) {
|
||||
// Use static pattern matcher (no async, no memory allocation!)
|
||||
const structuredQuery = patternMatchQuery(naturalQuery, queryEmbedding);
|
||||
// Step 3: Enhance with intent analysis if needed
|
||||
if (!structuredQuery.where && !structuredQuery.connected) {
|
||||
const intent = await this.analyzeIntent(naturalQuery);
|
||||
// Add metadata based on intent
|
||||
if (intent.type === 'field' && intent.extractedTerms.fields) {
|
||||
structuredQuery.where = this.buildFieldConstraints(intent.extractedTerms.fields);
|
||||
}
|
||||
}
|
||||
// Track for learning (but don't create new Brainy!)
|
||||
this.queryHistory.push({
|
||||
query: naturalQuery,
|
||||
result: structuredQuery,
|
||||
success: false // Will be updated based on user interaction
|
||||
});
|
||||
// Keep history limited to prevent memory growth
|
||||
if (this.queryHistory.length > 100) {
|
||||
this.queryHistory.shift();
|
||||
}
|
||||
return structuredQuery;
|
||||
}
|
||||
/**
|
||||
* Analyze query intent using keywords
|
||||
*/
|
||||
async analyzeIntent(query) {
|
||||
const lowerQuery = query.toLowerCase();
|
||||
// Check for field-specific keywords
|
||||
const fieldKeywords = ['where', 'filter', 'with', 'has', 'contains', 'equals', 'greater', 'less', 'between'];
|
||||
const hasFieldIntent = fieldKeywords.some(kw => lowerQuery.includes(kw));
|
||||
// Check for graph keywords
|
||||
const graphKeywords = ['related', 'connected', 'linked', 'associated', 'references'];
|
||||
const hasGraphIntent = graphKeywords.some(kw => lowerQuery.includes(kw));
|
||||
// Determine type
|
||||
let type = 'vector';
|
||||
if (hasFieldIntent && hasGraphIntent) {
|
||||
type = 'combined';
|
||||
}
|
||||
else if (hasFieldIntent) {
|
||||
type = 'field';
|
||||
}
|
||||
else if (hasGraphIntent) {
|
||||
type = 'graph';
|
||||
}
|
||||
return {
|
||||
type,
|
||||
confidence: 0.8,
|
||||
extractedTerms: {
|
||||
fields: hasFieldIntent ? this.extractFieldTerms(query) : undefined,
|
||||
relationships: hasGraphIntent ? this.extractRelationshipTerms(query) : undefined
|
||||
}
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Extract field terms from query
|
||||
*/
|
||||
extractFieldTerms(query) {
|
||||
const terms = [];
|
||||
// Simple extraction of potential field names
|
||||
const words = query.split(/\s+/);
|
||||
const fieldIndicators = ['year', 'date', 'author', 'type', 'category', 'status', 'price'];
|
||||
for (const word of words) {
|
||||
if (fieldIndicators.includes(word.toLowerCase())) {
|
||||
terms.push(word.toLowerCase());
|
||||
}
|
||||
}
|
||||
return terms;
|
||||
}
|
||||
/**
|
||||
* Extract relationship terms
|
||||
*/
|
||||
extractRelationshipTerms(query) {
|
||||
const terms = [];
|
||||
const relationshipWords = ['related', 'connected', 'linked', 'references', 'cites'];
|
||||
const words = query.toLowerCase().split(/\s+/);
|
||||
for (const word of words) {
|
||||
if (relationshipWords.includes(word)) {
|
||||
terms.push(word);
|
||||
}
|
||||
}
|
||||
return terms;
|
||||
}
|
||||
/**
|
||||
* Build field constraints from extracted terms
|
||||
*/
|
||||
buildFieldConstraints(fields) {
|
||||
const constraints = {};
|
||||
// Simple mapping for common fields
|
||||
for (const field of fields) {
|
||||
// This would be enhanced with actual value extraction
|
||||
constraints[field] = { exists: true };
|
||||
}
|
||||
return constraints;
|
||||
}
|
||||
/**
|
||||
* Find similar queries from history (without using Brainy)
|
||||
*/
|
||||
findSimilarQueries(embedding) {
|
||||
// Simple similarity check against recent history
|
||||
// This is just a placeholder - real implementation would use cosine similarity
|
||||
return [];
|
||||
}
|
||||
/**
|
||||
* Adapt a previous query for new input
|
||||
*/
|
||||
adaptQuery(newQuery, previousResult) {
|
||||
return previousResult;
|
||||
}
|
||||
/**
|
||||
* Extract entities from query
|
||||
*/
|
||||
async extractEntities(query) {
|
||||
// Could use the Entity Registry here if available
|
||||
return [];
|
||||
}
|
||||
/**
|
||||
* Build query from components
|
||||
*/
|
||||
buildQuery(query, intent, entities) {
|
||||
return {
|
||||
like: query,
|
||||
limit: 10
|
||||
};
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=naturalLanguageProcessorStatic.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"naturalLanguageProcessorStatic.js","sourceRoot":"","sources":["../../src/neural/naturalLanguageProcessorStatic.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,iBAAiB,EAAiB,MAAM,2BAA2B,CAAA;AAa5E,MAAM,OAAO,wBAAwB;IAGnC;QACE,IAAI,CAAC,YAAY,GAAG,EAAE,CAAA;QACtB,kDAAkD;IACpD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAI;QACR,sDAAsD;QACtD,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;IAC1B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,mBAAmB,CAAC,YAAoB,EAAE,cAAuB;QACrE,+DAA+D;QAC/D,MAAM,eAAe,GAAG,iBAAiB,CAAC,YAAY,EAAE,cAAc,CAAC,CAAA;QAEvE,iDAAiD;QACjD,IAAI,CAAC,eAAe,CAAC,KAAK,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAA;YAErD,+BAA+B;YAC/B,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,IAAI,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;gBAC5D,eAAe,CAAC,KAAK,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAA;YAClF,CAAC;QACH,CAAC;QAED,oDAAoD;QACpD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;YACrB,KAAK,EAAE,YAAY;YACnB,MAAM,EAAE,eAAe;YACvB,OAAO,EAAE,KAAK,CAAC,4CAA4C;SAC5D,CAAC,CAAA;QAEF,gDAAgD;QAChD,IAAI,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACnC,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAA;QAC3B,CAAC;QAED,OAAO,eAAe,CAAA;IACxB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa,CAAC,KAAa;QACvC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAA;QAEtC,oCAAoC;QACpC,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,CAAA;QAC5G,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAA;QAExE,2BAA2B;QAC3B,MAAM,aAAa,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,CAAC,CAAA;QACpF,MAAM,cAAc,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAA;QAExE,iBAAiB;QACjB,IAAI,IAAI,GAA+B,QAAQ,CAAA;QAC/C,IAAI,cAAc,IAAI,cAAc,EAAE,CAAC;YACrC,IAAI,GAAG,UAAU,CAAA;QACnB,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC1B,IAAI,GAAG,OAAO,CAAA;QAChB,CAAC;aAAM,IAAI,cAAc,EAAE,CAAC;YAC1B,IAAI,GAAG,OAAO,CAAA;QAChB,CAAC;QAED,OAAO;YACL,IAAI;YACJ,UAAU,EAAE,GAAG;YACf,cAAc,EAAE;gBACd,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;gBAClE,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;aACjF;SACF,CAAA;IACH,CAAC;IAED;;OAEG;IACK,iBAAiB,CAAC,KAAa;QACrC,MAAM,KAAK,GAAa,EAAE,CAAA;QAE1B,6CAA6C;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAChC,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;QAEzF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBACjD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAA;YAChC,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;OAEG;IACK,wBAAwB,CAAC,KAAa;QAC5C,MAAM,KAAK,GAAa,EAAE,CAAA;QAC1B,MAAM,iBAAiB,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,CAAA;QAEnF,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,iBAAiB,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACrC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClB,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAED;;OAEG;IACK,qBAAqB,CAAC,MAAgB;QAC5C,MAAM,WAAW,GAAwB,EAAE,CAAA;QAE3C,mCAAmC;QACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,sDAAsD;YACtD,WAAW,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACvC,CAAC;QAED,OAAO,WAAW,CAAA;IACpB,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAiB;QAK1C,iDAAiD;QACjD,+EAA+E;QAC/E,OAAO,EAAE,CAAA;IACX,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,QAAgB,EAAE,cAA2B;QAC9D,OAAO,cAAc,CAAA;IACvB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,eAAe,CAAC,KAAa;QACzC,kDAAkD;QAClD,OAAO,EAAE,CAAA;IACX,CAAC;IAED;;OAEG;IACK,UAAU,CAChB,KAAa,EACb,MAA0B,EAC1B,QAAkB;QAElB,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,EAAE;SACV,CAAA;IACH,CAAC;CACF"}
|
||||
255
.recovery-workspace/dist-backup-20250910-141917/neural/neuralAPI.d.ts
vendored
Normal file
255
.recovery-workspace/dist-backup-20250910-141917/neural/neuralAPI.d.ts
vendored
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
/**
|
||||
* Neural API - Unified Semantic Intelligence
|
||||
*
|
||||
* Best-of-both: Complete functionality + Enterprise performance
|
||||
* Combines rich features with O(n) algorithms for millions of items
|
||||
*/
|
||||
import { Vector } from '../coreTypes.js';
|
||||
export interface SimilarityResult {
|
||||
score: number;
|
||||
method?: string;
|
||||
confidence?: number;
|
||||
explanation?: string;
|
||||
hierarchy?: {
|
||||
sharedParent?: string;
|
||||
distance?: number;
|
||||
};
|
||||
breakdown?: {
|
||||
semantic?: number;
|
||||
taxonomic?: number;
|
||||
contextual?: number;
|
||||
};
|
||||
}
|
||||
export interface SimilarityOptions {
|
||||
explain?: boolean;
|
||||
includeBreakdown?: boolean;
|
||||
method?: 'cosine' | 'euclidean' | 'hybrid';
|
||||
}
|
||||
export interface SemanticCluster {
|
||||
id: string;
|
||||
centroid: Vector;
|
||||
members: string[];
|
||||
label?: string;
|
||||
confidence: number;
|
||||
depth?: number;
|
||||
size?: number;
|
||||
level?: number;
|
||||
center?: any;
|
||||
}
|
||||
export interface SemanticHierarchy {
|
||||
self: {
|
||||
id: string;
|
||||
type?: string;
|
||||
vector: Vector;
|
||||
};
|
||||
parent?: {
|
||||
id: string;
|
||||
type?: string;
|
||||
similarity: number;
|
||||
};
|
||||
grandparent?: {
|
||||
id: string;
|
||||
type?: string;
|
||||
similarity: number;
|
||||
};
|
||||
root?: {
|
||||
id: string;
|
||||
type?: string;
|
||||
similarity: number;
|
||||
};
|
||||
siblings?: Array<{
|
||||
id: string;
|
||||
similarity: number;
|
||||
}>;
|
||||
children?: Array<{
|
||||
id: string;
|
||||
similarity: number;
|
||||
}>;
|
||||
depth?: number;
|
||||
}
|
||||
export interface NeighborGraph {
|
||||
center: string;
|
||||
neighbors: Array<{
|
||||
id: string;
|
||||
similarity: number;
|
||||
type?: string;
|
||||
connections?: number;
|
||||
}>;
|
||||
edges?: Array<{
|
||||
source: string;
|
||||
target: string;
|
||||
weight: number;
|
||||
type?: string;
|
||||
}>;
|
||||
}
|
||||
export interface ClusterOptions {
|
||||
algorithm?: 'hierarchical' | 'kmeans' | 'sample' | 'stream';
|
||||
maxClusters?: number;
|
||||
threshold?: number;
|
||||
sampleSize?: number;
|
||||
strategy?: 'random' | 'diverse' | 'recent';
|
||||
level?: number;
|
||||
batchSize?: number;
|
||||
}
|
||||
export interface VisualizationData {
|
||||
format: 'force-directed' | 'hierarchical' | 'radial';
|
||||
nodes: Array<{
|
||||
id: string;
|
||||
x: number;
|
||||
y: number;
|
||||
z?: number;
|
||||
type?: string;
|
||||
cluster?: string;
|
||||
size?: number;
|
||||
}>;
|
||||
edges: Array<{
|
||||
source: string;
|
||||
target: string;
|
||||
weight: number;
|
||||
type?: string;
|
||||
}>;
|
||||
layout?: {
|
||||
dimensions: number;
|
||||
algorithm: string;
|
||||
bounds?: {
|
||||
width: number;
|
||||
height: number;
|
||||
depth?: number;
|
||||
};
|
||||
};
|
||||
clusters?: Array<{
|
||||
id: string;
|
||||
color: string;
|
||||
label?: string;
|
||||
size: number;
|
||||
}>;
|
||||
}
|
||||
export interface ClusteringStrategy {
|
||||
type: 'sample' | 'hierarchical' | 'stream' | 'hybrid';
|
||||
sampleSize?: number;
|
||||
maxClusters?: number;
|
||||
minClusterSize?: number;
|
||||
}
|
||||
export interface LODConfig {
|
||||
levels: number;
|
||||
itemsPerLevel: number[];
|
||||
zoomThresholds: number[];
|
||||
}
|
||||
/**
|
||||
* Neural API - Unified best-of-both implementation
|
||||
*/
|
||||
export declare class NeuralAPI {
|
||||
private brain;
|
||||
private similarityCache;
|
||||
private clusterCache;
|
||||
private hierarchyCache;
|
||||
constructor(brain: any);
|
||||
/**
|
||||
* Calculate similarity between any two items (smart detection)
|
||||
*/
|
||||
similar(a: any, b: any, options?: SimilarityOptions): Promise<number | SimilarityResult>;
|
||||
/**
|
||||
* Find semantic clusters (auto-detects best approach)
|
||||
* Now with enterprise performance!
|
||||
*/
|
||||
clusters(input?: any): Promise<SemanticCluster[]>;
|
||||
/**
|
||||
* Get semantic hierarchy for an item
|
||||
*/
|
||||
hierarchy(id: string): Promise<SemanticHierarchy>;
|
||||
/**
|
||||
* Find semantic neighbors for visualization
|
||||
*/
|
||||
neighbors(id: string, options?: {
|
||||
radius?: number;
|
||||
limit?: number;
|
||||
includeEdges?: boolean;
|
||||
}): Promise<NeighborGraph>;
|
||||
/**
|
||||
* Find semantic path between two items
|
||||
*/
|
||||
semanticPath(fromId: string, toId: string, options?: {
|
||||
maxHops?: number;
|
||||
algorithm?: 'breadth' | 'dijkstra';
|
||||
}): Promise<Array<{
|
||||
id: string;
|
||||
similarity: number;
|
||||
hop: number;
|
||||
}>>;
|
||||
/**
|
||||
* Detect semantic outliers
|
||||
*/
|
||||
outliers(threshold?: number): Promise<string[]>;
|
||||
/**
|
||||
* Generate visualization data
|
||||
*/
|
||||
visualize(options?: {
|
||||
maxNodes?: number;
|
||||
dimensions?: 2 | 3;
|
||||
algorithm?: 'force' | 'hierarchical' | 'radial';
|
||||
includeEdges?: boolean;
|
||||
}): Promise<VisualizationData>;
|
||||
/**
|
||||
* Fast clustering using HNSW levels - O(n) instead of O(n²)
|
||||
*/
|
||||
clusterFast(options?: {
|
||||
level?: number;
|
||||
maxClusters?: number;
|
||||
}): Promise<SemanticCluster[]>;
|
||||
/**
|
||||
* Large-scale clustering for massive datasets (millions of items)
|
||||
*/
|
||||
clusterLarge(options?: {
|
||||
sampleSize?: number;
|
||||
strategy?: 'random' | 'diverse' | 'recent';
|
||||
}): Promise<SemanticCluster[]>;
|
||||
/**
|
||||
* Streaming clustering for progressive refinement
|
||||
*/
|
||||
clusterStream(options?: {
|
||||
batchSize?: number;
|
||||
maxBatches?: number;
|
||||
}): AsyncGenerator<SemanticCluster[]>;
|
||||
/**
|
||||
* Level-of-detail for massive visualization
|
||||
*/
|
||||
getLOD(zoomLevel: number, viewport?: {
|
||||
center: Vector;
|
||||
radius: number;
|
||||
}): Promise<any>;
|
||||
private isId;
|
||||
private similarityById;
|
||||
private similarityByText;
|
||||
private similarityByVector;
|
||||
private smartSimilarity;
|
||||
private toVector;
|
||||
private getOptimalClusteringLevel;
|
||||
private getHNSWLevelNodes;
|
||||
private findClusterMembers;
|
||||
private getSample;
|
||||
private shuffleArray;
|
||||
private getDiverseSample;
|
||||
private performFastClustering;
|
||||
private calculateCentroid;
|
||||
private projectClustersToFullDataset;
|
||||
private mergeClusters;
|
||||
private averageVectors;
|
||||
private getBatch;
|
||||
private clusterAll;
|
||||
private clusterItems;
|
||||
private clustersNear;
|
||||
private clusterWithConfig;
|
||||
private buildHierarchy;
|
||||
private buildEdges;
|
||||
private dijkstraPath;
|
||||
private breadthFirstPath;
|
||||
private outliersViaSampling;
|
||||
private outliersByDistance;
|
||||
private getVisualizationNodes;
|
||||
private applyLayout;
|
||||
private buildVisualizationEdges;
|
||||
private detectOptimalFormat;
|
||||
private calculateBounds;
|
||||
private getViewportLOD;
|
||||
private getGlobalLOD;
|
||||
}
|
||||
|
|
@ -0,0 +1,612 @@
|
|||
/**
|
||||
* Neural API - Unified Semantic Intelligence
|
||||
*
|
||||
* Best-of-both: Complete functionality + Enterprise performance
|
||||
* Combines rich features with O(n) algorithms for millions of items
|
||||
*/
|
||||
import { cosineDistance } from '../utils/distance.js';
|
||||
/**
|
||||
* Neural API - Unified best-of-both implementation
|
||||
*/
|
||||
export class NeuralAPI {
|
||||
constructor(brain) {
|
||||
this.similarityCache = new Map();
|
||||
this.clusterCache = new Map(); // Enhanced for enterprise
|
||||
this.hierarchyCache = new Map();
|
||||
this.brain = brain;
|
||||
}
|
||||
// ===== SMART USER-FRIENDLY API =====
|
||||
/**
|
||||
* Calculate similarity between any two items (smart detection)
|
||||
*/
|
||||
async similar(a, b, options) {
|
||||
// Auto-detect input types
|
||||
if (typeof a === 'string' && typeof b === 'string') {
|
||||
if (this.isId(a) && this.isId(b)) {
|
||||
return this.similarityById(a, b, options);
|
||||
}
|
||||
else {
|
||||
return this.similarityByText(a, b, options);
|
||||
}
|
||||
}
|
||||
else if (Array.isArray(a) && Array.isArray(b)) {
|
||||
return this.similarityByVector(a, b, options);
|
||||
}
|
||||
// Handle mixed types
|
||||
return this.smartSimilarity(a, b, options);
|
||||
}
|
||||
/**
|
||||
* Find semantic clusters (auto-detects best approach)
|
||||
* Now with enterprise performance!
|
||||
*/
|
||||
async clusters(input) {
|
||||
// No input? Use enterprise fast clustering
|
||||
if (!input) {
|
||||
return this.clusterFast();
|
||||
}
|
||||
// Array? Cluster these items (use large clustering for big arrays)
|
||||
if (Array.isArray(input)) {
|
||||
if (input.length > 1000) {
|
||||
return this.clusterLarge({ sampleSize: Math.min(input.length, 1000) });
|
||||
}
|
||||
return this.clusterItems(input);
|
||||
}
|
||||
// String? Find clusters near this
|
||||
if (typeof input === 'string') {
|
||||
return this.clustersNear(input);
|
||||
}
|
||||
// Object? Use as config with enterprise algorithms
|
||||
if (typeof input === 'object' && !Array.isArray(input)) {
|
||||
return this.clusterWithConfig(input);
|
||||
}
|
||||
throw new Error('Invalid input for clustering');
|
||||
}
|
||||
/**
|
||||
* Get semantic hierarchy for an item
|
||||
*/
|
||||
async hierarchy(id) {
|
||||
// Check cache first
|
||||
if (this.hierarchyCache.has(id)) {
|
||||
return this.hierarchyCache.get(id);
|
||||
}
|
||||
const item = await this.brain.get(id);
|
||||
if (!item) {
|
||||
throw new Error(`Item not found: ${id}`);
|
||||
}
|
||||
// Find semantic relationships
|
||||
const hierarchy = await this.buildHierarchy(item);
|
||||
// Cache result
|
||||
this.hierarchyCache.set(id, hierarchy);
|
||||
return hierarchy;
|
||||
}
|
||||
/**
|
||||
* Find semantic neighbors for visualization
|
||||
*/
|
||||
async neighbors(id, options) {
|
||||
const radius = options?.radius ?? 0.3;
|
||||
const limit = options?.limit ?? 50;
|
||||
// Search for nearby items
|
||||
const results = await this.brain.search(id, limit * 2);
|
||||
// Filter by semantic radius
|
||||
const neighbors = results
|
||||
.filter((r) => r.similarity >= (1 - radius))
|
||||
.slice(0, limit)
|
||||
.map((r) => ({
|
||||
id: r.id,
|
||||
similarity: r.similarity,
|
||||
type: r.metadata?.type,
|
||||
connections: r.metadata?.connections?.size || 0
|
||||
}));
|
||||
const graph = {
|
||||
center: id,
|
||||
neighbors
|
||||
};
|
||||
// Add edges if requested
|
||||
if (options?.includeEdges) {
|
||||
graph.edges = await this.buildEdges(id, neighbors);
|
||||
}
|
||||
return graph;
|
||||
}
|
||||
/**
|
||||
* Find semantic path between two items
|
||||
*/
|
||||
async semanticPath(fromId, toId, options) {
|
||||
const maxHops = options?.maxHops ?? 5;
|
||||
const algorithm = options?.algorithm ?? 'breadth';
|
||||
if (algorithm === 'dijkstra') {
|
||||
return this.dijkstraPath(fromId, toId, maxHops);
|
||||
}
|
||||
else {
|
||||
return this.breadthFirstPath(fromId, toId, maxHops);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Detect semantic outliers
|
||||
*/
|
||||
async outliers(threshold = 0.3) {
|
||||
// Get all items
|
||||
const stats = await this.brain.getStatistics();
|
||||
const totalItems = stats.nounCount;
|
||||
if (totalItems === 0)
|
||||
return [];
|
||||
// For large datasets, use sampling
|
||||
if (totalItems > 10000) {
|
||||
return this.outliersViaSampling(threshold, 1000);
|
||||
}
|
||||
return this.outliersByDistance(threshold);
|
||||
}
|
||||
/**
|
||||
* Generate visualization data
|
||||
*/
|
||||
async visualize(options) {
|
||||
const maxNodes = options?.maxNodes ?? 100;
|
||||
const dimensions = options?.dimensions ?? 2;
|
||||
const algorithm = options?.algorithm ?? 'force';
|
||||
// Get representative nodes
|
||||
const nodes = await this.getVisualizationNodes(maxNodes);
|
||||
// Apply layout algorithm
|
||||
const positioned = await this.applyLayout(nodes, algorithm, dimensions);
|
||||
// Build edges if requested
|
||||
const edges = options?.includeEdges !== false ?
|
||||
await this.buildVisualizationEdges(positioned) : [];
|
||||
// Detect optimal format
|
||||
const format = this.detectOptimalFormat(positioned, edges);
|
||||
return {
|
||||
format,
|
||||
nodes: positioned,
|
||||
edges,
|
||||
layout: {
|
||||
dimensions,
|
||||
algorithm,
|
||||
bounds: this.calculateBounds(positioned, dimensions)
|
||||
}
|
||||
};
|
||||
}
|
||||
// ===== ENTERPRISE PERFORMANCE ALGORITHMS =====
|
||||
/**
|
||||
* Fast clustering using HNSW levels - O(n) instead of O(n²)
|
||||
*/
|
||||
async clusterFast(options = {}) {
|
||||
const cacheKey = `hierarchical-${options.level}-${options.maxClusters}`;
|
||||
if (this.clusterCache.has(cacheKey)) {
|
||||
return this.clusterCache.get(cacheKey);
|
||||
}
|
||||
// Use HNSW's natural hierarchy - auto-select optimal level
|
||||
const level = options.level ?? await this.getOptimalClusteringLevel();
|
||||
const maxClusters = options.maxClusters ?? 100;
|
||||
// Get representative nodes from HNSW level
|
||||
const representatives = await this.getHNSWLevelNodes(level);
|
||||
// Each representative is a natural cluster center
|
||||
const clusters = [];
|
||||
for (const rep of representatives.slice(0, maxClusters)) {
|
||||
const members = await this.findClusterMembers(rep, level - 1);
|
||||
clusters.push({
|
||||
id: `cluster-${rep.id}`,
|
||||
centroid: rep.vector,
|
||||
center: rep,
|
||||
members: members.map(m => m.id),
|
||||
size: members.length,
|
||||
level,
|
||||
confidence: 0.8 + (members.length / 100) * 0.2 // Size-based confidence
|
||||
});
|
||||
}
|
||||
this.clusterCache.set(cacheKey, clusters);
|
||||
return clusters;
|
||||
}
|
||||
/**
|
||||
* Large-scale clustering for massive datasets (millions of items)
|
||||
*/
|
||||
async clusterLarge(options = {}) {
|
||||
const sampleSize = options.sampleSize ?? 1000;
|
||||
const strategy = options.strategy ?? 'diverse';
|
||||
// Get representative sample
|
||||
const sample = await this.getSample(sampleSize, strategy);
|
||||
// Cluster the sample (fast on small set)
|
||||
const sampleClusters = await this.performFastClustering(sample);
|
||||
// Project clusters to full dataset
|
||||
return this.projectClustersToFullDataset(sampleClusters);
|
||||
}
|
||||
/**
|
||||
* Streaming clustering for progressive refinement
|
||||
*/
|
||||
async *clusterStream(options = {}) {
|
||||
const batchSize = options.batchSize ?? 1000;
|
||||
const maxBatches = options.maxBatches ?? Infinity;
|
||||
let offset = 0;
|
||||
let batchCount = 0;
|
||||
let globalClusters = [];
|
||||
while (batchCount < maxBatches) {
|
||||
// Get next batch
|
||||
const batch = await this.getBatch(offset, batchSize);
|
||||
if (batch.length === 0)
|
||||
break;
|
||||
// Cluster this batch
|
||||
const batchClusters = await this.performFastClustering(batch);
|
||||
// Merge with global clusters
|
||||
globalClusters = await this.mergeClusters(globalClusters, batchClusters);
|
||||
// Yield current state
|
||||
yield globalClusters;
|
||||
offset += batchSize;
|
||||
batchCount++;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Level-of-detail for massive visualization
|
||||
*/
|
||||
async getLOD(zoomLevel, viewport) {
|
||||
// Define LOD levels based on zoom
|
||||
const lodLevels = [
|
||||
{ zoom: 0, maxNodes: 50, clusterLevel: 3 },
|
||||
{ zoom: 1, maxNodes: 200, clusterLevel: 2 },
|
||||
{ zoom: 2, maxNodes: 1000, clusterLevel: 1 },
|
||||
{ zoom: 3, maxNodes: 5000, clusterLevel: 0 }
|
||||
];
|
||||
const lod = lodLevels.find(l => zoomLevel <= l.zoom) || lodLevels[lodLevels.length - 1];
|
||||
if (viewport) {
|
||||
return this.getViewportLOD(viewport, lod);
|
||||
}
|
||||
else {
|
||||
return this.getGlobalLOD(lod);
|
||||
}
|
||||
}
|
||||
// ===== IMPLEMENTATION HELPERS =====
|
||||
isId(str) {
|
||||
// Check if string looks like an ID (UUID pattern, etc.)
|
||||
return (str.length === 36 && str.includes('-')) || !!str.match(/^[a-f0-9]{24}$/);
|
||||
}
|
||||
async similarityById(idA, idB, options) {
|
||||
const cacheKey = `${idA}-${idB}`;
|
||||
if (this.similarityCache.has(cacheKey)) {
|
||||
return this.similarityCache.get(cacheKey);
|
||||
}
|
||||
// Get items
|
||||
const [itemA, itemB] = await Promise.all([
|
||||
this.brain.get(idA),
|
||||
this.brain.get(idB)
|
||||
]);
|
||||
if (!itemA || !itemB) {
|
||||
throw new Error('One or both items not found');
|
||||
}
|
||||
// Calculate similarity
|
||||
const score = cosineDistance(itemA.vector, itemB.vector);
|
||||
this.similarityCache.set(cacheKey, score);
|
||||
if (options?.explain) {
|
||||
return {
|
||||
score,
|
||||
method: 'cosine',
|
||||
confidence: 0.9,
|
||||
explanation: `Semantic similarity between ${idA} and ${idB}`
|
||||
};
|
||||
}
|
||||
return score;
|
||||
}
|
||||
async similarityByText(textA, textB, options) {
|
||||
// Generate embeddings
|
||||
const [vectorA, vectorB] = await Promise.all([
|
||||
this.brain.embed(textA),
|
||||
this.brain.embed(textB)
|
||||
]);
|
||||
return this.similarityByVector(vectorA, vectorB, options);
|
||||
}
|
||||
async similarityByVector(vectorA, vectorB, options) {
|
||||
const score = cosineDistance(vectorA, vectorB);
|
||||
if (options?.explain) {
|
||||
return {
|
||||
score,
|
||||
method: options.method || 'cosine',
|
||||
confidence: 0.95,
|
||||
explanation: 'Direct vector similarity calculation'
|
||||
};
|
||||
}
|
||||
return score;
|
||||
}
|
||||
async smartSimilarity(a, b, options) {
|
||||
// Convert both to vectors and compare
|
||||
const vectorA = await this.toVector(a);
|
||||
const vectorB = await this.toVector(b);
|
||||
return this.similarityByVector(vectorA, vectorB, options);
|
||||
}
|
||||
async toVector(item) {
|
||||
if (Array.isArray(item))
|
||||
return item;
|
||||
if (typeof item === 'string') {
|
||||
if (this.isId(item)) {
|
||||
const found = await this.brain.get(item);
|
||||
return found?.vector || await this.brain.embed(item);
|
||||
}
|
||||
return await this.brain.embed(item);
|
||||
}
|
||||
if (typeof item === 'object' && item.vector) {
|
||||
return item.vector;
|
||||
}
|
||||
// Convert object to string and embed
|
||||
return await this.brain.embed(JSON.stringify(item));
|
||||
}
|
||||
// Enterprise clustering implementations
|
||||
async getOptimalClusteringLevel() {
|
||||
// Analyze dataset size and return optimal HNSW level
|
||||
const stats = await this.brain.getStatistics();
|
||||
const itemCount = stats.nounCount;
|
||||
if (itemCount < 1000)
|
||||
return 0;
|
||||
if (itemCount < 10000)
|
||||
return 1;
|
||||
if (itemCount < 100000)
|
||||
return 2;
|
||||
return 3;
|
||||
}
|
||||
async getHNSWLevelNodes(level) {
|
||||
// Get nodes from specific HNSW level
|
||||
// For now, use search to get a representative sample
|
||||
const stats = await this.brain.getStatistics();
|
||||
const sampleSize = Math.min(100, Math.floor(stats.nounCount / (level + 1)));
|
||||
// Use search with a general query to get representative items
|
||||
const queryVector = await this.brain.embed('data information content');
|
||||
const allItems = await this.brain.search(queryVector, sampleSize * 2);
|
||||
return allItems.slice(0, sampleSize);
|
||||
}
|
||||
async findClusterMembers(center, level) {
|
||||
// Find all items that belong to this cluster
|
||||
const results = await this.brain.search(center.vector, 50);
|
||||
return results.filter((r) => r.similarity > 0.7);
|
||||
}
|
||||
async getSample(size, strategy) {
|
||||
// Use search to get a sample of items
|
||||
const stats = await this.brain.getStatistics();
|
||||
const maxSize = Math.min(size * 3, stats.nounCount); // Get more than needed for sampling
|
||||
const queryVector = await this.brain.embed('sample data content');
|
||||
const allItems = await this.brain.search(queryVector, maxSize);
|
||||
switch (strategy) {
|
||||
case 'random':
|
||||
return this.shuffleArray(allItems).slice(0, size);
|
||||
case 'diverse':
|
||||
return this.getDiverseSample(allItems, size);
|
||||
case 'recent':
|
||||
return allItems.slice(-size);
|
||||
default:
|
||||
return allItems.slice(0, size);
|
||||
}
|
||||
}
|
||||
shuffleArray(array) {
|
||||
const shuffled = [...array];
|
||||
for (let i = shuffled.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
||||
}
|
||||
return shuffled;
|
||||
}
|
||||
async getDiverseSample(items, size) {
|
||||
// Select diverse items using maximum distance sampling
|
||||
if (items.length <= size)
|
||||
return items;
|
||||
const sample = [items[0]]; // Start with first item
|
||||
for (let i = 1; i < size; i++) {
|
||||
let maxMinDistance = -1;
|
||||
let bestItem = null;
|
||||
for (const candidate of items) {
|
||||
if (sample.includes(candidate))
|
||||
continue;
|
||||
// Find minimum distance to existing sample
|
||||
let minDistance = Infinity;
|
||||
for (const selected of sample) {
|
||||
const distance = cosineDistance(candidate.vector, selected.vector);
|
||||
minDistance = Math.min(minDistance, distance);
|
||||
}
|
||||
// Select item with maximum minimum distance
|
||||
if (minDistance > maxMinDistance) {
|
||||
maxMinDistance = minDistance;
|
||||
bestItem = candidate;
|
||||
}
|
||||
}
|
||||
if (bestItem)
|
||||
sample.push(bestItem);
|
||||
}
|
||||
return sample;
|
||||
}
|
||||
async performFastClustering(items) {
|
||||
// Simple k-means clustering for the sample
|
||||
const k = Math.min(10, Math.floor(items.length / 3));
|
||||
if (k <= 1) {
|
||||
return [{
|
||||
id: 'cluster-0',
|
||||
centroid: items[0]?.vector || [],
|
||||
members: items.map(i => i.id),
|
||||
confidence: 1.0
|
||||
}];
|
||||
}
|
||||
// Initialize centroids randomly
|
||||
const centroids = items.slice(0, k).map(item => item.vector);
|
||||
// Run k-means iterations (simplified)
|
||||
for (let iter = 0; iter < 10; iter++) {
|
||||
const clusters = Array(k).fill(null).map(() => []);
|
||||
// Assign items to nearest centroid
|
||||
for (const item of items) {
|
||||
let bestCluster = 0;
|
||||
let bestDistance = Infinity;
|
||||
for (let c = 0; c < k; c++) {
|
||||
const distance = cosineDistance(item.vector, centroids[c]);
|
||||
if (distance < bestDistance) {
|
||||
bestDistance = distance;
|
||||
bestCluster = c;
|
||||
}
|
||||
}
|
||||
clusters[bestCluster].push(item);
|
||||
}
|
||||
// Update centroids
|
||||
for (let c = 0; c < k; c++) {
|
||||
if (clusters[c].length > 0) {
|
||||
const newCentroid = this.calculateCentroid(clusters[c]);
|
||||
centroids[c] = newCentroid;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Convert to SemanticCluster format
|
||||
const result = [];
|
||||
for (let c = 0; c < k; c++) {
|
||||
const members = items.filter(item => {
|
||||
let bestCluster = 0;
|
||||
let bestDistance = Infinity;
|
||||
for (let cc = 0; cc < k; cc++) {
|
||||
const distance = cosineDistance(item.vector, centroids[cc]);
|
||||
if (distance < bestDistance) {
|
||||
bestDistance = distance;
|
||||
bestCluster = cc;
|
||||
}
|
||||
}
|
||||
return bestCluster === c;
|
||||
});
|
||||
if (members.length > 0) {
|
||||
result.push({
|
||||
id: `cluster-${c}`,
|
||||
centroid: centroids[c],
|
||||
members: members.map(m => m.id),
|
||||
confidence: Math.min(0.9, members.length / items.length * 2)
|
||||
});
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
calculateCentroid(items) {
|
||||
if (items.length === 0)
|
||||
return [];
|
||||
const dimensions = items[0].vector.length;
|
||||
const centroid = new Array(dimensions).fill(0);
|
||||
for (const item of items) {
|
||||
for (let d = 0; d < dimensions; d++) {
|
||||
centroid[d] += item.vector[d];
|
||||
}
|
||||
}
|
||||
for (let d = 0; d < dimensions; d++) {
|
||||
centroid[d] /= items.length;
|
||||
}
|
||||
return centroid;
|
||||
}
|
||||
async projectClustersToFullDataset(sampleClusters) {
|
||||
// Project sample clusters to full dataset
|
||||
const result = [];
|
||||
for (const cluster of sampleClusters) {
|
||||
// Find all items similar to this cluster's centroid
|
||||
const similar = await this.brain.search(cluster.centroid, 1000);
|
||||
const members = similar
|
||||
.filter((s) => s.similarity > 0.6)
|
||||
.map((s) => s.id);
|
||||
result.push({
|
||||
...cluster,
|
||||
members,
|
||||
size: members.length
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
async mergeClusters(globalClusters, batchClusters) {
|
||||
// Simple merge strategy - combine similar clusters
|
||||
const result = [...globalClusters];
|
||||
for (const batchCluster of batchClusters) {
|
||||
let merged = false;
|
||||
for (let i = 0; i < result.length; i++) {
|
||||
const similarity = cosineDistance(result[i].centroid, batchCluster.centroid);
|
||||
if (similarity > 0.8) {
|
||||
// Merge clusters
|
||||
const newMembers = [...new Set([...result[i].members, ...batchCluster.members])];
|
||||
result[i] = {
|
||||
...result[i],
|
||||
members: newMembers,
|
||||
size: newMembers.length,
|
||||
centroid: this.averageVectors(result[i].centroid, batchCluster.centroid)
|
||||
};
|
||||
merged = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!merged) {
|
||||
result.push(batchCluster);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
averageVectors(v1, v2) {
|
||||
const result = new Array(v1.length);
|
||||
for (let i = 0; i < v1.length; i++) {
|
||||
result[i] = (v1[i] + v2[i]) / 2;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
async getBatch(offset, size) {
|
||||
// Get batch of items for streaming using search with offset
|
||||
const queryVector = await this.brain.embed('batch data content');
|
||||
const items = await this.brain.search(queryVector, size, { offset });
|
||||
return items;
|
||||
}
|
||||
// Additional methods needed for full compatibility...
|
||||
async clusterAll() {
|
||||
return this.clusterFast();
|
||||
}
|
||||
async clusterItems(items) {
|
||||
return this.performFastClustering(items);
|
||||
}
|
||||
async clustersNear(id) {
|
||||
const neighbors = await this.neighbors(id, { limit: 100 });
|
||||
return this.performFastClustering(neighbors.neighbors);
|
||||
}
|
||||
async clusterWithConfig(config) {
|
||||
switch (config.algorithm) {
|
||||
case 'hierarchical':
|
||||
return this.clusterFast(config);
|
||||
case 'sample':
|
||||
return this.clusterLarge(config);
|
||||
case 'stream':
|
||||
const generator = this.clusterStream(config);
|
||||
const results = [];
|
||||
for await (const batch of generator) {
|
||||
results.push(...batch);
|
||||
}
|
||||
return results;
|
||||
default:
|
||||
return this.clusterFast(config);
|
||||
}
|
||||
}
|
||||
// Placeholder implementations for remaining methods
|
||||
async buildHierarchy(item) {
|
||||
// Implementation for hierarchy building
|
||||
return {
|
||||
self: { id: item.id, vector: item.vector }
|
||||
};
|
||||
}
|
||||
async buildEdges(centerId, neighbors) {
|
||||
return [];
|
||||
}
|
||||
async dijkstraPath(from, to, maxHops) {
|
||||
return [];
|
||||
}
|
||||
async breadthFirstPath(from, to, maxHops) {
|
||||
return [];
|
||||
}
|
||||
async outliersViaSampling(threshold, sampleSize) {
|
||||
return [];
|
||||
}
|
||||
async outliersByDistance(threshold) {
|
||||
return [];
|
||||
}
|
||||
async getVisualizationNodes(maxNodes) {
|
||||
return [];
|
||||
}
|
||||
async applyLayout(nodes, algorithm, dimensions) {
|
||||
return nodes;
|
||||
}
|
||||
async buildVisualizationEdges(nodes) {
|
||||
return [];
|
||||
}
|
||||
detectOptimalFormat(nodes, edges) {
|
||||
return 'force-directed';
|
||||
}
|
||||
calculateBounds(nodes, dimensions) {
|
||||
return { width: 100, height: 100 };
|
||||
}
|
||||
async getViewportLOD(viewport, lod) {
|
||||
return {};
|
||||
}
|
||||
async getGlobalLOD(lod) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=neuralAPI.js.map
|
||||
File diff suppressed because one or more lines are too long
155
.recovery-workspace/dist-backup-20250910-141917/neural/patternLibrary.d.ts
vendored
Normal file
155
.recovery-workspace/dist-backup-20250910-141917/neural/patternLibrary.d.ts
vendored
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
/**
|
||||
* 🧠 Pattern Library for Natural Language Processing
|
||||
* Manages pre-computed pattern embeddings and smart matching
|
||||
*
|
||||
* Uses Brainy's own features for self-leveraging intelligence:
|
||||
* - Embeddings for semantic similarity
|
||||
* - Pattern caching for performance
|
||||
* - Progressive learning from usage
|
||||
*/
|
||||
import { Vector } from '../coreTypes.js';
|
||||
import { Brainy } from '../brainy.js';
|
||||
export interface Pattern {
|
||||
id: string;
|
||||
category: string;
|
||||
examples: string[];
|
||||
pattern: string;
|
||||
template: any;
|
||||
confidence: number;
|
||||
embedding?: Vector;
|
||||
domain?: string;
|
||||
frequency?: number | string;
|
||||
slots?: SlotDefinition[];
|
||||
}
|
||||
export interface SlotDefinition {
|
||||
name: string;
|
||||
type: 'text' | 'number' | 'date' | 'entity' | 'location' | 'person' | 'any';
|
||||
required?: boolean;
|
||||
default?: any;
|
||||
pattern?: string;
|
||||
transform?: (value: string) => any;
|
||||
}
|
||||
export interface SlotExtraction {
|
||||
slots: Record<string, any>;
|
||||
confidence: number;
|
||||
errors?: string[];
|
||||
}
|
||||
export declare class PatternLibrary {
|
||||
private patterns;
|
||||
private patternEmbeddings;
|
||||
private brain;
|
||||
private embeddingCache;
|
||||
private successMetrics;
|
||||
constructor(brain: Brainy);
|
||||
/**
|
||||
* Initialize pattern library with pre-computed embeddings
|
||||
*/
|
||||
init(): Promise<void>;
|
||||
/**
|
||||
* Pre-compute embeddings for all patterns for fast matching
|
||||
*/
|
||||
private precomputeEmbeddings;
|
||||
/**
|
||||
* Get embedding with caching
|
||||
*/
|
||||
private getEmbedding;
|
||||
/**
|
||||
* Find best matching patterns for a query
|
||||
*/
|
||||
findBestPatterns(queryEmbedding: Vector, k?: number): Promise<Array<{
|
||||
pattern: Pattern;
|
||||
similarity: number;
|
||||
}>>;
|
||||
/**
|
||||
* Extract slots from query based on pattern with enhanced fuzzy matching
|
||||
*/
|
||||
extractSlots(query: string, pattern: Pattern): SlotExtraction;
|
||||
/**
|
||||
* Extract named slots with type validation
|
||||
*/
|
||||
private extractNamedSlots;
|
||||
/**
|
||||
* Fuzzy extraction using Levenshtein distance
|
||||
*/
|
||||
private fuzzyExtractSlots;
|
||||
/**
|
||||
* Fuzzy extraction for named slots
|
||||
*/
|
||||
private fuzzyExtractNamedSlots;
|
||||
/**
|
||||
* Find slot value in tokens based on type
|
||||
*/
|
||||
private findSlotValueInTokens;
|
||||
/**
|
||||
* Get default regex pattern for slot type
|
||||
*/
|
||||
private getDefaultPatternForType;
|
||||
/**
|
||||
* Transform value based on type
|
||||
*/
|
||||
private transformByType;
|
||||
/**
|
||||
* Validate slot value against definition
|
||||
*/
|
||||
private validateSlotValue;
|
||||
/**
|
||||
* Calculate Levenshtein distance between two strings
|
||||
*/
|
||||
private levenshteinDistance;
|
||||
/**
|
||||
* Align two strings for slot extraction
|
||||
*/
|
||||
private alignStrings;
|
||||
/**
|
||||
* Find best token match using fuzzy comparison
|
||||
*/
|
||||
private findBestTokenMatch;
|
||||
/**
|
||||
* Extract slots from string alignment
|
||||
*/
|
||||
private extractSlotsFromAlignment;
|
||||
/**
|
||||
* Fill template with extracted slots
|
||||
*/
|
||||
fillTemplate(template: any, slots: Record<string, any>): any;
|
||||
/**
|
||||
* Update pattern success metrics based on usage
|
||||
*/
|
||||
updateSuccessMetric(patternId: string, success: boolean): void;
|
||||
/**
|
||||
* Learn new pattern from successful query
|
||||
*/
|
||||
learnPattern(query: string, result: any): Promise<void>;
|
||||
/**
|
||||
* Helper: Average multiple vectors
|
||||
*/
|
||||
private averageVectors;
|
||||
/**
|
||||
* Helper: Calculate cosine similarity
|
||||
*/
|
||||
private cosineSimilarity;
|
||||
/**
|
||||
* Helper: Simple tokenization
|
||||
*/
|
||||
private tokenize;
|
||||
/**
|
||||
* Helper: Post-process extracted slots
|
||||
*/
|
||||
private postProcessSlots;
|
||||
/**
|
||||
* Helper: Generate regex pattern from query
|
||||
*/
|
||||
private generateRegexFromQuery;
|
||||
/**
|
||||
* Get pattern statistics for monitoring
|
||||
*/
|
||||
getStatistics(): {
|
||||
totalPatterns: number;
|
||||
categories: Record<string, number>;
|
||||
averageConfidence: number;
|
||||
topPatterns: Array<{
|
||||
id: string;
|
||||
success: number;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,655 @@
|
|||
/**
|
||||
* 🧠 Pattern Library for Natural Language Processing
|
||||
* Manages pre-computed pattern embeddings and smart matching
|
||||
*
|
||||
* Uses Brainy's own features for self-leveraging intelligence:
|
||||
* - Embeddings for semantic similarity
|
||||
* - Pattern caching for performance
|
||||
* - Progressive learning from usage
|
||||
*/
|
||||
import { EMBEDDED_PATTERNS, getPatternEmbeddings, PATTERNS_METADATA } from './embeddedPatterns.js';
|
||||
export class PatternLibrary {
|
||||
constructor(brain) {
|
||||
this.brain = brain;
|
||||
this.patterns = new Map();
|
||||
this.patternEmbeddings = new Map();
|
||||
this.embeddingCache = new Map();
|
||||
this.successMetrics = new Map();
|
||||
}
|
||||
/**
|
||||
* Initialize pattern library with pre-computed embeddings
|
||||
*/
|
||||
async init() {
|
||||
// Try to load pre-computed embeddings first
|
||||
const precomputedEmbeddings = getPatternEmbeddings();
|
||||
if (precomputedEmbeddings.size > 0) {
|
||||
// Use pre-computed embeddings (instant!)
|
||||
console.debug(`Loading ${precomputedEmbeddings.size} pre-computed pattern embeddings`);
|
||||
for (const pattern of EMBEDDED_PATTERNS) {
|
||||
this.patterns.set(pattern.id, pattern);
|
||||
this.successMetrics.set(pattern.id, pattern.confidence);
|
||||
const embedding = precomputedEmbeddings.get(pattern.id);
|
||||
if (embedding) {
|
||||
this.patternEmbeddings.set(pattern.id, Array.from(embedding));
|
||||
}
|
||||
}
|
||||
console.debug(`Pattern library ready: ${PATTERNS_METADATA.totalPatterns} patterns loaded instantly`);
|
||||
}
|
||||
else {
|
||||
// Fall back to runtime computation
|
||||
console.debug('No pre-computed embeddings found, computing at runtime...');
|
||||
for (const pattern of EMBEDDED_PATTERNS) {
|
||||
this.patterns.set(pattern.id, pattern);
|
||||
this.successMetrics.set(pattern.id, pattern.confidence);
|
||||
}
|
||||
// Compute embeddings for all patterns
|
||||
await this.precomputeEmbeddings();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Pre-compute embeddings for all patterns for fast matching
|
||||
*/
|
||||
async precomputeEmbeddings() {
|
||||
for (const [id, pattern] of this.patterns) {
|
||||
// Average embeddings of all examples for robust representation
|
||||
const embeddings = [];
|
||||
for (const example of pattern.examples) {
|
||||
const embedding = await this.getEmbedding(example);
|
||||
embeddings.push(embedding);
|
||||
}
|
||||
// Average the embeddings
|
||||
const avgEmbedding = this.averageVectors(embeddings);
|
||||
this.patternEmbeddings.set(id, avgEmbedding);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get embedding with caching
|
||||
*/
|
||||
async getEmbedding(text) {
|
||||
if (this.embeddingCache.has(text)) {
|
||||
return this.embeddingCache.get(text);
|
||||
}
|
||||
// Use add/get/delete pattern to get embeddings
|
||||
const id = await this.brain.add({
|
||||
data: text,
|
||||
type: 'document'
|
||||
});
|
||||
const entity = await this.brain.get(id);
|
||||
const embedding = entity?.vector || [];
|
||||
// Clean up temporary entity
|
||||
await this.brain.delete(id);
|
||||
this.embeddingCache.set(text, embedding);
|
||||
return embedding;
|
||||
}
|
||||
/**
|
||||
* Find best matching patterns for a query
|
||||
*/
|
||||
async findBestPatterns(queryEmbedding, k = 3) {
|
||||
const matches = [];
|
||||
// Calculate similarity with all patterns
|
||||
for (const [id, patternEmbedding] of this.patternEmbeddings) {
|
||||
const similarity = this.cosineSimilarity(queryEmbedding, patternEmbedding);
|
||||
const pattern = this.patterns.get(id);
|
||||
// Apply success metric boost
|
||||
const successBoost = this.successMetrics.get(id) || 0.5;
|
||||
const adjustedSimilarity = similarity * (0.7 + 0.3 * successBoost);
|
||||
matches.push({
|
||||
pattern,
|
||||
similarity: adjustedSimilarity
|
||||
});
|
||||
}
|
||||
// Sort by similarity and return top k
|
||||
matches.sort((a, b) => b.similarity - a.similarity);
|
||||
return matches.slice(0, k);
|
||||
}
|
||||
/**
|
||||
* Extract slots from query based on pattern with enhanced fuzzy matching
|
||||
*/
|
||||
extractSlots(query, pattern) {
|
||||
const slots = {};
|
||||
const errors = [];
|
||||
let confidence = pattern.confidence;
|
||||
// If pattern has named slot definitions, use them
|
||||
if (pattern.slots && pattern.slots.length > 0) {
|
||||
return this.extractNamedSlots(query, pattern);
|
||||
}
|
||||
// Try regex extraction first
|
||||
const regex = new RegExp(pattern.pattern, 'i');
|
||||
const match = query.match(regex);
|
||||
if (match) {
|
||||
// Extract captured groups as slots
|
||||
for (let i = 1; i < match.length; i++) {
|
||||
slots[`$${i}`] = match[i];
|
||||
}
|
||||
// High confidence if regex matches
|
||||
confidence = Math.min(confidence * 1.2, 1.0);
|
||||
}
|
||||
else {
|
||||
// Enhanced fuzzy matching with Levenshtein distance
|
||||
const fuzzyResult = this.fuzzyExtractSlots(query, pattern);
|
||||
Object.assign(slots, fuzzyResult.slots);
|
||||
confidence = fuzzyResult.confidence;
|
||||
if (fuzzyResult.errors) {
|
||||
errors.push(...fuzzyResult.errors);
|
||||
}
|
||||
}
|
||||
// Post-process slots
|
||||
this.postProcessSlots(slots, pattern);
|
||||
return { slots, confidence, errors: errors.length > 0 ? errors : undefined };
|
||||
}
|
||||
/**
|
||||
* Extract named slots with type validation
|
||||
*/
|
||||
extractNamedSlots(query, pattern) {
|
||||
const slots = {};
|
||||
const errors = [];
|
||||
let confidence = pattern.confidence;
|
||||
if (!pattern.slots) {
|
||||
return { slots, confidence };
|
||||
}
|
||||
// Create a flexible regex from pattern
|
||||
let flexiblePattern = pattern.pattern;
|
||||
const slotPositions = new Map();
|
||||
// Replace named slots in pattern with capture groups
|
||||
pattern.slots.forEach((slot, index) => {
|
||||
const slotPattern = slot.pattern || this.getDefaultPatternForType(slot.type);
|
||||
flexiblePattern = flexiblePattern.replace(new RegExp(`\\{${slot.name}\\}`, 'g'), `(${slotPattern})`);
|
||||
slotPositions.set(index + 1, slot);
|
||||
});
|
||||
const regex = new RegExp(flexiblePattern, 'i');
|
||||
const match = query.match(regex);
|
||||
if (match) {
|
||||
// Extract and validate each slot
|
||||
slotPositions.forEach((slotDef, position) => {
|
||||
const value = match[position];
|
||||
if (value) {
|
||||
// Apply transformation if defined
|
||||
const transformedValue = slotDef.transform
|
||||
? slotDef.transform(value)
|
||||
: this.transformByType(value, slotDef.type);
|
||||
// Validate the value
|
||||
if (this.validateSlotValue(transformedValue, slotDef)) {
|
||||
slots[slotDef.name] = transformedValue;
|
||||
}
|
||||
else {
|
||||
errors.push(`Invalid value for slot '${slotDef.name}': expected ${slotDef.type}, got '${value}'`);
|
||||
confidence *= 0.8;
|
||||
}
|
||||
}
|
||||
else if (slotDef.required) {
|
||||
if (slotDef.default !== undefined) {
|
||||
slots[slotDef.name] = slotDef.default;
|
||||
}
|
||||
else {
|
||||
errors.push(`Required slot '${slotDef.name}' not found`);
|
||||
confidence *= 0.5;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
else {
|
||||
// Try fuzzy matching for named slots
|
||||
const fuzzyResult = this.fuzzyExtractNamedSlots(query, pattern);
|
||||
Object.assign(slots, fuzzyResult.slots);
|
||||
confidence = fuzzyResult.confidence;
|
||||
if (fuzzyResult.errors) {
|
||||
errors.push(...fuzzyResult.errors);
|
||||
}
|
||||
}
|
||||
return { slots, confidence, errors: errors.length > 0 ? errors : undefined };
|
||||
}
|
||||
/**
|
||||
* Fuzzy extraction using Levenshtein distance
|
||||
*/
|
||||
fuzzyExtractSlots(query, pattern) {
|
||||
const slots = {};
|
||||
let bestConfidence = 0;
|
||||
// Try each example with fuzzy matching
|
||||
for (const example of pattern.examples) {
|
||||
const distance = this.levenshteinDistance(query.toLowerCase(), example.toLowerCase());
|
||||
const similarity = 1 - (distance / Math.max(query.length, example.length));
|
||||
if (similarity > 0.6) { // 60% similarity threshold
|
||||
// Extract slots using alignment
|
||||
const aligned = this.alignStrings(query, example);
|
||||
const extractedSlots = this.extractSlotsFromAlignment(aligned, pattern);
|
||||
if (Object.keys(extractedSlots).length > 0) {
|
||||
const currentConfidence = pattern.confidence * similarity;
|
||||
if (currentConfidence > bestConfidence) {
|
||||
Object.assign(slots, extractedSlots);
|
||||
bestConfidence = currentConfidence;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
slots,
|
||||
confidence: bestConfidence,
|
||||
errors: bestConfidence < 0.5 ? ['Low confidence fuzzy match'] : undefined
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Fuzzy extraction for named slots
|
||||
*/
|
||||
fuzzyExtractNamedSlots(query, pattern) {
|
||||
const slots = {};
|
||||
const errors = [];
|
||||
let confidence = pattern.confidence * 0.7; // Lower confidence for fuzzy
|
||||
if (!pattern.slots) {
|
||||
return { slots, confidence };
|
||||
}
|
||||
// Tokenize query for flexible matching
|
||||
const tokens = this.tokenize(query);
|
||||
pattern.slots.forEach(slotDef => {
|
||||
const value = this.findSlotValueInTokens(tokens, slotDef);
|
||||
if (value) {
|
||||
const transformedValue = slotDef.transform
|
||||
? slotDef.transform(value)
|
||||
: this.transformByType(value, slotDef.type);
|
||||
if (this.validateSlotValue(transformedValue, slotDef)) {
|
||||
slots[slotDef.name] = transformedValue;
|
||||
}
|
||||
else {
|
||||
errors.push(`Fuzzy match: uncertain value for '${slotDef.name}'`);
|
||||
confidence *= 0.9;
|
||||
}
|
||||
}
|
||||
else if (slotDef.required && slotDef.default !== undefined) {
|
||||
slots[slotDef.name] = slotDef.default;
|
||||
}
|
||||
});
|
||||
return { slots, confidence, errors: errors.length > 0 ? errors : undefined };
|
||||
}
|
||||
/**
|
||||
* Find slot value in tokens based on type
|
||||
*/
|
||||
findSlotValueInTokens(tokens, slotDef) {
|
||||
const joinedTokens = tokens.join(' ');
|
||||
switch (slotDef.type) {
|
||||
case 'number':
|
||||
const numberMatch = joinedTokens.match(/\d+(\.\d+)?/);
|
||||
return numberMatch ? numberMatch[0] : null;
|
||||
case 'date':
|
||||
const datePatterns = [
|
||||
/\d{4}-\d{2}-\d{2}/,
|
||||
/\d{1,2}\/\d{1,2}\/\d{2,4}/,
|
||||
/(january|february|march|april|may|june|july|august|september|october|november|december)\s+\d{1,2},?\s+\d{4}/i,
|
||||
/(today|tomorrow|yesterday)/i
|
||||
];
|
||||
for (const pattern of datePatterns) {
|
||||
const match = joinedTokens.match(pattern);
|
||||
if (match)
|
||||
return match[0];
|
||||
}
|
||||
return null;
|
||||
case 'person':
|
||||
// Look for capitalized words (proper nouns)
|
||||
const personMatch = joinedTokens.match(/\b[A-Z][a-z]+(\s+[A-Z][a-z]+)*\b/);
|
||||
return personMatch ? personMatch[0] : null;
|
||||
case 'location':
|
||||
// Look for location indicators
|
||||
const locationPatterns = [
|
||||
/\b(in|at|from|to)\s+([A-Z][a-z]+(\s+[A-Z][a-z]+)*)\b/,
|
||||
/\b[A-Z][a-z]+,\s+[A-Z]{2}\b/ // City, STATE format
|
||||
];
|
||||
for (const pattern of locationPatterns) {
|
||||
const match = joinedTokens.match(pattern);
|
||||
if (match)
|
||||
return match[2] || match[0];
|
||||
}
|
||||
return null;
|
||||
case 'entity':
|
||||
case 'text':
|
||||
case 'any':
|
||||
default:
|
||||
// Return first non-common word as potential value
|
||||
const commonWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for']);
|
||||
const significantToken = tokens.find(t => !commonWords.has(t.toLowerCase()));
|
||||
return significantToken || null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Get default regex pattern for slot type
|
||||
*/
|
||||
getDefaultPatternForType(type) {
|
||||
switch (type) {
|
||||
case 'number':
|
||||
return '\\d+(?:\\.\\d+)?';
|
||||
case 'date':
|
||||
return '[\\w\\s,/-]+';
|
||||
case 'person':
|
||||
return '[A-Z][a-z]+(?:\\s+[A-Z][a-z]+)*';
|
||||
case 'location':
|
||||
return '[A-Z][a-z]+(?:[\\s,]+[A-Z][a-z]+)*';
|
||||
case 'entity':
|
||||
return '[\\w\\s-]+';
|
||||
case 'text':
|
||||
case 'any':
|
||||
default:
|
||||
return '.+';
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Transform value based on type
|
||||
*/
|
||||
transformByType(value, type) {
|
||||
switch (type) {
|
||||
case 'number':
|
||||
const num = parseFloat(value);
|
||||
return isNaN(num) ? value : num;
|
||||
case 'date':
|
||||
// Simple date parsing
|
||||
if (value.toLowerCase() === 'today') {
|
||||
return new Date().toISOString().split('T')[0];
|
||||
}
|
||||
else if (value.toLowerCase() === 'tomorrow') {
|
||||
const tomorrow = new Date();
|
||||
tomorrow.setDate(tomorrow.getDate() + 1);
|
||||
return tomorrow.toISOString().split('T')[0];
|
||||
}
|
||||
else if (value.toLowerCase() === 'yesterday') {
|
||||
const yesterday = new Date();
|
||||
yesterday.setDate(yesterday.getDate() - 1);
|
||||
return yesterday.toISOString().split('T')[0];
|
||||
}
|
||||
return value;
|
||||
case 'person':
|
||||
case 'location':
|
||||
case 'entity':
|
||||
// Capitalize properly
|
||||
return value.split(' ')
|
||||
.map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
|
||||
.join(' ');
|
||||
default:
|
||||
return value.trim();
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Validate slot value against definition
|
||||
*/
|
||||
validateSlotValue(value, slotDef) {
|
||||
if (value === null || value === undefined) {
|
||||
return !slotDef.required;
|
||||
}
|
||||
switch (slotDef.type) {
|
||||
case 'number':
|
||||
return typeof value === 'number' && !isNaN(value);
|
||||
case 'date':
|
||||
return typeof value === 'string' && value.length > 0;
|
||||
case 'text':
|
||||
case 'person':
|
||||
case 'location':
|
||||
case 'entity':
|
||||
return typeof value === 'string' && value.length > 0;
|
||||
case 'any':
|
||||
return true;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Calculate Levenshtein distance between two strings
|
||||
*/
|
||||
levenshteinDistance(s1, s2) {
|
||||
const len1 = s1.length;
|
||||
const len2 = s2.length;
|
||||
const matrix = [];
|
||||
for (let i = 0; i <= len1; i++) {
|
||||
matrix[i] = [i];
|
||||
}
|
||||
for (let j = 0; j <= len2; j++) {
|
||||
matrix[0][j] = j;
|
||||
}
|
||||
for (let i = 1; i <= len1; i++) {
|
||||
for (let j = 1; j <= len2; j++) {
|
||||
const cost = s1[i - 1] === s2[j - 1] ? 0 : 1;
|
||||
matrix[i][j] = Math.min(matrix[i - 1][j] + 1, // deletion
|
||||
matrix[i][j - 1] + 1, // insertion
|
||||
matrix[i - 1][j - 1] + cost // substitution
|
||||
);
|
||||
}
|
||||
}
|
||||
return matrix[len1][len2];
|
||||
}
|
||||
/**
|
||||
* Align two strings for slot extraction
|
||||
*/
|
||||
alignStrings(query, example) {
|
||||
const queryTokens = this.tokenize(query);
|
||||
const exampleTokens = this.tokenize(example);
|
||||
const aligned = [];
|
||||
let i = 0, j = 0;
|
||||
while (i < queryTokens.length && j < exampleTokens.length) {
|
||||
if (queryTokens[i] === exampleTokens[j]) {
|
||||
aligned.push([queryTokens[i], exampleTokens[j]]);
|
||||
i++;
|
||||
j++;
|
||||
}
|
||||
else {
|
||||
// Try to find best match
|
||||
const bestMatch = this.findBestTokenMatch(queryTokens[i], exampleTokens.slice(j, j + 3));
|
||||
if (bestMatch.index >= 0) {
|
||||
j += bestMatch.index;
|
||||
aligned.push([queryTokens[i], exampleTokens[j]]);
|
||||
}
|
||||
else {
|
||||
aligned.push([queryTokens[i], exampleTokens[j]]);
|
||||
}
|
||||
i++;
|
||||
j++;
|
||||
}
|
||||
}
|
||||
return aligned;
|
||||
}
|
||||
/**
|
||||
* Find best token match using fuzzy comparison
|
||||
*/
|
||||
findBestTokenMatch(token, candidates) {
|
||||
let bestIndex = -1;
|
||||
let bestSimilarity = 0;
|
||||
candidates.forEach((candidate, index) => {
|
||||
const distance = this.levenshteinDistance(token.toLowerCase(), candidate.toLowerCase());
|
||||
const similarity = 1 - (distance / Math.max(token.length, candidate.length));
|
||||
if (similarity > bestSimilarity && similarity > 0.6) {
|
||||
bestIndex = index;
|
||||
bestSimilarity = similarity;
|
||||
}
|
||||
});
|
||||
return { index: bestIndex, similarity: bestSimilarity };
|
||||
}
|
||||
/**
|
||||
* Extract slots from string alignment
|
||||
*/
|
||||
extractSlotsFromAlignment(aligned, _pattern) {
|
||||
const slots = {};
|
||||
let slotIndex = 1;
|
||||
aligned.forEach(([queryToken, exampleToken]) => {
|
||||
if (exampleToken.startsWith('$')) {
|
||||
slots[`$${slotIndex}`] = queryToken;
|
||||
slotIndex++;
|
||||
}
|
||||
});
|
||||
return slots;
|
||||
}
|
||||
/**
|
||||
* Fill template with extracted slots
|
||||
*/
|
||||
fillTemplate(template, slots) {
|
||||
const filled = JSON.parse(JSON.stringify(template));
|
||||
// Recursively replace slot placeholders
|
||||
const replacePlaceholders = (obj) => {
|
||||
if (typeof obj === 'string') {
|
||||
// Replace ${1}, ${2}, etc. with slot values
|
||||
return obj.replace(/\$\{(\d+)\}/g, (_, num) => {
|
||||
return slots[`$${num}`] || '';
|
||||
});
|
||||
}
|
||||
else if (Array.isArray(obj)) {
|
||||
return obj.map(item => replacePlaceholders(item));
|
||||
}
|
||||
else if (typeof obj === 'object' && obj !== null) {
|
||||
const result = {};
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
const newKey = replacePlaceholders(key);
|
||||
result[newKey] = replacePlaceholders(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return obj;
|
||||
};
|
||||
return replacePlaceholders(filled);
|
||||
}
|
||||
/**
|
||||
* Update pattern success metrics based on usage
|
||||
*/
|
||||
updateSuccessMetric(patternId, success) {
|
||||
const current = this.successMetrics.get(patternId) || 0.5;
|
||||
// Exponential moving average
|
||||
const alpha = 0.1;
|
||||
const newMetric = success
|
||||
? current + alpha * (1 - current)
|
||||
: current - alpha * current;
|
||||
this.successMetrics.set(patternId, newMetric);
|
||||
}
|
||||
/**
|
||||
* Learn new pattern from successful query
|
||||
*/
|
||||
async learnPattern(query, result) {
|
||||
// Find similar existing patterns
|
||||
const queryEmbedding = await this.getEmbedding(query);
|
||||
const similar = await this.findBestPatterns(queryEmbedding, 1);
|
||||
if (similar[0]?.similarity < 0.7) {
|
||||
// This is a new pattern type - add it
|
||||
const newPattern = {
|
||||
id: `learned_${Date.now()}`,
|
||||
category: 'learned',
|
||||
examples: [query],
|
||||
pattern: this.generateRegexFromQuery(query),
|
||||
template: result,
|
||||
confidence: 0.6 // Start with moderate confidence
|
||||
};
|
||||
this.patterns.set(newPattern.id, newPattern);
|
||||
this.patternEmbeddings.set(newPattern.id, queryEmbedding);
|
||||
this.successMetrics.set(newPattern.id, 0.6);
|
||||
}
|
||||
else {
|
||||
// Similar pattern exists - add as example
|
||||
const pattern = similar[0].pattern;
|
||||
if (!pattern.examples.includes(query)) {
|
||||
pattern.examples.push(query);
|
||||
// Update pattern embedding with new example
|
||||
const embeddings = await Promise.all(pattern.examples.map(ex => this.getEmbedding(ex)));
|
||||
const newEmbedding = this.averageVectors(embeddings);
|
||||
this.patternEmbeddings.set(pattern.id, newEmbedding);
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Helper: Average multiple vectors
|
||||
*/
|
||||
averageVectors(vectors) {
|
||||
if (vectors.length === 0)
|
||||
return [];
|
||||
const dim = vectors[0].length;
|
||||
const avg = new Array(dim).fill(0);
|
||||
for (const vec of vectors) {
|
||||
for (let i = 0; i < dim; i++) {
|
||||
avg[i] += vec[i];
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < dim; i++) {
|
||||
avg[i] /= vectors.length;
|
||||
}
|
||||
return avg;
|
||||
}
|
||||
/**
|
||||
* Helper: Calculate cosine similarity
|
||||
*/
|
||||
cosineSimilarity(a, b) {
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
normA = Math.sqrt(normA);
|
||||
normB = Math.sqrt(normB);
|
||||
if (normA === 0 || normB === 0)
|
||||
return 0;
|
||||
return dotProduct / (normA * normB);
|
||||
}
|
||||
/**
|
||||
* Helper: Simple tokenization
|
||||
*/
|
||||
tokenize(text) {
|
||||
return text.toLowerCase().split(/\s+/).filter(t => t.length > 0);
|
||||
}
|
||||
/**
|
||||
* Helper: Post-process extracted slots
|
||||
*/
|
||||
postProcessSlots(slots, _pattern) {
|
||||
// Convert string numbers to actual numbers
|
||||
for (const [key, value] of Object.entries(slots)) {
|
||||
if (typeof value === 'string') {
|
||||
// Check if it's a number
|
||||
const num = parseFloat(value);
|
||||
if (!isNaN(num) && value.match(/^\d+(\.\d+)?$/)) {
|
||||
slots[key] = num;
|
||||
}
|
||||
// Parse dates
|
||||
if (value.match(/\d{4}/) || value.match(/(january|february|march|april|may|june|july|august|september|october|november|december)/i)) {
|
||||
// Simple year extraction
|
||||
const year = value.match(/\d{4}/);
|
||||
if (year) {
|
||||
slots[key] = parseInt(year[0]);
|
||||
}
|
||||
}
|
||||
// Clean up captured values
|
||||
slots[key] = value.trim();
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Helper: Generate regex pattern from query
|
||||
*/
|
||||
generateRegexFromQuery(query) {
|
||||
// Simple pattern generation - replace variable parts with capture groups
|
||||
let pattern = query.toLowerCase();
|
||||
// Replace numbers with \d+ capture
|
||||
pattern = pattern.replace(/\d+/g, '(\\d+)');
|
||||
// Replace quoted strings with .+ capture
|
||||
pattern = pattern.replace(/"[^"]+"/g, '(.+)');
|
||||
// Replace proper nouns (capitalized words) with capture
|
||||
pattern = pattern.replace(/\b[A-Z]\w+\b/g, '([A-Z][\\w]+)');
|
||||
return pattern;
|
||||
}
|
||||
/**
|
||||
* Get pattern statistics for monitoring
|
||||
*/
|
||||
getStatistics() {
|
||||
const stats = {
|
||||
totalPatterns: this.patterns.size,
|
||||
categories: {},
|
||||
averageConfidence: 0,
|
||||
topPatterns: []
|
||||
};
|
||||
// Count by category
|
||||
for (const pattern of this.patterns.values()) {
|
||||
stats.categories[pattern.category] = (stats.categories[pattern.category] || 0) + 1;
|
||||
}
|
||||
// Calculate average confidence
|
||||
let totalConfidence = 0;
|
||||
for (const confidence of this.successMetrics.values()) {
|
||||
totalConfidence += confidence;
|
||||
}
|
||||
stats.averageConfidence = totalConfidence / this.successMetrics.size;
|
||||
// Get top patterns by success
|
||||
const sortedPatterns = Array.from(this.successMetrics.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, 10);
|
||||
stats.topPatterns = sortedPatterns.map(([id, success]) => ({ id, success }));
|
||||
return stats;
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=patternLibrary.js.map
|
||||
File diff suppressed because one or more lines are too long
27
.recovery-workspace/dist-backup-20250910-141917/neural/patterns.d.ts
vendored
Normal file
27
.recovery-workspace/dist-backup-20250910-141917/neural/patterns.d.ts
vendored
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
/**
|
||||
* Core Pattern Library with Pre-computed Embeddings
|
||||
*
|
||||
* This file is auto-generated by scripts/buildPatterns.ts
|
||||
* DO NOT EDIT MANUALLY - edit src/patterns/comprehensive-library.json instead
|
||||
*
|
||||
* Storage strategy:
|
||||
* - Patterns are bundled directly into Brainy for zero-latency access
|
||||
* - Embeddings are pre-computed and stored as binary Float32Array
|
||||
* - Total size: ~140KB (negligible for a neural library)
|
||||
* - No external files needed, works in all environments
|
||||
*/
|
||||
import type { Pattern } from './patternLibrary.js';
|
||||
export declare const CORE_PATTERNS: Pattern[];
|
||||
export declare const PATTERN_EMBEDDINGS_BINARY: Uint8Array | null;
|
||||
export declare function getPatternEmbeddings(): Map<string, Float32Array>;
|
||||
export declare const PATTERNS_VERSION = "2.0.0";
|
||||
export declare const PATTERNS_METADATA: {
|
||||
totalPatterns: number;
|
||||
categories: string[];
|
||||
embeddingDimensions: number;
|
||||
storageSize: {
|
||||
patterns: string;
|
||||
embeddings: string;
|
||||
total: string;
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/**
|
||||
* Core Pattern Library with Pre-computed Embeddings
|
||||
*
|
||||
* This file is auto-generated by scripts/buildPatterns.ts
|
||||
* DO NOT EDIT MANUALLY - edit src/patterns/comprehensive-library.json instead
|
||||
*
|
||||
* Storage strategy:
|
||||
* - Patterns are bundled directly into Brainy for zero-latency access
|
||||
* - Embeddings are pre-computed and stored as binary Float32Array
|
||||
* - Total size: ~140KB (negligible for a neural library)
|
||||
* - No external files needed, works in all environments
|
||||
*/
|
||||
// Pattern data embedded directly for reliability
|
||||
export const CORE_PATTERNS = [
|
||||
// Informational queries
|
||||
{
|
||||
id: "info_what_is",
|
||||
category: "informational",
|
||||
examples: ["what is artificial intelligence", "what is machine learning"],
|
||||
pattern: "what is (.+)",
|
||||
template: { like: "${1}" },
|
||||
confidence: 0.9
|
||||
},
|
||||
{
|
||||
id: "info_how_does",
|
||||
category: "informational",
|
||||
examples: ["how does neural network work", "how does deep learning work"],
|
||||
pattern: "how does (.+) work",
|
||||
template: { like: "${1}" },
|
||||
confidence: 0.85
|
||||
},
|
||||
// ... more patterns loaded from library.json at build time
|
||||
];
|
||||
// Pre-computed embeddings as binary data
|
||||
// Generated by scripts/buildPatterns.ts using Brainy's embedding model
|
||||
export const PATTERN_EMBEDDINGS_BINARY = null; // Will be populated at build
|
||||
// Helper to decode embeddings
|
||||
export function getPatternEmbeddings() {
|
||||
if (!PATTERN_EMBEDDINGS_BINARY) {
|
||||
return new Map(); // Will compute at runtime if not pre-built
|
||||
}
|
||||
const embeddings = new Map();
|
||||
const view = new DataView(PATTERN_EMBEDDINGS_BINARY.buffer);
|
||||
const embeddingSize = 384; // Standard size
|
||||
CORE_PATTERNS.forEach((pattern, index) => {
|
||||
const offset = index * embeddingSize * 4; // 4 bytes per float
|
||||
const embedding = new Float32Array(embeddingSize);
|
||||
for (let i = 0; i < embeddingSize; i++) {
|
||||
embedding[i] = view.getFloat32(offset + i * 4, true);
|
||||
}
|
||||
embeddings.set(pattern.id, embedding);
|
||||
});
|
||||
return embeddings;
|
||||
}
|
||||
// Version for cache invalidation
|
||||
export const PATTERNS_VERSION = "2.0.0";
|
||||
// Export metadata for monitoring
|
||||
export const PATTERNS_METADATA = {
|
||||
totalPatterns: CORE_PATTERNS.length,
|
||||
categories: [...new Set(CORE_PATTERNS.map(p => p.category))],
|
||||
embeddingDimensions: 384,
|
||||
storageSize: {
|
||||
patterns: "24KB",
|
||||
embeddings: "98KB",
|
||||
total: "122KB"
|
||||
}
|
||||
};
|
||||
//# sourceMappingURL=patterns.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"patterns.js","sourceRoot":"","sources":["../../src/neural/patterns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,iDAAiD;AACjD,MAAM,CAAC,MAAM,aAAa,GAAc;IACtC,wBAAwB;IACxB;QACE,EAAE,EAAE,cAAc;QAClB,QAAQ,EAAE,eAAe;QACzB,QAAQ,EAAE,CAAC,iCAAiC,EAAE,0BAA0B,CAAC;QACzE,OAAO,EAAE,cAAc;QACvB,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QAC1B,UAAU,EAAE,GAAG;KAChB;IACD;QACE,EAAE,EAAE,eAAe;QACnB,QAAQ,EAAE,eAAe;QACzB,QAAQ,EAAE,CAAC,8BAA8B,EAAE,6BAA6B,CAAC;QACzE,OAAO,EAAE,oBAAoB;QAC7B,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE;QAC1B,UAAU,EAAE,IAAI;KACjB;IACD,2DAA2D;CAC5D,CAAA;AAED,yCAAyC;AACzC,uEAAuE;AACvE,MAAM,CAAC,MAAM,yBAAyB,GAAsB,IAAI,CAAA,CAAC,6BAA6B;AAE9F,8BAA8B;AAC9B,MAAM,UAAU,oBAAoB;IAClC,IAAI,CAAC,yBAAyB,EAAE,CAAC;QAC/B,OAAO,IAAI,GAAG,EAAE,CAAA,CAAC,2CAA2C;IAC9D,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAwB,CAAA;IAClD,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,yBAAyB,CAAC,MAAM,CAAC,CAAA;IAC3D,MAAM,aAAa,GAAG,GAAG,CAAA,CAAC,gBAAgB;IAE1C,aAAa,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QACvC,MAAM,MAAM,GAAG,KAAK,GAAG,aAAa,GAAG,CAAC,CAAA,CAAC,oBAAoB;QAC7D,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,aAAa,CAAC,CAAA;QAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;QACtD,CAAC;QAED,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;IACvC,CAAC,CAAC,CAAA;IAEF,OAAO,UAAU,CAAA;AACnB,CAAC;AAED,iCAAiC;AACjC,MAAM,CAAC,MAAM,gBAAgB,GAAG,OAAO,CAAA;AAEvC,iCAAiC;AACjC,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,aAAa,EAAE,aAAa,CAAC,MAAM;IACnC,UAAU,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5D,mBAAmB,EAAE,GAAG;IACxB,WAAW,EAAE;QACX,QAAQ,EAAE,MAAM;QAChB,UAAU,EAAE,MAAM;QAClB,KAAK,EAAE,OAAO;KACf;CACF,CAAA"}
|
||||
35
.recovery-workspace/dist-backup-20250910-141917/neural/staticPatternMatcher.d.ts
vendored
Normal file
35
.recovery-workspace/dist-backup-20250910-141917/neural/staticPatternMatcher.d.ts
vendored
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/**
|
||||
* Static Pattern Matcher - NO runtime initialization, NO Brainy needed
|
||||
*
|
||||
* All patterns and embeddings are pre-computed at build time
|
||||
* This is pure pattern matching with zero dependencies
|
||||
*/
|
||||
import { EMBEDDED_PATTERNS } from './embeddedPatterns.js';
|
||||
import type { Vector } from '../coreTypes.js';
|
||||
import type { TripleQuery } from '../triple/TripleIntelligence.js';
|
||||
/**
|
||||
* Match query against all patterns using embeddings
|
||||
*/
|
||||
export declare function findBestPatterns(queryEmbedding: Vector, k?: number): Array<{
|
||||
pattern: typeof EMBEDDED_PATTERNS[0];
|
||||
similarity: number;
|
||||
}>;
|
||||
/**
|
||||
* Match query against patterns using regex
|
||||
*/
|
||||
export declare function matchPatternByRegex(query: string): {
|
||||
pattern: typeof EMBEDDED_PATTERNS[0];
|
||||
slots: Record<string, string>;
|
||||
query: TripleQuery;
|
||||
} | null;
|
||||
/**
|
||||
* Convert natural language to structured query using STATIC patterns
|
||||
* NO initialization needed, NO Brainy required
|
||||
*/
|
||||
export declare function patternMatchQuery(query: string, queryEmbedding?: Vector): TripleQuery;
|
||||
export declare const PATTERN_STATS: {
|
||||
totalPatterns: number;
|
||||
categories: string[];
|
||||
domains: string[];
|
||||
hasEmbeddings: boolean;
|
||||
};
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
/**
|
||||
* Static Pattern Matcher - NO runtime initialization, NO Brainy needed
|
||||
*
|
||||
* All patterns and embeddings are pre-computed at build time
|
||||
* This is pure pattern matching with zero dependencies
|
||||
*/
|
||||
import { EMBEDDED_PATTERNS, getPatternEmbeddings } from './embeddedPatterns.js';
|
||||
// Pre-load patterns and embeddings at module load time (happens once)
|
||||
const patterns = new Map(EMBEDDED_PATTERNS.map(p => [p.id, p]));
|
||||
const patternEmbeddings = getPatternEmbeddings();
|
||||
/**
|
||||
* Cosine similarity between two vectors
|
||||
*/
|
||||
function cosineSimilarity(a, b) {
|
||||
if (!a || !b || a.length !== b.length)
|
||||
return 0;
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
const denominator = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
return denominator === 0 ? 0 : dotProduct / denominator;
|
||||
}
|
||||
/**
|
||||
* Extract slots from matched pattern
|
||||
*/
|
||||
function extractSlots(query, pattern) {
|
||||
try {
|
||||
const regex = new RegExp(pattern, 'i');
|
||||
const match = query.match(regex);
|
||||
if (!match)
|
||||
return null;
|
||||
const slots = {};
|
||||
for (let i = 1; i < match.length; i++) {
|
||||
if (match[i]) {
|
||||
slots[`$${i}`] = match[i];
|
||||
}
|
||||
}
|
||||
return Object.keys(slots).length > 0 ? slots : null;
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Apply template with extracted slots
|
||||
*/
|
||||
function applyTemplate(template, slots) {
|
||||
if (!template || !slots)
|
||||
return template;
|
||||
const result = JSON.parse(JSON.stringify(template));
|
||||
const applySlots = (obj) => {
|
||||
if (typeof obj === 'string') {
|
||||
return obj.replace(/\$\{(\d+)\}/g, (_, num) => slots[`$${num}`] || '');
|
||||
}
|
||||
if (Array.isArray(obj)) {
|
||||
return obj.map(applySlots);
|
||||
}
|
||||
if (typeof obj === 'object' && obj !== null) {
|
||||
const newObj = {};
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
newObj[key] = applySlots(value);
|
||||
}
|
||||
return newObj;
|
||||
}
|
||||
return obj;
|
||||
};
|
||||
return applySlots(result);
|
||||
}
|
||||
/**
|
||||
* Match query against all patterns using embeddings
|
||||
*/
|
||||
export function findBestPatterns(queryEmbedding, k = 3) {
|
||||
const matches = [];
|
||||
for (const pattern of EMBEDDED_PATTERNS) {
|
||||
const patternEmbedding = patternEmbeddings.get(pattern.id);
|
||||
if (!patternEmbedding)
|
||||
continue;
|
||||
// Pass Float32Array directly, no need for Array.from()!
|
||||
const similarity = cosineSimilarity(queryEmbedding, patternEmbedding);
|
||||
if (similarity > 0.5) { // Threshold for relevance
|
||||
matches.push({ pattern, similarity });
|
||||
}
|
||||
}
|
||||
// Sort by similarity and return top k
|
||||
return matches
|
||||
.sort((a, b) => b.similarity - a.similarity)
|
||||
.slice(0, k);
|
||||
}
|
||||
/**
|
||||
* Match query against patterns using regex
|
||||
*/
|
||||
export function matchPatternByRegex(query) {
|
||||
// Try direct regex matching first (fastest)
|
||||
for (const pattern of EMBEDDED_PATTERNS) {
|
||||
const slots = extractSlots(query, pattern.pattern);
|
||||
if (slots) {
|
||||
const templatedQuery = applyTemplate(pattern.template, slots);
|
||||
return {
|
||||
pattern,
|
||||
slots,
|
||||
query: templatedQuery
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* Convert natural language to structured query using STATIC patterns
|
||||
* NO initialization needed, NO Brainy required
|
||||
*/
|
||||
export function patternMatchQuery(query, queryEmbedding) {
|
||||
// ALWAYS use vector similarity when we have embeddings (which we always do!)
|
||||
if (queryEmbedding && queryEmbedding.length === 384) {
|
||||
const bestPatterns = findBestPatterns(queryEmbedding, 5); // Get top 5 matches
|
||||
// Try to extract slots from best matching patterns
|
||||
for (const { pattern, similarity } of bestPatterns) {
|
||||
// Only try patterns with good similarity
|
||||
if (similarity < 0.7)
|
||||
break;
|
||||
const slots = extractSlots(query, pattern.pattern);
|
||||
if (slots) {
|
||||
// Found a good match with extractable slots!
|
||||
const result = applyTemplate(pattern.template, slots);
|
||||
console.log('[NLP] Applied template with slots:', JSON.stringify(result));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
// If no slots extracted but we have a good match, use the template as-is
|
||||
if (bestPatterns.length > 0 && bestPatterns[0].similarity > 0.75) {
|
||||
console.log('[NLP] Returning template as-is:', JSON.stringify(bestPatterns[0].pattern.template));
|
||||
return bestPatterns[0].pattern.template;
|
||||
}
|
||||
}
|
||||
// Fallback: simple vector search (should rarely happen)
|
||||
console.log('[NLP] Fallback - returning simple query');
|
||||
return {
|
||||
like: query,
|
||||
limit: 10
|
||||
};
|
||||
}
|
||||
// Export pattern statistics for monitoring
|
||||
export const PATTERN_STATS = {
|
||||
totalPatterns: EMBEDDED_PATTERNS.length,
|
||||
categories: [...new Set(EMBEDDED_PATTERNS.map(p => p.category))],
|
||||
domains: [...new Set(EMBEDDED_PATTERNS.filter(p => p.domain).map(p => p.domain))],
|
||||
hasEmbeddings: patternEmbeddings.size > 0
|
||||
};
|
||||
//# sourceMappingURL=staticPatternMatcher.js.map
|
||||
File diff suppressed because one or more lines are too long
287
.recovery-workspace/dist-backup-20250910-141917/neural/types.d.ts
vendored
Normal file
287
.recovery-workspace/dist-backup-20250910-141917/neural/types.d.ts
vendored
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
/**
|
||||
* Neural API Type Definitions
|
||||
* Comprehensive interfaces for clustering, similarity, and analysis
|
||||
*/
|
||||
export interface Vector {
|
||||
[index: number]: number;
|
||||
length: number;
|
||||
}
|
||||
export interface SemanticCluster {
|
||||
id: string;
|
||||
centroid: Vector;
|
||||
members: string[];
|
||||
size: number;
|
||||
confidence: number;
|
||||
label?: string;
|
||||
metadata?: Record<string, any>;
|
||||
cohesion?: number;
|
||||
level?: number;
|
||||
}
|
||||
export interface ClusterEdge {
|
||||
id: string;
|
||||
source: string;
|
||||
target: string;
|
||||
type: string;
|
||||
weight?: number;
|
||||
isInterCluster: boolean;
|
||||
sourceCluster?: string;
|
||||
targetCluster?: string;
|
||||
}
|
||||
export interface EnhancedSemanticCluster extends SemanticCluster {
|
||||
intraClusterEdges: ClusterEdge[];
|
||||
interClusterEdges: ClusterEdge[];
|
||||
relationshipSummary: {
|
||||
totalEdges: number;
|
||||
intraClusterEdges: number;
|
||||
interClusterEdges: number;
|
||||
edgeTypes: Record<string, number>;
|
||||
};
|
||||
}
|
||||
export interface DomainCluster extends SemanticCluster {
|
||||
domain: string;
|
||||
domainConfidence: number;
|
||||
crossDomainMembers?: string[];
|
||||
}
|
||||
export interface TemporalCluster extends SemanticCluster {
|
||||
timeWindow: TimeWindow;
|
||||
trend?: 'increasing' | 'decreasing' | 'stable';
|
||||
temporal: {
|
||||
startTime: Date;
|
||||
endTime: Date;
|
||||
peakTime?: Date;
|
||||
frequency?: number;
|
||||
};
|
||||
}
|
||||
export interface ExplainableCluster extends SemanticCluster {
|
||||
explanation: {
|
||||
primaryFeatures: string[];
|
||||
commonTerms: string[];
|
||||
reasoning: string;
|
||||
confidence: number;
|
||||
};
|
||||
subClusters?: ExplainableCluster[];
|
||||
}
|
||||
export interface ConfidentCluster extends SemanticCluster {
|
||||
minConfidence: number;
|
||||
uncertainMembers: string[];
|
||||
certainMembers: string[];
|
||||
}
|
||||
export interface BaseClusteringOptions {
|
||||
maxClusters?: number;
|
||||
minClusterSize?: number;
|
||||
threshold?: number;
|
||||
cacheResults?: boolean;
|
||||
}
|
||||
export interface ClusteringOptions extends BaseClusteringOptions {
|
||||
algorithm?: 'auto' | 'hierarchical' | 'kmeans' | 'dbscan' | 'sample' | 'semantic' | 'graph' | 'multimodal';
|
||||
sampleSize?: number;
|
||||
strategy?: 'random' | 'diverse' | 'recent' | 'important';
|
||||
memoryLimit?: string;
|
||||
includeOutliers?: boolean;
|
||||
maxIterations?: number;
|
||||
tolerance?: number;
|
||||
}
|
||||
export interface DomainClusteringOptions extends BaseClusteringOptions {
|
||||
domainField?: string;
|
||||
crossDomainThreshold?: number;
|
||||
preserveDomainBoundaries?: boolean;
|
||||
}
|
||||
export interface TemporalClusteringOptions extends BaseClusteringOptions {
|
||||
timeField: string;
|
||||
windows: TimeWindow[];
|
||||
overlapStrategy?: 'merge' | 'separate' | 'hierarchical';
|
||||
trendAnalysis?: boolean;
|
||||
}
|
||||
export interface StreamClusteringOptions extends BaseClusteringOptions {
|
||||
batchSize?: number;
|
||||
updateInterval?: number;
|
||||
adaptiveThreshold?: boolean;
|
||||
decayFactor?: number;
|
||||
}
|
||||
export interface SimilarityOptions {
|
||||
detailed?: boolean;
|
||||
metric?: 'cosine' | 'euclidean' | 'manhattan' | 'jaccard';
|
||||
normalized?: boolean;
|
||||
}
|
||||
export interface SimilarityResult {
|
||||
score: number;
|
||||
confidence: number;
|
||||
explanation?: string;
|
||||
metric?: string;
|
||||
}
|
||||
export interface NeighborOptions {
|
||||
limit?: number;
|
||||
radius?: number;
|
||||
minSimilarity?: number;
|
||||
includeMetadata?: boolean;
|
||||
sortBy?: 'similarity' | 'importance' | 'recency';
|
||||
}
|
||||
export interface Neighbor {
|
||||
id: string;
|
||||
similarity: number;
|
||||
data?: any;
|
||||
metadata?: Record<string, any>;
|
||||
distance?: number;
|
||||
}
|
||||
export interface NeighborsResult {
|
||||
neighbors: Neighbor[];
|
||||
queryId: string;
|
||||
totalFound: number;
|
||||
averageSimilarity: number;
|
||||
}
|
||||
export interface SemanticHierarchy {
|
||||
self: {
|
||||
id: string;
|
||||
vector?: Vector;
|
||||
metadata?: any;
|
||||
};
|
||||
parent?: {
|
||||
id: string;
|
||||
similarity: number;
|
||||
};
|
||||
children?: Array<{
|
||||
id: string;
|
||||
similarity: number;
|
||||
}>;
|
||||
siblings?: Array<{
|
||||
id: string;
|
||||
similarity: number;
|
||||
}>;
|
||||
level?: number;
|
||||
depth?: number;
|
||||
}
|
||||
export interface HierarchyOptions {
|
||||
maxDepth?: number;
|
||||
minSimilarity?: number;
|
||||
includeMetadata?: boolean;
|
||||
buildStrategy?: 'similarity' | 'metadata' | 'mixed';
|
||||
}
|
||||
export interface VisualizationOptions {
|
||||
maxNodes?: number;
|
||||
dimensions?: 2 | 3;
|
||||
algorithm?: 'force' | 'spring' | 'circular' | 'hierarchical';
|
||||
includeEdges?: boolean;
|
||||
clusterColors?: boolean;
|
||||
nodeSize?: 'uniform' | 'importance' | 'connections';
|
||||
}
|
||||
export interface VisualizationNode {
|
||||
id: string;
|
||||
x: number;
|
||||
y: number;
|
||||
z?: number;
|
||||
cluster?: string;
|
||||
size?: number;
|
||||
color?: string;
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
export interface VisualizationEdge {
|
||||
source: string;
|
||||
target: string;
|
||||
weight: number;
|
||||
color?: string;
|
||||
type?: string;
|
||||
}
|
||||
export interface VisualizationResult {
|
||||
nodes: VisualizationNode[];
|
||||
edges: VisualizationEdge[];
|
||||
clusters?: Array<{
|
||||
id: string;
|
||||
color: string;
|
||||
size: number;
|
||||
label?: string;
|
||||
}>;
|
||||
metadata: {
|
||||
algorithm: string;
|
||||
dimensions: number;
|
||||
totalNodes: number;
|
||||
totalEdges: number;
|
||||
generatedAt: Date;
|
||||
};
|
||||
}
|
||||
export interface TimeWindow {
|
||||
start: Date;
|
||||
end: Date;
|
||||
label?: string;
|
||||
weight?: number;
|
||||
}
|
||||
export interface ClusterFeedback {
|
||||
clusterId: string;
|
||||
action: 'merge' | 'split' | 'relabel' | 'adjust';
|
||||
parameters?: Record<string, any>;
|
||||
confidence?: number;
|
||||
}
|
||||
export interface OutlierOptions {
|
||||
threshold?: number;
|
||||
method?: 'isolation' | 'statistical' | 'cluster-based';
|
||||
minNeighbors?: number;
|
||||
includeReasons?: boolean;
|
||||
}
|
||||
export interface Outlier {
|
||||
id: string;
|
||||
score: number;
|
||||
reasons?: string[];
|
||||
nearestNeighbors?: Neighbor[];
|
||||
metadata?: Record<string, any>;
|
||||
}
|
||||
export interface PerformanceMetrics {
|
||||
executionTime: number;
|
||||
memoryUsed: number;
|
||||
itemsProcessed: number;
|
||||
cacheHits: number;
|
||||
cacheMisses: number;
|
||||
algorithm: string;
|
||||
}
|
||||
export interface ClusteringResult<T = SemanticCluster> {
|
||||
clusters: T[];
|
||||
metrics: PerformanceMetrics;
|
||||
metadata: {
|
||||
totalItems: number;
|
||||
clustersFound: number;
|
||||
averageClusterSize: number;
|
||||
silhouetteScore?: number;
|
||||
timestamp: Date;
|
||||
semanticTypes?: number;
|
||||
hnswLevel?: number;
|
||||
kValue?: number;
|
||||
hasConverged?: boolean;
|
||||
outlierCount?: number;
|
||||
eps?: number;
|
||||
minPts?: number;
|
||||
averageModularity?: number;
|
||||
fusionMethod?: string;
|
||||
componentAlgorithms?: string[];
|
||||
sampleSize?: number;
|
||||
samplingStrategy?: string;
|
||||
};
|
||||
}
|
||||
export interface StreamingBatch<T = SemanticCluster> {
|
||||
clusters: T[];
|
||||
batchNumber: number;
|
||||
isComplete: boolean;
|
||||
progress: {
|
||||
processed: number;
|
||||
total: number;
|
||||
percentage: number;
|
||||
};
|
||||
metrics: PerformanceMetrics;
|
||||
}
|
||||
export declare class NeuralAPIError extends Error {
|
||||
code: string;
|
||||
context?: Record<string, any> | undefined;
|
||||
constructor(message: string, code: string, context?: Record<string, any> | undefined);
|
||||
}
|
||||
export declare class ClusteringError extends NeuralAPIError {
|
||||
constructor(message: string, context?: Record<string, any>);
|
||||
}
|
||||
export declare class SimilarityError extends NeuralAPIError {
|
||||
constructor(message: string, context?: Record<string, any>);
|
||||
}
|
||||
export interface NeuralAPIConfig {
|
||||
cacheSize?: number;
|
||||
defaultAlgorithm?: string;
|
||||
similarityMetric?: 'cosine' | 'euclidean' | 'manhattan';
|
||||
performanceTracking?: boolean;
|
||||
maxMemoryUsage?: string;
|
||||
parallelProcessing?: boolean;
|
||||
streamingBatchSize?: number;
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
/**
|
||||
* Neural API Type Definitions
|
||||
* Comprehensive interfaces for clustering, similarity, and analysis
|
||||
*/
|
||||
// ===== ERROR TYPES =====
|
||||
export class NeuralAPIError extends Error {
|
||||
constructor(message, code, context) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
this.context = context;
|
||||
this.name = 'NeuralAPIError';
|
||||
}
|
||||
}
|
||||
export class ClusteringError extends NeuralAPIError {
|
||||
constructor(message, context) {
|
||||
super(message, 'CLUSTERING_ERROR', context);
|
||||
}
|
||||
}
|
||||
export class SimilarityError extends NeuralAPIError {
|
||||
constructor(message, context) {
|
||||
super(message, 'SIMILARITY_ERROR', context);
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=types.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/neural/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA6SH,0BAA0B;AAE1B,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YACE,OAAe,EACR,IAAY,EACZ,OAA6B;QAEpC,KAAK,CAAC,OAAO,CAAC,CAAA;QAHP,SAAI,GAAJ,IAAI,CAAQ;QACZ,YAAO,GAAP,OAAO,CAAsB;QAGpC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA;IAC9B,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,cAAc;IACjD,YAAY,OAAe,EAAE,OAA6B;QACxD,KAAK,CAAC,OAAO,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAA;IAC7C,CAAC;CACF;AAED,MAAM,OAAO,eAAgB,SAAQ,cAAc;IACjD,YAAY,OAAe,EAAE,OAA6B;QACxD,KAAK,CAAC,OAAO,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAA;IAC7C,CAAC;CACF"}
|
||||
Loading…
Add table
Add a link
Reference in a new issue