chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
161
.recovery-workspace/dist-backup-20250910-141917/triple/TripleIntelligence.d.ts
vendored
Normal file
161
.recovery-workspace/dist-backup-20250910-141917/triple/TripleIntelligence.d.ts
vendored
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
/**
|
||||
* Triple Intelligence Engine
|
||||
* Revolutionary unified search combining Vector + Graph + Field intelligence
|
||||
*
|
||||
* This is Brainy's killer feature - no other database can do this!
|
||||
*/
|
||||
import { Vector, SearchResult } from '../coreTypes.js';
|
||||
import type { Brainy } from '../brainy.js';
|
||||
export interface TripleQuery {
|
||||
like?: string | Vector | any;
|
||||
similar?: string | Vector | any;
|
||||
connected?: {
|
||||
to?: string | string[];
|
||||
from?: string | string[];
|
||||
type?: string | string[];
|
||||
depth?: number;
|
||||
maxDepth?: number;
|
||||
direction?: 'in' | 'out' | 'both';
|
||||
};
|
||||
where?: Record<string, any>;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
mode?: 'auto' | 'vector' | 'graph' | 'metadata' | 'fusion';
|
||||
boost?: 'recent' | 'popular' | 'verified' | string;
|
||||
explain?: boolean;
|
||||
threshold?: number;
|
||||
}
|
||||
export interface TripleResult extends SearchResult {
|
||||
vectorScore?: number;
|
||||
graphScore?: number;
|
||||
fieldScore?: number;
|
||||
fusionScore: number;
|
||||
explanation?: {
|
||||
plan: string;
|
||||
timing: Record<string, number>;
|
||||
boosts: string[];
|
||||
};
|
||||
}
|
||||
export interface QueryPlan {
|
||||
startWith: 'vector' | 'graph' | 'field';
|
||||
canParallelize: boolean;
|
||||
estimatedCost: number;
|
||||
steps: QueryStep[];
|
||||
}
|
||||
export interface QueryStep {
|
||||
type: 'vector' | 'graph' | 'field' | 'fusion';
|
||||
operation: string;
|
||||
estimated: number;
|
||||
}
|
||||
/**
|
||||
* The Triple Intelligence Engine
|
||||
* Unifies vector, graph, and field search into one beautiful API
|
||||
*/
|
||||
export declare class TripleIntelligenceEngine {
|
||||
private brain;
|
||||
private api;
|
||||
private planCache;
|
||||
constructor(brain: Brainy<any>);
|
||||
/**
|
||||
* The magic happens here - one query to rule them all
|
||||
*/
|
||||
find(query: TripleQuery): Promise<TripleResult[]>;
|
||||
/**
|
||||
* Generate optimal execution plan based on query shape and statistics
|
||||
*/
|
||||
private optimizeQuery;
|
||||
/**
|
||||
* Calculate real costs for each operation based on statistics
|
||||
*/
|
||||
private calculateOperationCosts;
|
||||
/**
|
||||
* Estimate selectivity of field filters
|
||||
*/
|
||||
private estimateFieldSelectivity;
|
||||
/**
|
||||
* Build optimal execution plan based on costs
|
||||
*/
|
||||
private buildOptimalPlan;
|
||||
/**
|
||||
* Build progressive execution steps
|
||||
*/
|
||||
private buildProgressiveSteps;
|
||||
/**
|
||||
* Build parallel execution steps
|
||||
*/
|
||||
private buildParallelSteps;
|
||||
/**
|
||||
* Execute searches in parallel for maximum speed
|
||||
*/
|
||||
private parallelSearch;
|
||||
/**
|
||||
* Progressive filtering for efficiency
|
||||
*/
|
||||
private progressiveSearch;
|
||||
/**
|
||||
* Vector similarity search
|
||||
*/
|
||||
private vectorSearch;
|
||||
/**
|
||||
* Graph traversal
|
||||
*/
|
||||
private graphTraversal;
|
||||
/**
|
||||
* Field-based filtering using MetadataIndex for O(log n) performance
|
||||
* NO FALLBACKS - Requires proper where clause and MetadataIndex
|
||||
*/
|
||||
private fieldFilter;
|
||||
/**
|
||||
* Execute a single signal query directly
|
||||
*/
|
||||
private executeSingleSignal;
|
||||
/**
|
||||
* Expand graph connections from existing candidates
|
||||
*/
|
||||
private graphExpand;
|
||||
/**
|
||||
* Vector search within existing candidates
|
||||
*/
|
||||
private vectorSearchWithin;
|
||||
/**
|
||||
* Apply field filter to existing candidates
|
||||
*/
|
||||
private applyFieldFilter;
|
||||
/**
|
||||
* Check if metadata matches filter conditions
|
||||
*/
|
||||
private matchesFilter;
|
||||
/**
|
||||
* Calculate cosine similarity between two vectors
|
||||
*/
|
||||
private cosineSimilarity;
|
||||
/**
|
||||
* Fusion ranking using Reciprocal Rank Fusion (RRF)
|
||||
* This is the same algorithm used by Google and Elasticsearch
|
||||
*/
|
||||
private fusionRank;
|
||||
/**
|
||||
* Calculate dynamic signal weights based on query characteristics
|
||||
*/
|
||||
private calculateSignalWeights;
|
||||
/**
|
||||
* Apply boost strategies
|
||||
*/
|
||||
private applyBoosts;
|
||||
/**
|
||||
* Add query explanations for debugging
|
||||
*/
|
||||
private addExplanations;
|
||||
/**
|
||||
* Optimize plan based on historical patterns
|
||||
*/
|
||||
/**
|
||||
* Clear query optimization cache
|
||||
*/
|
||||
clearCache(): void;
|
||||
/**
|
||||
* Get optimization statistics
|
||||
*/
|
||||
getStats(): any;
|
||||
}
|
||||
export declare function find(brain: Brainy<any>, query: TripleQuery): Promise<TripleResult[]>;
|
||||
|
|
@ -0,0 +1,814 @@
|
|||
/**
|
||||
* Triple Intelligence Engine
|
||||
* Revolutionary unified search combining Vector + Graph + Field intelligence
|
||||
*
|
||||
* This is Brainy's killer feature - no other database can do this!
|
||||
*/
|
||||
/**
|
||||
* The Triple Intelligence Engine
|
||||
* Unifies vector, graph, and field search into one beautiful API
|
||||
*/
|
||||
export class TripleIntelligenceEngine {
|
||||
constructor(brain) {
|
||||
this.planCache = new Map();
|
||||
this.brain = brain;
|
||||
this.api = brain.getTripleIntelligenceAPI();
|
||||
// Query history removed - unnecessary complexity for minimal gain
|
||||
}
|
||||
/**
|
||||
* The magic happens here - one query to rule them all
|
||||
*/
|
||||
async find(query) {
|
||||
const startTime = Date.now();
|
||||
// Generate optimal query plan
|
||||
const plan = await this.optimizeQuery(query);
|
||||
// Execute based on plan
|
||||
let results;
|
||||
if (plan.canParallelize) {
|
||||
// Run all three paths in parallel for maximum speed
|
||||
results = await this.parallelSearch(query, plan);
|
||||
}
|
||||
else {
|
||||
// Progressive filtering for efficiency
|
||||
results = await this.progressiveSearch(query, plan);
|
||||
}
|
||||
// Apply boosts if requested
|
||||
if (query.boost) {
|
||||
results = this.applyBoosts(results, query.boost);
|
||||
}
|
||||
// Add explanations if requested
|
||||
if (query.explain) {
|
||||
const timing = Date.now() - startTime;
|
||||
results = this.addExplanations(results, plan, timing);
|
||||
}
|
||||
// Query history removed - no learning needed
|
||||
// Apply limit
|
||||
if (query.limit) {
|
||||
results = results.slice(0, query.limit);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Generate optimal execution plan based on query shape and statistics
|
||||
*/
|
||||
async optimizeQuery(query) {
|
||||
// Short-circuit optimization for single-signal queries
|
||||
const hasVector = !!(query.like || query.similar);
|
||||
const hasGraph = !!(query.connected);
|
||||
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||||
const signalCount = [hasVector, hasGraph, hasField].filter(Boolean).length;
|
||||
// Single signal - skip fusion entirely!
|
||||
if (signalCount === 1) {
|
||||
const singleType = hasVector ? 'vector' : hasGraph ? 'graph' : 'field';
|
||||
return {
|
||||
startWith: singleType,
|
||||
canParallelize: false,
|
||||
estimatedCost: 1,
|
||||
steps: [{
|
||||
type: singleType,
|
||||
operation: 'direct', // Direct execution, no fusion
|
||||
estimated: 50
|
||||
}]
|
||||
};
|
||||
}
|
||||
// Check cache first
|
||||
const cacheKey = JSON.stringify(query);
|
||||
if (this.planCache.has(cacheKey)) {
|
||||
return this.planCache.get(cacheKey);
|
||||
}
|
||||
// Get real statistics for cost-based optimization
|
||||
const stats = await this.api.getStatistics();
|
||||
// Calculate costs for each operation
|
||||
const costs = await this.calculateOperationCosts(query, stats);
|
||||
// Build optimal plan based on actual costs
|
||||
const plan = this.buildOptimalPlan(query, costs, stats);
|
||||
this.planCache.set(cacheKey, plan);
|
||||
return plan;
|
||||
}
|
||||
/**
|
||||
* Calculate real costs for each operation based on statistics
|
||||
*/
|
||||
async calculateOperationCosts(query, stats) {
|
||||
const costs = {
|
||||
vector: Infinity,
|
||||
graph: Infinity,
|
||||
field: Infinity
|
||||
};
|
||||
// Vector search cost - O(log n) with HNSW
|
||||
if (query.like || query.similar) {
|
||||
// HNSW search complexity: O(log n) * ef
|
||||
const ef = 200; // exploration factor
|
||||
costs.vector = Math.log2(stats.totalCount) * ef;
|
||||
}
|
||||
// Graph traversal cost - depends on connectivity
|
||||
if (query.connected) {
|
||||
const depth = query.connected.maxDepth || query.connected.depth || 2;
|
||||
// Assume average branching factor of 10
|
||||
const branchingFactor = 10;
|
||||
costs.graph = Math.pow(branchingFactor, depth);
|
||||
}
|
||||
// Field filter cost - depends on selectivity
|
||||
if (query.where) {
|
||||
const selectivity = await this.estimateFieldSelectivity(query.where, stats);
|
||||
costs.field = stats.totalCount * selectivity;
|
||||
// If we have an index, cost is O(log n)
|
||||
if (this.api.hasMetadataIndex()) {
|
||||
costs.field = Math.log2(stats.totalCount) + costs.field;
|
||||
}
|
||||
}
|
||||
return costs;
|
||||
}
|
||||
/**
|
||||
* Estimate selectivity of field filters
|
||||
*/
|
||||
async estimateFieldSelectivity(where, stats) {
|
||||
let selectivity = 1.0;
|
||||
for (const [field, condition] of Object.entries(where)) {
|
||||
const fieldStats = stats.fieldStats[field];
|
||||
if (!fieldStats) {
|
||||
// Unknown field - assume 10% selectivity
|
||||
selectivity *= 0.1;
|
||||
continue;
|
||||
}
|
||||
if (typeof condition === 'object' && condition !== null) {
|
||||
// Handle operators
|
||||
if ('$eq' in condition) {
|
||||
// Equality - 1/cardinality
|
||||
selectivity *= 1.0 / (fieldStats.cardinality || 100);
|
||||
}
|
||||
else if ('$gt' in condition || '$gte' in condition) {
|
||||
// Range query - estimate based on distribution
|
||||
const threshold = condition.$gt || condition.$gte;
|
||||
if (typeof threshold === 'number' && fieldStats.type === 'number') {
|
||||
const range = fieldStats.max - fieldStats.min;
|
||||
const remainingRange = fieldStats.max - threshold;
|
||||
selectivity *= remainingRange / range;
|
||||
}
|
||||
else {
|
||||
selectivity *= 0.3; // Default for non-numeric
|
||||
}
|
||||
}
|
||||
else if ('$lt' in condition || '$lte' in condition) {
|
||||
// Range query - estimate based on distribution
|
||||
const threshold = condition.$lt || condition.$lte;
|
||||
if (typeof threshold === 'number' && fieldStats.type === 'number') {
|
||||
const range = fieldStats.max - fieldStats.min;
|
||||
const remainingRange = threshold - fieldStats.min;
|
||||
selectivity *= remainingRange / range;
|
||||
}
|
||||
else {
|
||||
selectivity *= 0.3; // Default for non-numeric
|
||||
}
|
||||
}
|
||||
else if ('$in' in condition) {
|
||||
// IN query
|
||||
selectivity *= condition.$in.length / (fieldStats.cardinality || 100);
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Direct equality
|
||||
selectivity *= 1.0 / (fieldStats.cardinality || 100);
|
||||
}
|
||||
}
|
||||
return Math.max(0.0001, Math.min(1.0, selectivity));
|
||||
}
|
||||
/**
|
||||
* Build optimal execution plan based on costs
|
||||
*/
|
||||
buildOptimalPlan(query, costs, stats) {
|
||||
const hasVector = !!(query.like || query.similar);
|
||||
const hasGraph = !!(query.connected);
|
||||
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||||
// Find the most selective operation
|
||||
const sortedOps = Object.entries(costs)
|
||||
.filter(([op]) => {
|
||||
return (op === 'vector' && hasVector) ||
|
||||
(op === 'graph' && hasGraph) ||
|
||||
(op === 'field' && hasField);
|
||||
})
|
||||
.sort((a, b) => a[1] - b[1]);
|
||||
// If the most selective operation filters out > 99%, start with it
|
||||
const mostSelective = sortedOps[0];
|
||||
if (mostSelective && mostSelective[1] < stats.totalCount * 0.01) {
|
||||
// Progressive plan - start with most selective
|
||||
return {
|
||||
startWith: mostSelective[0],
|
||||
canParallelize: false,
|
||||
estimatedCost: mostSelective[1],
|
||||
steps: this.buildProgressiveSteps(sortedOps, query)
|
||||
};
|
||||
}
|
||||
// If operations have similar costs, parallelize
|
||||
if (sortedOps.length > 1) {
|
||||
const ratio = sortedOps[1][1] / sortedOps[0][1];
|
||||
if (ratio < 10) {
|
||||
// Costs are within 10x - parallelize
|
||||
return {
|
||||
startWith: sortedOps[0][0],
|
||||
canParallelize: true,
|
||||
estimatedCost: Math.max(...sortedOps.map(op => op[1])),
|
||||
steps: this.buildParallelSteps(sortedOps, query)
|
||||
};
|
||||
}
|
||||
}
|
||||
// Default progressive plan
|
||||
return {
|
||||
startWith: sortedOps[0][0],
|
||||
canParallelize: false,
|
||||
estimatedCost: sortedOps.reduce((sum, op) => sum + op[1], 0),
|
||||
steps: this.buildProgressiveSteps(sortedOps, query)
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Build progressive execution steps
|
||||
*/
|
||||
buildProgressiveSteps(sortedOps, query) {
|
||||
const steps = [];
|
||||
for (const [op, cost] of sortedOps) {
|
||||
steps.push({
|
||||
type: op,
|
||||
operation: op === 'vector' ? 'search' : op === 'graph' ? 'traverse' : 'filter',
|
||||
estimated: Math.round(cost)
|
||||
});
|
||||
}
|
||||
// Add fusion step if multiple operations
|
||||
if (steps.length > 1) {
|
||||
steps.push({
|
||||
type: 'fusion',
|
||||
operation: 'rank',
|
||||
estimated: Math.round(sortedOps.length * 50)
|
||||
});
|
||||
}
|
||||
return steps;
|
||||
}
|
||||
/**
|
||||
* Build parallel execution steps
|
||||
*/
|
||||
buildParallelSteps(sortedOps, query) {
|
||||
const steps = sortedOps.map(([op, cost]) => ({
|
||||
type: op,
|
||||
operation: op === 'vector' ? 'search' : op === 'graph' ? 'traverse' : 'filter',
|
||||
estimated: Math.round(cost)
|
||||
}));
|
||||
// Always add fusion for parallel execution
|
||||
steps.push({
|
||||
type: 'fusion',
|
||||
operation: 'rank',
|
||||
estimated: Math.round(sortedOps.length * 100)
|
||||
});
|
||||
return steps;
|
||||
}
|
||||
/**
|
||||
* Execute searches in parallel for maximum speed
|
||||
*/
|
||||
async parallelSearch(query, plan) {
|
||||
// Check for single-signal optimization
|
||||
if (plan.steps.length === 1 && plan.steps[0].operation === 'direct') {
|
||||
// Skip fusion for single signal queries
|
||||
const results = await this.executeSingleSignal(query, plan.steps[0].type);
|
||||
return results.map(r => ({
|
||||
...r,
|
||||
fusionScore: r.score || 1.0,
|
||||
score: r.score || 1.0
|
||||
}));
|
||||
}
|
||||
const tasks = [];
|
||||
// Vector search
|
||||
if (query.like || query.similar) {
|
||||
tasks.push(this.vectorSearch(query.like || query.similar, query.limit));
|
||||
}
|
||||
// Graph traversal
|
||||
if (query.connected) {
|
||||
tasks.push(this.graphTraversal(query.connected));
|
||||
}
|
||||
// Field filtering
|
||||
if (query.where) {
|
||||
tasks.push(this.fieldFilter(query.where));
|
||||
}
|
||||
// Run all in parallel
|
||||
const results = await Promise.all(tasks);
|
||||
// Fusion ranking combines all signals
|
||||
return this.fusionRank(results, query);
|
||||
}
|
||||
/**
|
||||
* Progressive filtering for efficiency
|
||||
*/
|
||||
async progressiveSearch(query, plan) {
|
||||
let candidates = [];
|
||||
for (const step of plan.steps) {
|
||||
switch (step.type) {
|
||||
case 'field':
|
||||
if (candidates.length === 0) {
|
||||
// Initial field filter
|
||||
candidates = await this.fieldFilter(query.where);
|
||||
}
|
||||
else {
|
||||
// Filter existing candidates
|
||||
candidates = this.applyFieldFilter(candidates, query.where);
|
||||
}
|
||||
break;
|
||||
case 'vector':
|
||||
// CRITICAL: If we have a previous step that returned 0 candidates,
|
||||
// we must respect that and not do a fresh search
|
||||
if (candidates.length === 0 && plan.steps[0].type === 'vector') {
|
||||
// This is the first step - do initial vector search
|
||||
const results = await this.vectorSearch(query.like || query.similar, query.limit);
|
||||
candidates = results;
|
||||
}
|
||||
else if (candidates.length > 0) {
|
||||
// Vector search within existing candidates
|
||||
candidates = await this.vectorSearchWithin(query.like || query.similar, candidates);
|
||||
}
|
||||
// If candidates.length === 0 and this isn't the first step, keep empty candidates
|
||||
break;
|
||||
case 'graph':
|
||||
// CRITICAL: Same logic as vector - respect empty candidates from previous steps
|
||||
if (candidates.length === 0 && plan.steps[0].type === 'graph') {
|
||||
// This is the first step - do initial graph traversal
|
||||
candidates = await this.graphTraversal(query.connected);
|
||||
}
|
||||
else if (candidates.length > 0) {
|
||||
// Graph expansion from existing candidates
|
||||
candidates = await this.graphExpand(candidates);
|
||||
}
|
||||
// If candidates.length === 0 and this isn't the first step, keep empty candidates
|
||||
break;
|
||||
case 'fusion':
|
||||
// Final fusion ranking
|
||||
return this.fusionRank([candidates], query);
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
/**
|
||||
* Vector similarity search
|
||||
*/
|
||||
async vectorSearch(query, limit) {
|
||||
// Use clean internal vector search API to avoid circular dependency
|
||||
// This is the proper architecture: find() uses internal methods, not public search()
|
||||
return this.api.vectorSearch(query, limit || 100);
|
||||
}
|
||||
/**
|
||||
* Graph traversal
|
||||
*/
|
||||
async graphTraversal(connected) {
|
||||
// Get starting nodes
|
||||
const startNodes = connected.from ?
|
||||
(Array.isArray(connected.from) ? connected.from : [connected.from]) :
|
||||
connected.to ?
|
||||
(Array.isArray(connected.to) ? connected.to : [connected.to]) :
|
||||
[];
|
||||
// Use the API for graph traversal
|
||||
const options = {
|
||||
start: startNodes,
|
||||
type: connected.type,
|
||||
direction: connected.direction || 'both',
|
||||
maxDepth: connected.maxDepth || connected.depth || 2
|
||||
};
|
||||
const results = await this.api.graphTraversal(options);
|
||||
// Convert to expected format
|
||||
return results.map(r => ({
|
||||
id: r.id,
|
||||
type: connected.type || 'relates_to',
|
||||
score: r.score
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* Field-based filtering using MetadataIndex for O(log n) performance
|
||||
* NO FALLBACKS - Requires proper where clause and MetadataIndex
|
||||
*/
|
||||
async fieldFilter(where) {
|
||||
// Require a valid where clause - no empty queries allowed
|
||||
if (!where || Object.keys(where).length === 0) {
|
||||
throw new Error('Field filter requires a where clause. ' +
|
||||
'For retrieving all items, use a different query type or specify explicit criteria.');
|
||||
}
|
||||
// Verify MetadataIndex is available
|
||||
if (!this.api.hasMetadataIndex || !this.api.hasMetadataIndex()) {
|
||||
throw new Error('MetadataIndex not available - cannot perform O(log n) field queries. ' +
|
||||
'Initialize Brainy with enableMetadataIndex: true');
|
||||
}
|
||||
// Use the MetadataIndex for O(log n) performance
|
||||
// This uses B-tree indexes for range queries and hash indexes for exact matches
|
||||
const startTime = performance.now();
|
||||
const matchingIds = await this.api.metadataQuery(where);
|
||||
// Verify we got results from the fast path
|
||||
if (!matchingIds) {
|
||||
throw new Error('MetadataIndex query failed - no fallback allowed');
|
||||
}
|
||||
// Track performance metrics
|
||||
const queryTime = performance.now() - startTime;
|
||||
const expectedTime = Math.log2(1000000) * 5; // Assume max 1M items, 5ms per log operation
|
||||
if (queryTime > expectedTime * 2) {
|
||||
console.warn(`Field filter performance warning: ${queryTime.toFixed(2)}ms > expected ${expectedTime.toFixed(2)}ms`);
|
||||
}
|
||||
// Convert matching IDs to result format with full entities
|
||||
const results = [];
|
||||
const idsArray = Array.from(matchingIds);
|
||||
// Process results in batches for efficiency
|
||||
const batchSize = 100;
|
||||
for (let i = 0; i < Math.min(idsArray.length, 1000); i += batchSize) {
|
||||
const batch = idsArray.slice(i, i + batchSize);
|
||||
const entities = await Promise.all(batch.map(id => this.api.getEntity(id)));
|
||||
for (let j = 0; j < entities.length; j++) {
|
||||
const entity = entities[j];
|
||||
if (entity) {
|
||||
results.push({
|
||||
id: batch[j],
|
||||
score: 1.0, // Field matches are binary
|
||||
entity,
|
||||
metadata: entity.metadata || {}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Execute a single signal query directly
|
||||
*/
|
||||
async executeSingleSignal(query, signalType) {
|
||||
switch (signalType) {
|
||||
case 'vector':
|
||||
return this.vectorSearch(query.like || query.similar, query.limit);
|
||||
case 'graph':
|
||||
return this.graphTraversal(query.connected);
|
||||
case 'field':
|
||||
return this.fieldFilter(query.where);
|
||||
default:
|
||||
throw new Error(`Unknown signal type: ${signalType}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Expand graph connections from existing candidates
|
||||
*/
|
||||
async graphExpand(candidates) {
|
||||
const expanded = [];
|
||||
const visited = new Set();
|
||||
// For each candidate, find its graph neighbors
|
||||
for (const candidate of candidates) {
|
||||
const id = candidate.id || candidate;
|
||||
if (visited.has(id))
|
||||
continue;
|
||||
visited.add(id);
|
||||
// Get connections for this node
|
||||
const [sourceVerbs, targetVerbs] = await Promise.all([
|
||||
this.api.getVerbsBySource(id),
|
||||
this.api.getVerbsByTarget(id)
|
||||
]);
|
||||
// Add the original candidate
|
||||
expanded.push(candidate);
|
||||
// Add connected nodes
|
||||
for (const verb of sourceVerbs) {
|
||||
if (!visited.has(verb.targetId)) {
|
||||
const entity = await this.api.getEntity(verb.targetId);
|
||||
if (entity) {
|
||||
expanded.push({
|
||||
id: verb.targetId,
|
||||
score: (candidate.score || 1.0) * 0.8, // Decay score by distance
|
||||
entity,
|
||||
metadata: entity.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const verb of targetVerbs) {
|
||||
if (!visited.has(verb.sourceId)) {
|
||||
const entity = await this.api.getEntity(verb.sourceId);
|
||||
if (entity) {
|
||||
expanded.push({
|
||||
id: verb.sourceId,
|
||||
score: (candidate.score || 1.0) * 0.8, // Decay score by distance
|
||||
entity,
|
||||
metadata: entity.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return expanded;
|
||||
}
|
||||
/**
|
||||
* Vector search within existing candidates
|
||||
*/
|
||||
async vectorSearchWithin(query, candidates) {
|
||||
// Get the query vector
|
||||
const queryVector = typeof query === 'string' ?
|
||||
(await this.api.vectorSearch(query, 1))[0]?.entity?.vector :
|
||||
query;
|
||||
if (!queryVector)
|
||||
return candidates;
|
||||
// Score each candidate by vector similarity
|
||||
const scored = [];
|
||||
for (const candidate of candidates) {
|
||||
const entity = candidate.entity || await this.api.getEntity(candidate.id);
|
||||
if (entity && entity.vector) {
|
||||
const similarity = this.cosineSimilarity(queryVector, entity.vector);
|
||||
scored.push({
|
||||
...candidate,
|
||||
score: similarity,
|
||||
entity
|
||||
});
|
||||
}
|
||||
}
|
||||
// Sort by similarity and return
|
||||
return scored.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
/**
|
||||
* Apply field filter to existing candidates
|
||||
*/
|
||||
applyFieldFilter(candidates, where) {
|
||||
return candidates.filter(candidate => {
|
||||
const metadata = candidate.metadata || candidate.entity?.metadata || {};
|
||||
return this.matchesFilter(metadata, where);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Check if metadata matches filter conditions
|
||||
*/
|
||||
matchesFilter(metadata, where) {
|
||||
for (const [field, condition] of Object.entries(where)) {
|
||||
const value = metadata[field];
|
||||
if (typeof condition === 'object' && condition !== null) {
|
||||
// Handle operators
|
||||
if ('$eq' in condition && value !== condition.$eq)
|
||||
return false;
|
||||
if ('$ne' in condition && value === condition.$ne)
|
||||
return false;
|
||||
if ('$gt' in condition && !(value > condition.$gt))
|
||||
return false;
|
||||
if ('$gte' in condition && !(value >= condition.$gte))
|
||||
return false;
|
||||
if ('$lt' in condition && !(value < condition.$lt))
|
||||
return false;
|
||||
if ('$lte' in condition && !(value <= condition.$lte))
|
||||
return false;
|
||||
if ('$in' in condition && !condition.$in.includes(value))
|
||||
return false;
|
||||
if ('$nin' in condition && condition.$nin.includes(value))
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
// Direct equality
|
||||
if (value !== condition)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Calculate cosine similarity between two vectors
|
||||
*/
|
||||
cosineSimilarity(a, b) {
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
||||
}
|
||||
/**
|
||||
* Fusion ranking using Reciprocal Rank Fusion (RRF)
|
||||
* This is the same algorithm used by Google and Elasticsearch
|
||||
*/
|
||||
fusionRank(resultSets, query) {
|
||||
// RRF constant - 60 is empirically proven optimal
|
||||
const k = 60;
|
||||
// Calculate dynamic weights based on query
|
||||
const weights = this.calculateSignalWeights(query);
|
||||
// Determine which result sets we have based on query
|
||||
let vectorResultsIdx = -1;
|
||||
let graphResultsIdx = -1;
|
||||
let metadataResultsIdx = -1;
|
||||
let currentIdx = 0;
|
||||
if (query.like || query.similar) {
|
||||
vectorResultsIdx = currentIdx++;
|
||||
}
|
||||
if (query.where) {
|
||||
metadataResultsIdx = currentIdx++;
|
||||
}
|
||||
// If we have metadata filters AND other searches, apply intersection
|
||||
if (metadataResultsIdx >= 0 && resultSets.length > 1) {
|
||||
const metadataResults = resultSets[metadataResultsIdx];
|
||||
// CRITICAL: If metadata filter returned no results, entire query should return empty
|
||||
if (metadataResults.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const metadataIds = new Set(metadataResults.map(r => r.id || r));
|
||||
// Filter ALL other result sets to only include items that match metadata
|
||||
for (let i = 0; i < resultSets.length; i++) {
|
||||
if (i !== metadataResultsIdx) {
|
||||
resultSets[i] = resultSets[i].filter(r => metadataIds.has(r.id || r));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Build fusion scores using RRF
|
||||
const fusionScores = new Map();
|
||||
// Process each result set with RRF
|
||||
resultSets.forEach((results, setIndex) => {
|
||||
// Determine signal type
|
||||
let signalType;
|
||||
let weight = 1.0;
|
||||
if (setIndex === vectorResultsIdx) {
|
||||
signalType = 'vector';
|
||||
weight = weights.vector;
|
||||
}
|
||||
else if (setIndex === graphResultsIdx) {
|
||||
signalType = 'graph';
|
||||
weight = weights.graph;
|
||||
}
|
||||
else if (setIndex === metadataResultsIdx) {
|
||||
signalType = 'field';
|
||||
weight = weights.field;
|
||||
}
|
||||
else {
|
||||
return; // Skip unknown signal types
|
||||
}
|
||||
// Apply RRF to each result
|
||||
results.forEach((result, rank) => {
|
||||
const id = result.id || result;
|
||||
// Calculate RRF score: 1 / (k + rank + 1)
|
||||
const rrfScore = weight * (1.0 / (k + rank + 1));
|
||||
if (!fusionScores.has(id)) {
|
||||
fusionScores.set(id, {
|
||||
entity: result.entity || result,
|
||||
vectorScore: 0,
|
||||
graphScore: 0,
|
||||
fieldScore: 0,
|
||||
rrfScore: 0,
|
||||
fusionScore: 0,
|
||||
metadata: result.metadata || {}
|
||||
});
|
||||
}
|
||||
const fusion = fusionScores.get(id);
|
||||
// Track individual signal scores
|
||||
if (signalType === 'vector') {
|
||||
fusion.vectorScore = result.score || 1.0;
|
||||
}
|
||||
else if (signalType === 'graph') {
|
||||
fusion.graphScore = result.score || 1.0;
|
||||
}
|
||||
else if (signalType === 'field') {
|
||||
fusion.fieldScore = result.score || 1.0;
|
||||
}
|
||||
// Accumulate RRF score
|
||||
fusion.rrfScore += rrfScore;
|
||||
});
|
||||
});
|
||||
// Convert to results array
|
||||
const results = Array.from(fusionScores.entries()).map(([id, fusion]) => ({
|
||||
id,
|
||||
entity: fusion.entity,
|
||||
score: fusion.rrfScore, // Use RRF score as primary score
|
||||
vector: fusion.entity?.vector || new Float32Array(0), // Include vector for SearchResult compatibility
|
||||
vectorScore: fusion.vectorScore,
|
||||
graphScore: fusion.graphScore,
|
||||
fieldScore: fusion.fieldScore,
|
||||
fusionScore: fusion.rrfScore, // RRF score is the fusion score
|
||||
metadata: fusion.metadata
|
||||
}));
|
||||
// Sort by fusion score (descending)
|
||||
results.sort((a, b) => b.fusionScore - a.fusionScore);
|
||||
// Apply offset and limit
|
||||
let final = results;
|
||||
if (query.offset && query.offset > 0) {
|
||||
final = final.slice(query.offset);
|
||||
}
|
||||
if (query.limit) {
|
||||
final = final.slice(0, query.limit);
|
||||
}
|
||||
return final;
|
||||
}
|
||||
/**
|
||||
* Calculate dynamic signal weights based on query characteristics
|
||||
*/
|
||||
calculateSignalWeights(query) {
|
||||
const hasVector = !!(query.like || query.similar);
|
||||
const hasGraph = !!(query.connected);
|
||||
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||||
// Count active signals
|
||||
const activeSignals = [hasVector, hasGraph, hasField].filter(Boolean).length;
|
||||
if (activeSignals === 1) {
|
||||
// Single signal - full weight
|
||||
return {
|
||||
vector: hasVector ? 1.0 : 0,
|
||||
graph: hasGraph ? 1.0 : 0,
|
||||
field: hasField ? 1.0 : 0
|
||||
};
|
||||
}
|
||||
// Multiple signals - adaptive weights
|
||||
if (hasVector && hasGraph && hasField) {
|
||||
// All three signals - balanced with slight vector preference
|
||||
return {
|
||||
vector: 0.4, // Semantic search is often most relevant
|
||||
graph: 0.35, // Relationships are important
|
||||
field: 0.25 // Metadata is supportive
|
||||
};
|
||||
}
|
||||
else if (hasVector && hasGraph) {
|
||||
// Vector + Graph - emphasize semantics
|
||||
return {
|
||||
vector: 0.6,
|
||||
graph: 0.4,
|
||||
field: 0
|
||||
};
|
||||
}
|
||||
else if (hasVector && hasField) {
|
||||
// Vector + Field - balanced
|
||||
return {
|
||||
vector: 0.5,
|
||||
graph: 0,
|
||||
field: 0.5
|
||||
};
|
||||
}
|
||||
else if (hasGraph && hasField) {
|
||||
// Graph + Field - emphasize relationships
|
||||
return {
|
||||
vector: 0,
|
||||
graph: 0.6,
|
||||
field: 0.4
|
||||
};
|
||||
}
|
||||
// Default balanced weights
|
||||
return {
|
||||
vector: 0.34,
|
||||
graph: 0.33,
|
||||
field: 0.33
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Apply boost strategies
|
||||
*/
|
||||
applyBoosts(results, boost) {
|
||||
return results.map(r => {
|
||||
let boostFactor = 1.0;
|
||||
switch (boost) {
|
||||
case 'recent':
|
||||
// Boost recent items
|
||||
const age = Date.now() - (r.metadata?.timestamp || 0);
|
||||
boostFactor = Math.exp(-age / (30 * 24 * 60 * 60 * 1000)); // 30-day half-life
|
||||
break;
|
||||
case 'popular':
|
||||
// Boost by view count or connections
|
||||
boostFactor = Math.log10((r.metadata?.views || 0) + 10) / 2;
|
||||
break;
|
||||
case 'verified':
|
||||
// Boost verified content
|
||||
boostFactor = r.metadata?.verified ? 1.5 : 1.0;
|
||||
break;
|
||||
}
|
||||
return {
|
||||
...r,
|
||||
fusionScore: r.fusionScore * boostFactor
|
||||
};
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Add query explanations for debugging
|
||||
*/
|
||||
addExplanations(results, plan, totalTime) {
|
||||
return results.map(r => ({
|
||||
...r,
|
||||
explanation: {
|
||||
plan: plan.steps.map(s => `${s.type}:${s.operation}`).join(' → '),
|
||||
timing: {
|
||||
total: totalTime,
|
||||
...plan.steps.reduce((acc, step) => ({
|
||||
...acc,
|
||||
[step.type]: step.estimated
|
||||
}), {})
|
||||
},
|
||||
boosts: []
|
||||
}
|
||||
}));
|
||||
}
|
||||
// Query learning removed - unnecessary complexity
|
||||
/**
|
||||
* Optimize plan based on historical patterns
|
||||
*/
|
||||
// Query optimization from history removed
|
||||
/**
|
||||
* Clear query optimization cache
|
||||
*/
|
||||
clearCache() {
|
||||
this.planCache.clear();
|
||||
}
|
||||
/**
|
||||
* Get optimization statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
cachedPlans: this.planCache.size,
|
||||
historySize: 0 // Query history removed
|
||||
};
|
||||
}
|
||||
}
|
||||
// Export a beautiful, simple API
|
||||
export async function find(brain, query) {
|
||||
const engine = new TripleIntelligenceEngine(brain);
|
||||
return engine.find(query);
|
||||
}
|
||||
//# sourceMappingURL=TripleIntelligence.js.map
|
||||
File diff suppressed because one or more lines are too long
159
.recovery-workspace/dist-backup-20250910-141917/triple/TripleIntelligenceSystem.d.ts
vendored
Normal file
159
.recovery-workspace/dist-backup-20250910-141917/triple/TripleIntelligenceSystem.d.ts
vendored
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
/**
|
||||
* Triple Intelligence System - Consolidated, Production-Ready Implementation
|
||||
*
|
||||
* NO FALLBACKS - NO MOCKS - NO STUBS - REAL PERFORMANCE
|
||||
*
|
||||
* This is the single source of truth for Triple Intelligence operations.
|
||||
* All operations MUST use fast paths or FAIL LOUDLY.
|
||||
*
|
||||
* Performance Guarantees:
|
||||
* - Vector search: O(log n) via HNSW
|
||||
* - Range queries: O(log n) via B-tree indexes
|
||||
* - Graph traversal: O(1) adjacency list lookups
|
||||
* - Fusion: O(k log k) where k = result count
|
||||
*/
|
||||
import { HNSWIndex } from '../hnsw/hnswIndex.js';
|
||||
import { MetadataIndexManager } from '../utils/metadataIndex.js';
|
||||
import { Vector } from '../coreTypes.js';
|
||||
export interface TripleQuery {
|
||||
similar?: string;
|
||||
like?: string;
|
||||
vector?: Vector;
|
||||
where?: Record<string, any>;
|
||||
connected?: {
|
||||
from?: string;
|
||||
to?: string;
|
||||
type?: string;
|
||||
direction?: 'in' | 'out' | 'both';
|
||||
depth?: number;
|
||||
};
|
||||
limit?: number;
|
||||
}
|
||||
export interface TripleOptions {
|
||||
fusion?: {
|
||||
strategy?: 'rrf' | 'weighted' | 'adaptive';
|
||||
weights?: Record<string, number>;
|
||||
k?: number;
|
||||
};
|
||||
}
|
||||
interface GraphAdjacencyIndex {
|
||||
getNeighbors(id: string, direction?: 'in' | 'out' | 'both'): Promise<string[]>;
|
||||
size(): number;
|
||||
}
|
||||
/**
|
||||
* Performance metrics for monitoring and assertions
|
||||
*/
|
||||
export declare class PerformanceMetrics {
|
||||
private operations;
|
||||
private slowQueries;
|
||||
private totalItems;
|
||||
recordOperation(type: string, elapsed: number, itemCount?: number): void;
|
||||
private getExpectedTime;
|
||||
setTotalItems(count: number): void;
|
||||
getReport(): PerformanceReport;
|
||||
reset(): void;
|
||||
}
|
||||
/**
|
||||
* The main Triple Intelligence System
|
||||
*/
|
||||
export declare class TripleIntelligenceSystem {
|
||||
private metadataIndex;
|
||||
private hnswIndex;
|
||||
private graphIndex;
|
||||
private metrics;
|
||||
private planner;
|
||||
private embedder;
|
||||
private storage;
|
||||
constructor(metadataIndex: MetadataIndexManager, hnswIndex: HNSWIndex, graphIndex: GraphAdjacencyIndex, embedder: (text: string) => Promise<Vector>, storage: any);
|
||||
/**
|
||||
* Main find method - executes Triple Intelligence queries
|
||||
*/
|
||||
find(query: TripleQuery, options?: TripleOptions): Promise<TripleResult[]>;
|
||||
/**
|
||||
* Vector search using HNSW for O(log n) performance
|
||||
*/
|
||||
private vectorSearch;
|
||||
/**
|
||||
* Field filtering using MetadataIndex for O(log n) performance
|
||||
*/
|
||||
private fieldFilter;
|
||||
/**
|
||||
* Graph traversal using adjacency lists for O(1) lookups
|
||||
*/
|
||||
private graphTraversal;
|
||||
/**
|
||||
* Execute the query plan
|
||||
*/
|
||||
private executeQueryPlan;
|
||||
/**
|
||||
* Fuse results using Reciprocal Rank Fusion (RRF)
|
||||
*/
|
||||
private fuseResults;
|
||||
/**
|
||||
* Validate query parameters
|
||||
*/
|
||||
private validateQuery;
|
||||
/**
|
||||
* Verify required indexes are available
|
||||
*/
|
||||
private verifyIndexes;
|
||||
/**
|
||||
* Assert performance guarantees
|
||||
*/
|
||||
private assertPerformance;
|
||||
/**
|
||||
* Check if where clause has range operators
|
||||
*/
|
||||
private hasRangeOperators;
|
||||
/**
|
||||
* Update item count for metrics
|
||||
*/
|
||||
private updateItemCount;
|
||||
/**
|
||||
* Get total item count across all indexes
|
||||
*/
|
||||
private getTotalItems;
|
||||
/**
|
||||
* Get performance metrics
|
||||
*/
|
||||
getMetrics(): PerformanceMetrics;
|
||||
/**
|
||||
* Reset performance metrics
|
||||
*/
|
||||
resetMetrics(): void;
|
||||
}
|
||||
interface QueryLog {
|
||||
type: string;
|
||||
elapsed: number;
|
||||
expectedTime: number;
|
||||
timestamp: number;
|
||||
itemCount: number;
|
||||
}
|
||||
interface PerformanceReport {
|
||||
operations: Record<string, {
|
||||
avgTime: number;
|
||||
maxTime: number;
|
||||
minTime: number;
|
||||
violations: number;
|
||||
violationRate: number;
|
||||
totalCalls: number;
|
||||
}>;
|
||||
violations: Array<{
|
||||
type: string;
|
||||
count: number;
|
||||
rate: number;
|
||||
}>;
|
||||
slowQueries: QueryLog[];
|
||||
}
|
||||
interface TripleResult {
|
||||
id: string;
|
||||
score: number;
|
||||
entity: any;
|
||||
metadata: Record<string, any>;
|
||||
vectorScore?: number;
|
||||
fieldScore?: number;
|
||||
graphScore?: number;
|
||||
fusionScore?: number;
|
||||
depth?: number;
|
||||
}
|
||||
export {};
|
||||
|
|
@ -0,0 +1,519 @@
|
|||
/**
|
||||
* Triple Intelligence System - Consolidated, Production-Ready Implementation
|
||||
*
|
||||
* NO FALLBACKS - NO MOCKS - NO STUBS - REAL PERFORMANCE
|
||||
*
|
||||
* This is the single source of truth for Triple Intelligence operations.
|
||||
* All operations MUST use fast paths or FAIL LOUDLY.
|
||||
*
|
||||
* Performance Guarantees:
|
||||
* - Vector search: O(log n) via HNSW
|
||||
* - Range queries: O(log n) via B-tree indexes
|
||||
* - Graph traversal: O(1) adjacency list lookups
|
||||
* - Fusion: O(k log k) where k = result count
|
||||
*/
|
||||
/**
|
||||
* Performance metrics for monitoring and assertions
|
||||
*/
|
||||
export class PerformanceMetrics {
|
||||
constructor() {
|
||||
this.operations = new Map();
|
||||
this.slowQueries = [];
|
||||
this.totalItems = 0;
|
||||
}
|
||||
recordOperation(type, elapsed, itemCount) {
|
||||
const stats = this.operations.get(type) || {
|
||||
count: 0,
|
||||
totalTime: 0,
|
||||
maxTime: 0,
|
||||
minTime: Infinity,
|
||||
violations: 0
|
||||
};
|
||||
stats.count++;
|
||||
stats.totalTime += elapsed;
|
||||
stats.maxTime = Math.max(stats.maxTime, elapsed);
|
||||
stats.minTime = Math.min(stats.minTime, elapsed);
|
||||
// Check for O(log n) violation
|
||||
const expectedTime = this.getExpectedTime(type, itemCount || this.totalItems);
|
||||
if (elapsed > expectedTime * 2) {
|
||||
stats.violations++;
|
||||
console.error(`⚠️ Performance violation in ${type}: ${elapsed.toFixed(2)}ms > expected ${expectedTime.toFixed(2)}ms`);
|
||||
this.slowQueries.push({
|
||||
type,
|
||||
elapsed,
|
||||
expectedTime,
|
||||
timestamp: Date.now(),
|
||||
itemCount: itemCount || this.totalItems
|
||||
});
|
||||
}
|
||||
this.operations.set(type, stats);
|
||||
}
|
||||
getExpectedTime(type, itemCount) {
|
||||
// O(log n) operations should complete in roughly log2(n) * k milliseconds
|
||||
// where k is a constant based on the operation type
|
||||
const logN = Math.log2(Math.max(1, itemCount));
|
||||
switch (type) {
|
||||
case 'vector_search':
|
||||
return logN * 5; // HNSW is very efficient
|
||||
case 'field_filter':
|
||||
return logN * 3; // B-tree operations are fast
|
||||
case 'graph_traversal':
|
||||
return 10; // O(1) adjacency list lookups
|
||||
case 'fusion':
|
||||
return Math.log2(Math.max(1, itemCount)) * 2; // O(k log k) sorting
|
||||
default:
|
||||
return logN * 10; // Conservative estimate
|
||||
}
|
||||
}
|
||||
setTotalItems(count) {
|
||||
this.totalItems = count;
|
||||
}
|
||||
getReport() {
|
||||
const report = {
|
||||
operations: {},
|
||||
violations: [],
|
||||
slowQueries: this.slowQueries.slice(-100) // Last 100 slow queries
|
||||
};
|
||||
for (const [type, stats] of this.operations) {
|
||||
report.operations[type] = {
|
||||
avgTime: stats.totalTime / stats.count,
|
||||
maxTime: stats.maxTime,
|
||||
minTime: stats.minTime,
|
||||
violations: stats.violations,
|
||||
violationRate: stats.violations / stats.count,
|
||||
totalCalls: stats.count
|
||||
};
|
||||
if (stats.violations > 0) {
|
||||
report.violations.push({
|
||||
type,
|
||||
count: stats.violations,
|
||||
rate: stats.violations / stats.count
|
||||
});
|
||||
}
|
||||
}
|
||||
return report;
|
||||
}
|
||||
reset() {
|
||||
this.operations.clear();
|
||||
this.slowQueries = [];
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Query execution planner - optimizes query execution order
|
||||
*/
|
||||
class QueryPlanner {
|
||||
/**
|
||||
* Build an optimized execution plan for a query
|
||||
*/
|
||||
buildPlan(query) {
|
||||
const plan = {
|
||||
steps: [],
|
||||
estimatedCost: 0,
|
||||
requiresIndexes: []
|
||||
};
|
||||
// Determine which indexes are required
|
||||
if (query.similar || query.like) {
|
||||
plan.requiresIndexes.push('hnsw');
|
||||
}
|
||||
if (query.where) {
|
||||
plan.requiresIndexes.push('metadata');
|
||||
}
|
||||
if (query.connected) {
|
||||
plan.requiresIndexes.push('graph');
|
||||
}
|
||||
// Order operations by selectivity (most selective first)
|
||||
// This minimizes the working set for subsequent operations
|
||||
// 1. Field filters are usually most selective
|
||||
if (query.where) {
|
||||
plan.steps.push({
|
||||
type: 'field',
|
||||
operation: 'filter',
|
||||
requiresFastPath: true,
|
||||
estimatedSelectivity: 0.1 // Assume 10% match rate
|
||||
});
|
||||
}
|
||||
// 2. Graph traversal is moderately selective
|
||||
if (query.connected) {
|
||||
plan.steps.push({
|
||||
type: 'graph',
|
||||
operation: 'traverse',
|
||||
requiresFastPath: true,
|
||||
estimatedSelectivity: 0.3
|
||||
});
|
||||
}
|
||||
// 3. Vector search is least selective (returns top-k)
|
||||
if (query.similar || query.like) {
|
||||
plan.steps.push({
|
||||
type: 'vector',
|
||||
operation: 'search',
|
||||
requiresFastPath: true,
|
||||
estimatedSelectivity: 1.0
|
||||
});
|
||||
}
|
||||
// Calculate estimated cost
|
||||
plan.estimatedCost = plan.steps.reduce((cost, step) => {
|
||||
return cost + (1 / step.estimatedSelectivity);
|
||||
}, 0);
|
||||
return plan;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* The main Triple Intelligence System
|
||||
*/
|
||||
export class TripleIntelligenceSystem {
|
||||
constructor(metadataIndex, hnswIndex, graphIndex, embedder, storage) {
|
||||
// REQUIRE all components - no fallbacks
|
||||
if (!metadataIndex) {
|
||||
throw new Error('MetadataIndex required for Triple Intelligence');
|
||||
}
|
||||
if (!hnswIndex) {
|
||||
throw new Error('HNSW index required for Triple Intelligence');
|
||||
}
|
||||
if (!graphIndex) {
|
||||
throw new Error('Graph index required for Triple Intelligence');
|
||||
}
|
||||
if (!embedder) {
|
||||
throw new Error('Embedding function required for Triple Intelligence');
|
||||
}
|
||||
if (!storage) {
|
||||
throw new Error('Storage adapter required for Triple Intelligence');
|
||||
}
|
||||
this.metadataIndex = metadataIndex;
|
||||
this.hnswIndex = hnswIndex;
|
||||
this.graphIndex = graphIndex;
|
||||
this.embedder = embedder;
|
||||
this.storage = storage;
|
||||
this.metrics = new PerformanceMetrics();
|
||||
this.planner = new QueryPlanner();
|
||||
// Set initial item count for metrics
|
||||
this.updateItemCount();
|
||||
}
|
||||
/**
|
||||
* Main find method - executes Triple Intelligence queries
|
||||
*/
|
||||
async find(query, options) {
|
||||
const startTime = performance.now();
|
||||
// Validate query
|
||||
this.validateQuery(query);
|
||||
// Build optimized query plan
|
||||
const plan = this.planner.buildPlan(query);
|
||||
// Verify all required indexes are available
|
||||
this.verifyIndexes(plan.requiresIndexes);
|
||||
// Execute query plan with NO FALLBACKS
|
||||
const results = await this.executeQueryPlan(plan, query, options);
|
||||
// Record metrics
|
||||
const elapsed = performance.now() - startTime;
|
||||
this.metrics.recordOperation('find_query', elapsed, results.length);
|
||||
// ASSERT performance guarantees
|
||||
this.assertPerformance(elapsed, results.length);
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Vector search using HNSW for O(log n) performance
|
||||
*/
|
||||
async vectorSearch(query, limit) {
|
||||
const startTime = performance.now();
|
||||
// Convert text to vector if needed
|
||||
const vector = typeof query === 'string'
|
||||
? await this.embedder(query)
|
||||
: query;
|
||||
// Search using HNSW index - O(log n) guaranteed
|
||||
const searchResults = await this.hnswIndex.search(vector, limit);
|
||||
// Convert to result format
|
||||
const results = [];
|
||||
for (const [id, score] of searchResults) {
|
||||
const entity = await this.storage.getNoun(id);
|
||||
if (entity) {
|
||||
results.push({
|
||||
id,
|
||||
score,
|
||||
entity,
|
||||
metadata: entity.metadata || {},
|
||||
vectorScore: score
|
||||
});
|
||||
}
|
||||
}
|
||||
const elapsed = performance.now() - startTime;
|
||||
this.metrics.recordOperation('vector_search', elapsed, results.length);
|
||||
// Assert O(log n) performance
|
||||
const expectedTime = Math.log2(this.hnswIndex.size()) * 5;
|
||||
if (elapsed > expectedTime * 2) {
|
||||
throw new Error(`Vector search O(log n) violation: ${elapsed.toFixed(2)}ms > ${expectedTime.toFixed(2)}ms`);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Field filtering using MetadataIndex for O(log n) performance
|
||||
*/
|
||||
async fieldFilter(where, limit) {
|
||||
const startTime = performance.now();
|
||||
// Use MetadataIndex for O(log n) performance
|
||||
const matchingIds = await this.metadataIndex.getIdsForFilter(where);
|
||||
if (!matchingIds || matchingIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
// Convert to results with full entities
|
||||
const results = [];
|
||||
const idsToProcess = limit
|
||||
? matchingIds.slice(0, limit)
|
||||
: matchingIds;
|
||||
// Process in parallel batches for efficiency
|
||||
const batchSize = 100;
|
||||
for (let i = 0; i < idsToProcess.length; i += batchSize) {
|
||||
const batch = idsToProcess.slice(i, i + batchSize);
|
||||
const entities = await Promise.all(batch.map(id => this.storage.getNoun(id)));
|
||||
for (let j = 0; j < entities.length; j++) {
|
||||
const entity = entities[j];
|
||||
if (entity) {
|
||||
results.push({
|
||||
id: batch[j],
|
||||
score: 1.0, // Field matches are binary
|
||||
entity,
|
||||
metadata: entity.metadata || {},
|
||||
fieldScore: 1.0
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
const elapsed = performance.now() - startTime;
|
||||
this.metrics.recordOperation('field_filter', elapsed, results.length);
|
||||
// Assert O(log n) for range queries
|
||||
if (this.hasRangeOperators(where)) {
|
||||
const expectedTime = Math.log2(1000000) * 3; // Assume max 1M items
|
||||
if (elapsed > expectedTime * 2) {
|
||||
throw new Error(`Field filter O(log n) violation: ${elapsed.toFixed(2)}ms > ${expectedTime.toFixed(2)}ms`);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Graph traversal using adjacency lists for O(1) lookups
|
||||
*/
|
||||
async graphTraversal(params) {
|
||||
const startTime = performance.now();
|
||||
const maxDepth = params.depth || 2;
|
||||
const results = [];
|
||||
const visited = new Set();
|
||||
// BFS traversal with O(1) adjacency lookups
|
||||
const queue = [];
|
||||
// Initialize queue with starting node(s)
|
||||
if (params.from) {
|
||||
queue.push({ id: params.from, depth: 0, score: 1.0 });
|
||||
}
|
||||
while (queue.length > 0) {
|
||||
const { id, depth, score } = queue.shift();
|
||||
if (visited.has(id) || depth > maxDepth) {
|
||||
continue;
|
||||
}
|
||||
visited.add(id);
|
||||
// Get entity
|
||||
const entity = await this.storage.getNoun(id);
|
||||
if (entity) {
|
||||
results.push({
|
||||
id,
|
||||
score: score * Math.pow(0.8, depth), // Decay by distance
|
||||
entity,
|
||||
metadata: entity.metadata || {},
|
||||
graphScore: score,
|
||||
depth
|
||||
});
|
||||
}
|
||||
// Get neighbors - O(1) adjacency list lookup
|
||||
if (depth < maxDepth) {
|
||||
const neighbors = await this.graphIndex.getNeighbors(id, params.direction);
|
||||
for (const neighborId of neighbors) {
|
||||
if (!visited.has(neighborId)) {
|
||||
queue.push({
|
||||
id: neighborId,
|
||||
depth: depth + 1,
|
||||
score: score * 0.8
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const elapsed = performance.now() - startTime;
|
||||
this.metrics.recordOperation('graph_traversal', elapsed, results.length);
|
||||
// Graph traversal should be fast due to O(1) adjacency lookups
|
||||
const expectedTime = visited.size * 0.5; // 0.5ms per node
|
||||
if (elapsed > expectedTime * 3) {
|
||||
throw new Error(`Graph traversal performance warning: ${elapsed.toFixed(2)}ms > ${expectedTime.toFixed(2)}ms`);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Execute the query plan
|
||||
*/
|
||||
async executeQueryPlan(plan, query, options) {
|
||||
const limit = query.limit || 10;
|
||||
const intermediateResults = new Map();
|
||||
// Execute each step in the plan
|
||||
for (const step of plan.steps) {
|
||||
const stepStartTime = performance.now();
|
||||
let stepResults = [];
|
||||
switch (step.type) {
|
||||
case 'vector':
|
||||
stepResults = await this.vectorSearch(query.similar || query.like, limit * 3 // Over-fetch for fusion
|
||||
);
|
||||
break;
|
||||
case 'field':
|
||||
stepResults = await this.fieldFilter(query.where, limit * 3);
|
||||
break;
|
||||
case 'graph':
|
||||
stepResults = await this.graphTraversal(query.connected);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown query step type: ${step.type}`);
|
||||
}
|
||||
intermediateResults.set(step.type, stepResults);
|
||||
const stepElapsed = performance.now() - stepStartTime;
|
||||
console.log(`Step ${step.type}:${step.operation} completed in ${stepElapsed.toFixed(2)}ms with ${stepResults.length} results`);
|
||||
}
|
||||
// Fuse results if multiple signals
|
||||
if (intermediateResults.size > 1) {
|
||||
return this.fuseResults(intermediateResults, limit, options);
|
||||
}
|
||||
// Single signal - return as is
|
||||
const singleResults = Array.from(intermediateResults.values())[0];
|
||||
return singleResults.slice(0, limit);
|
||||
}
|
||||
/**
|
||||
* Fuse results using Reciprocal Rank Fusion (RRF)
|
||||
*/
|
||||
fuseResults(resultSets, limit, options) {
|
||||
const startTime = performance.now();
|
||||
const k = options?.fusion?.k || 60; // RRF constant
|
||||
const weights = options?.fusion?.weights || {
|
||||
vector: 0.5,
|
||||
field: 0.3,
|
||||
graph: 0.2
|
||||
};
|
||||
// Calculate RRF scores
|
||||
const fusionScores = new Map();
|
||||
const entityMap = new Map();
|
||||
for (const [signalType, results] of resultSets) {
|
||||
const weight = weights[signalType] || 1.0;
|
||||
results.forEach((result, rank) => {
|
||||
const rrfScore = weight / (k + rank + 1);
|
||||
const currentScore = fusionScores.get(result.id) || 0;
|
||||
fusionScores.set(result.id, currentScore + rrfScore);
|
||||
// Keep the result with the most information
|
||||
if (!entityMap.has(result.id)) {
|
||||
entityMap.set(result.id, result);
|
||||
}
|
||||
});
|
||||
}
|
||||
// Sort by fusion score
|
||||
const sortedIds = Array.from(fusionScores.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, limit);
|
||||
// Build final results
|
||||
const results = [];
|
||||
for (const [id, fusionScore] of sortedIds) {
|
||||
const result = entityMap.get(id);
|
||||
results.push({
|
||||
...result,
|
||||
fusionScore,
|
||||
score: fusionScore // Use fusion score as primary score
|
||||
});
|
||||
}
|
||||
const elapsed = performance.now() - startTime;
|
||||
this.metrics.recordOperation('fusion', elapsed, results.length);
|
||||
// Fusion should be O(k log k)
|
||||
const expectedTime = Math.log2(Math.max(1, fusionScores.size)) * 2;
|
||||
if (elapsed > expectedTime * 3) {
|
||||
console.warn(`Fusion performance warning: ${elapsed.toFixed(2)}ms > ${expectedTime.toFixed(2)}ms`);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Validate query parameters
|
||||
*/
|
||||
validateQuery(query) {
|
||||
if (!query.similar && !query.like && !query.where && !query.connected) {
|
||||
throw new Error('Query must specify at least one of: similar, like, where, or connected');
|
||||
}
|
||||
if (query.limit && (query.limit < 1 || query.limit > 10000)) {
|
||||
throw new Error('Query limit must be between 1 and 10000');
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Verify required indexes are available
|
||||
*/
|
||||
verifyIndexes(required) {
|
||||
for (const index of required) {
|
||||
switch (index) {
|
||||
case 'hnsw':
|
||||
if (!this.hnswIndex || this.hnswIndex.size() === 0) {
|
||||
throw new Error('HNSW index not available or empty');
|
||||
}
|
||||
break;
|
||||
case 'metadata':
|
||||
if (!this.metadataIndex) {
|
||||
throw new Error('Metadata index not available');
|
||||
}
|
||||
break;
|
||||
case 'graph':
|
||||
if (!this.graphIndex) {
|
||||
throw new Error('Graph index not available');
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Assert performance guarantees
|
||||
*/
|
||||
assertPerformance(elapsed, resultCount) {
|
||||
const itemCount = this.getTotalItems();
|
||||
const expectedTime = Math.log2(Math.max(1, itemCount)) * 20; // 20ms per log operation
|
||||
if (elapsed > expectedTime * 3) {
|
||||
throw new Error(`Query performance violation: ${elapsed.toFixed(2)}ms > expected ${expectedTime.toFixed(2)}ms ` +
|
||||
`for ${itemCount} items`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if where clause has range operators
|
||||
*/
|
||||
hasRangeOperators(where) {
|
||||
for (const value of Object.values(where)) {
|
||||
if (typeof value === 'object' && value !== null) {
|
||||
const keys = Object.keys(value);
|
||||
if (keys.some(k => ['$gt', '$gte', '$lt', '$lte', '$between'].includes(k))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Update item count for metrics
|
||||
*/
|
||||
updateItemCount() {
|
||||
const count = this.getTotalItems();
|
||||
this.metrics.setTotalItems(count);
|
||||
}
|
||||
/**
|
||||
* Get total item count across all indexes
|
||||
*/
|
||||
getTotalItems() {
|
||||
// Get the largest count from available indexes
|
||||
// Note: MetadataIndexManager might not have a size() method
|
||||
// so we'll use HNSW index size as primary indicator
|
||||
return Math.max(this.hnswIndex?.size() || 0, 1000000, // Assume max 1M items for now
|
||||
this.graphIndex?.size() || 0);
|
||||
}
|
||||
/**
|
||||
* Get performance metrics
|
||||
*/
|
||||
getMetrics() {
|
||||
return this.metrics;
|
||||
}
|
||||
/**
|
||||
* Reset performance metrics
|
||||
*/
|
||||
resetMetrics() {
|
||||
this.metrics.reset();
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=TripleIntelligenceSystem.js.map
|
||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Add a link
Reference in a new issue