814 lines
31 KiB
JavaScript
814 lines
31 KiB
JavaScript
|
/**
|
||
|
|
* Triple Intelligence Engine
|
||
|
|
* Revolutionary unified search combining Vector + Graph + Field intelligence
|
||
|
|
*
|
||
|
|
* This is Brainy's killer feature - no other database can do this!
|
||
|
|
*/
|
||
|
|
/**
|
||
|
|
* The Triple Intelligence Engine
|
||
|
|
* Unifies vector, graph, and field search into one beautiful API
|
||
|
|
*/
|
||
|
|
export class TripleIntelligenceEngine {
|
||
|
|
constructor(brain) {
|
||
|
|
this.planCache = new Map();
|
||
|
|
this.brain = brain;
|
||
|
|
this.api = brain.getTripleIntelligenceAPI();
|
||
|
|
// Query history removed - unnecessary complexity for minimal gain
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* The magic happens here - one query to rule them all
|
||
|
|
*/
|
||
|
|
async find(query) {
|
||
|
|
const startTime = Date.now();
|
||
|
|
// Generate optimal query plan
|
||
|
|
const plan = await this.optimizeQuery(query);
|
||
|
|
// Execute based on plan
|
||
|
|
let results;
|
||
|
|
if (plan.canParallelize) {
|
||
|
|
// Run all three paths in parallel for maximum speed
|
||
|
|
results = await this.parallelSearch(query, plan);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Progressive filtering for efficiency
|
||
|
|
results = await this.progressiveSearch(query, plan);
|
||
|
|
}
|
||
|
|
// Apply boosts if requested
|
||
|
|
if (query.boost) {
|
||
|
|
results = this.applyBoosts(results, query.boost);
|
||
|
|
}
|
||
|
|
// Add explanations if requested
|
||
|
|
if (query.explain) {
|
||
|
|
const timing = Date.now() - startTime;
|
||
|
|
results = this.addExplanations(results, plan, timing);
|
||
|
|
}
|
||
|
|
// Query history removed - no learning needed
|
||
|
|
// Apply limit
|
||
|
|
if (query.limit) {
|
||
|
|
results = results.slice(0, query.limit);
|
||
|
|
}
|
||
|
|
return results;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Generate optimal execution plan based on query shape and statistics
|
||
|
|
*/
|
||
|
|
async optimizeQuery(query) {
|
||
|
|
// Short-circuit optimization for single-signal queries
|
||
|
|
const hasVector = !!(query.like || query.similar);
|
||
|
|
const hasGraph = !!(query.connected);
|
||
|
|
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||
|
|
const signalCount = [hasVector, hasGraph, hasField].filter(Boolean).length;
|
||
|
|
// Single signal - skip fusion entirely!
|
||
|
|
if (signalCount === 1) {
|
||
|
|
const singleType = hasVector ? 'vector' : hasGraph ? 'graph' : 'field';
|
||
|
|
return {
|
||
|
|
startWith: singleType,
|
||
|
|
canParallelize: false,
|
||
|
|
estimatedCost: 1,
|
||
|
|
steps: [{
|
||
|
|
type: singleType,
|
||
|
|
operation: 'direct', // Direct execution, no fusion
|
||
|
|
estimated: 50
|
||
|
|
}]
|
||
|
|
};
|
||
|
|
}
|
||
|
|
// Check cache first
|
||
|
|
const cacheKey = JSON.stringify(query);
|
||
|
|
if (this.planCache.has(cacheKey)) {
|
||
|
|
return this.planCache.get(cacheKey);
|
||
|
|
}
|
||
|
|
// Get real statistics for cost-based optimization
|
||
|
|
const stats = await this.api.getStatistics();
|
||
|
|
// Calculate costs for each operation
|
||
|
|
const costs = await this.calculateOperationCosts(query, stats);
|
||
|
|
// Build optimal plan based on actual costs
|
||
|
|
const plan = this.buildOptimalPlan(query, costs, stats);
|
||
|
|
this.planCache.set(cacheKey, plan);
|
||
|
|
return plan;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Calculate real costs for each operation based on statistics
|
||
|
|
*/
|
||
|
|
async calculateOperationCosts(query, stats) {
|
||
|
|
const costs = {
|
||
|
|
vector: Infinity,
|
||
|
|
graph: Infinity,
|
||
|
|
field: Infinity
|
||
|
|
};
|
||
|
|
// Vector search cost - O(log n) with HNSW
|
||
|
|
if (query.like || query.similar) {
|
||
|
|
// HNSW search complexity: O(log n) * ef
|
||
|
|
const ef = 200; // exploration factor
|
||
|
|
costs.vector = Math.log2(stats.totalCount) * ef;
|
||
|
|
}
|
||
|
|
// Graph traversal cost - depends on connectivity
|
||
|
|
if (query.connected) {
|
||
|
|
const depth = query.connected.maxDepth || query.connected.depth || 2;
|
||
|
|
// Assume average branching factor of 10
|
||
|
|
const branchingFactor = 10;
|
||
|
|
costs.graph = Math.pow(branchingFactor, depth);
|
||
|
|
}
|
||
|
|
// Field filter cost - depends on selectivity
|
||
|
|
if (query.where) {
|
||
|
|
const selectivity = await this.estimateFieldSelectivity(query.where, stats);
|
||
|
|
costs.field = stats.totalCount * selectivity;
|
||
|
|
// If we have an index, cost is O(log n)
|
||
|
|
if (this.api.hasMetadataIndex()) {
|
||
|
|
costs.field = Math.log2(stats.totalCount) + costs.field;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return costs;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Estimate selectivity of field filters
|
||
|
|
*/
|
||
|
|
async estimateFieldSelectivity(where, stats) {
|
||
|
|
let selectivity = 1.0;
|
||
|
|
for (const [field, condition] of Object.entries(where)) {
|
||
|
|
const fieldStats = stats.fieldStats[field];
|
||
|
|
if (!fieldStats) {
|
||
|
|
// Unknown field - assume 10% selectivity
|
||
|
|
selectivity *= 0.1;
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
if (typeof condition === 'object' && condition !== null) {
|
||
|
|
// Handle operators
|
||
|
|
if ('$eq' in condition) {
|
||
|
|
// Equality - 1/cardinality
|
||
|
|
selectivity *= 1.0 / (fieldStats.cardinality || 100);
|
||
|
|
}
|
||
|
|
else if ('$gt' in condition || '$gte' in condition) {
|
||
|
|
// Range query - estimate based on distribution
|
||
|
|
const threshold = condition.$gt || condition.$gte;
|
||
|
|
if (typeof threshold === 'number' && fieldStats.type === 'number') {
|
||
|
|
const range = fieldStats.max - fieldStats.min;
|
||
|
|
const remainingRange = fieldStats.max - threshold;
|
||
|
|
selectivity *= remainingRange / range;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
selectivity *= 0.3; // Default for non-numeric
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if ('$lt' in condition || '$lte' in condition) {
|
||
|
|
// Range query - estimate based on distribution
|
||
|
|
const threshold = condition.$lt || condition.$lte;
|
||
|
|
if (typeof threshold === 'number' && fieldStats.type === 'number') {
|
||
|
|
const range = fieldStats.max - fieldStats.min;
|
||
|
|
const remainingRange = threshold - fieldStats.min;
|
||
|
|
selectivity *= remainingRange / range;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
selectivity *= 0.3; // Default for non-numeric
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else if ('$in' in condition) {
|
||
|
|
// IN query
|
||
|
|
selectivity *= condition.$in.length / (fieldStats.cardinality || 100);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Direct equality
|
||
|
|
selectivity *= 1.0 / (fieldStats.cardinality || 100);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return Math.max(0.0001, Math.min(1.0, selectivity));
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Build optimal execution plan based on costs
|
||
|
|
*/
|
||
|
|
buildOptimalPlan(query, costs, stats) {
|
||
|
|
const hasVector = !!(query.like || query.similar);
|
||
|
|
const hasGraph = !!(query.connected);
|
||
|
|
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||
|
|
// Find the most selective operation
|
||
|
|
const sortedOps = Object.entries(costs)
|
||
|
|
.filter(([op]) => {
|
||
|
|
return (op === 'vector' && hasVector) ||
|
||
|
|
(op === 'graph' && hasGraph) ||
|
||
|
|
(op === 'field' && hasField);
|
||
|
|
})
|
||
|
|
.sort((a, b) => a[1] - b[1]);
|
||
|
|
// If the most selective operation filters out > 99%, start with it
|
||
|
|
const mostSelective = sortedOps[0];
|
||
|
|
if (mostSelective && mostSelective[1] < stats.totalCount * 0.01) {
|
||
|
|
// Progressive plan - start with most selective
|
||
|
|
return {
|
||
|
|
startWith: mostSelective[0],
|
||
|
|
canParallelize: false,
|
||
|
|
estimatedCost: mostSelective[1],
|
||
|
|
steps: this.buildProgressiveSteps(sortedOps, query)
|
||
|
|
};
|
||
|
|
}
|
||
|
|
// If operations have similar costs, parallelize
|
||
|
|
if (sortedOps.length > 1) {
|
||
|
|
const ratio = sortedOps[1][1] / sortedOps[0][1];
|
||
|
|
if (ratio < 10) {
|
||
|
|
// Costs are within 10x - parallelize
|
||
|
|
return {
|
||
|
|
startWith: sortedOps[0][0],
|
||
|
|
canParallelize: true,
|
||
|
|
estimatedCost: Math.max(...sortedOps.map(op => op[1])),
|
||
|
|
steps: this.buildParallelSteps(sortedOps, query)
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Default progressive plan
|
||
|
|
return {
|
||
|
|
startWith: sortedOps[0][0],
|
||
|
|
canParallelize: false,
|
||
|
|
estimatedCost: sortedOps.reduce((sum, op) => sum + op[1], 0),
|
||
|
|
steps: this.buildProgressiveSteps(sortedOps, query)
|
||
|
|
};
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Build progressive execution steps
|
||
|
|
*/
|
||
|
|
buildProgressiveSteps(sortedOps, query) {
|
||
|
|
const steps = [];
|
||
|
|
for (const [op, cost] of sortedOps) {
|
||
|
|
steps.push({
|
||
|
|
type: op,
|
||
|
|
operation: op === 'vector' ? 'search' : op === 'graph' ? 'traverse' : 'filter',
|
||
|
|
estimated: Math.round(cost)
|
||
|
|
});
|
||
|
|
}
|
||
|
|
// Add fusion step if multiple operations
|
||
|
|
if (steps.length > 1) {
|
||
|
|
steps.push({
|
||
|
|
type: 'fusion',
|
||
|
|
operation: 'rank',
|
||
|
|
estimated: Math.round(sortedOps.length * 50)
|
||
|
|
});
|
||
|
|
}
|
||
|
|
return steps;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Build parallel execution steps
|
||
|
|
*/
|
||
|
|
buildParallelSteps(sortedOps, query) {
|
||
|
|
const steps = sortedOps.map(([op, cost]) => ({
|
||
|
|
type: op,
|
||
|
|
operation: op === 'vector' ? 'search' : op === 'graph' ? 'traverse' : 'filter',
|
||
|
|
estimated: Math.round(cost)
|
||
|
|
}));
|
||
|
|
// Always add fusion for parallel execution
|
||
|
|
steps.push({
|
||
|
|
type: 'fusion',
|
||
|
|
operation: 'rank',
|
||
|
|
estimated: Math.round(sortedOps.length * 100)
|
||
|
|
});
|
||
|
|
return steps;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Execute searches in parallel for maximum speed
|
||
|
|
*/
|
||
|
|
async parallelSearch(query, plan) {
|
||
|
|
// Check for single-signal optimization
|
||
|
|
if (plan.steps.length === 1 && plan.steps[0].operation === 'direct') {
|
||
|
|
// Skip fusion for single signal queries
|
||
|
|
const results = await this.executeSingleSignal(query, plan.steps[0].type);
|
||
|
|
return results.map(r => ({
|
||
|
|
...r,
|
||
|
|
fusionScore: r.score || 1.0,
|
||
|
|
score: r.score || 1.0
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
const tasks = [];
|
||
|
|
// Vector search
|
||
|
|
if (query.like || query.similar) {
|
||
|
|
tasks.push(this.vectorSearch(query.like || query.similar, query.limit));
|
||
|
|
}
|
||
|
|
// Graph traversal
|
||
|
|
if (query.connected) {
|
||
|
|
tasks.push(this.graphTraversal(query.connected));
|
||
|
|
}
|
||
|
|
// Field filtering
|
||
|
|
if (query.where) {
|
||
|
|
tasks.push(this.fieldFilter(query.where));
|
||
|
|
}
|
||
|
|
// Run all in parallel
|
||
|
|
const results = await Promise.all(tasks);
|
||
|
|
// Fusion ranking combines all signals
|
||
|
|
return this.fusionRank(results, query);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Progressive filtering for efficiency
|
||
|
|
*/
|
||
|
|
async progressiveSearch(query, plan) {
|
||
|
|
let candidates = [];
|
||
|
|
for (const step of plan.steps) {
|
||
|
|
switch (step.type) {
|
||
|
|
case 'field':
|
||
|
|
if (candidates.length === 0) {
|
||
|
|
// Initial field filter
|
||
|
|
candidates = await this.fieldFilter(query.where);
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Filter existing candidates
|
||
|
|
candidates = this.applyFieldFilter(candidates, query.where);
|
||
|
|
}
|
||
|
|
break;
|
||
|
|
case 'vector':
|
||
|
|
// CRITICAL: If we have a previous step that returned 0 candidates,
|
||
|
|
// we must respect that and not do a fresh search
|
||
|
|
if (candidates.length === 0 && plan.steps[0].type === 'vector') {
|
||
|
|
// This is the first step - do initial vector search
|
||
|
|
const results = await this.vectorSearch(query.like || query.similar, query.limit);
|
||
|
|
candidates = results;
|
||
|
|
}
|
||
|
|
else if (candidates.length > 0) {
|
||
|
|
// Vector search within existing candidates
|
||
|
|
candidates = await this.vectorSearchWithin(query.like || query.similar, candidates);
|
||
|
|
}
|
||
|
|
// If candidates.length === 0 and this isn't the first step, keep empty candidates
|
||
|
|
break;
|
||
|
|
case 'graph':
|
||
|
|
// CRITICAL: Same logic as vector - respect empty candidates from previous steps
|
||
|
|
if (candidates.length === 0 && plan.steps[0].type === 'graph') {
|
||
|
|
// This is the first step - do initial graph traversal
|
||
|
|
candidates = await this.graphTraversal(query.connected);
|
||
|
|
}
|
||
|
|
else if (candidates.length > 0) {
|
||
|
|
// Graph expansion from existing candidates
|
||
|
|
candidates = await this.graphExpand(candidates);
|
||
|
|
}
|
||
|
|
// If candidates.length === 0 and this isn't the first step, keep empty candidates
|
||
|
|
break;
|
||
|
|
case 'fusion':
|
||
|
|
// Final fusion ranking
|
||
|
|
return this.fusionRank([candidates], query);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return candidates;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Vector similarity search
|
||
|
|
*/
|
||
|
|
async vectorSearch(query, limit) {
|
||
|
|
// Use clean internal vector search API to avoid circular dependency
|
||
|
|
// This is the proper architecture: find() uses internal methods, not public search()
|
||
|
|
return this.api.vectorSearch(query, limit || 100);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Graph traversal
|
||
|
|
*/
|
||
|
|
async graphTraversal(connected) {
|
||
|
|
// Get starting nodes
|
||
|
|
const startNodes = connected.from ?
|
||
|
|
(Array.isArray(connected.from) ? connected.from : [connected.from]) :
|
||
|
|
connected.to ?
|
||
|
|
(Array.isArray(connected.to) ? connected.to : [connected.to]) :
|
||
|
|
[];
|
||
|
|
// Use the API for graph traversal
|
||
|
|
const options = {
|
||
|
|
start: startNodes,
|
||
|
|
type: connected.type,
|
||
|
|
direction: connected.direction || 'both',
|
||
|
|
maxDepth: connected.maxDepth || connected.depth || 2
|
||
|
|
};
|
||
|
|
const results = await this.api.graphTraversal(options);
|
||
|
|
// Convert to expected format
|
||
|
|
return results.map(r => ({
|
||
|
|
id: r.id,
|
||
|
|
type: connected.type || 'relates_to',
|
||
|
|
score: r.score
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Field-based filtering using MetadataIndex for O(log n) performance
|
||
|
|
* NO FALLBACKS - Requires proper where clause and MetadataIndex
|
||
|
|
*/
|
||
|
|
async fieldFilter(where) {
|
||
|
|
// Require a valid where clause - no empty queries allowed
|
||
|
|
if (!where || Object.keys(where).length === 0) {
|
||
|
|
throw new Error('Field filter requires a where clause. ' +
|
||
|
|
'For retrieving all items, use a different query type or specify explicit criteria.');
|
||
|
|
}
|
||
|
|
// Verify MetadataIndex is available
|
||
|
|
if (!this.api.hasMetadataIndex || !this.api.hasMetadataIndex()) {
|
||
|
|
throw new Error('MetadataIndex not available - cannot perform O(log n) field queries. ' +
|
||
|
|
'Initialize Brainy with enableMetadataIndex: true');
|
||
|
|
}
|
||
|
|
// Use the MetadataIndex for O(log n) performance
|
||
|
|
// This uses B-tree indexes for range queries and hash indexes for exact matches
|
||
|
|
const startTime = performance.now();
|
||
|
|
const matchingIds = await this.api.metadataQuery(where);
|
||
|
|
// Verify we got results from the fast path
|
||
|
|
if (!matchingIds) {
|
||
|
|
throw new Error('MetadataIndex query failed - no fallback allowed');
|
||
|
|
}
|
||
|
|
// Track performance metrics
|
||
|
|
const queryTime = performance.now() - startTime;
|
||
|
|
const expectedTime = Math.log2(1000000) * 5; // Assume max 1M items, 5ms per log operation
|
||
|
|
if (queryTime > expectedTime * 2) {
|
||
|
|
console.warn(`Field filter performance warning: ${queryTime.toFixed(2)}ms > expected ${expectedTime.toFixed(2)}ms`);
|
||
|
|
}
|
||
|
|
// Convert matching IDs to result format with full entities
|
||
|
|
const results = [];
|
||
|
|
const idsArray = Array.from(matchingIds);
|
||
|
|
// Process results in batches for efficiency
|
||
|
|
const batchSize = 100;
|
||
|
|
for (let i = 0; i < Math.min(idsArray.length, 1000); i += batchSize) {
|
||
|
|
const batch = idsArray.slice(i, i + batchSize);
|
||
|
|
const entities = await Promise.all(batch.map(id => this.api.getEntity(id)));
|
||
|
|
for (let j = 0; j < entities.length; j++) {
|
||
|
|
const entity = entities[j];
|
||
|
|
if (entity) {
|
||
|
|
results.push({
|
||
|
|
id: batch[j],
|
||
|
|
score: 1.0, // Field matches are binary
|
||
|
|
entity,
|
||
|
|
metadata: entity.metadata || {}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return results;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Execute a single signal query directly
|
||
|
|
*/
|
||
|
|
async executeSingleSignal(query, signalType) {
|
||
|
|
switch (signalType) {
|
||
|
|
case 'vector':
|
||
|
|
return this.vectorSearch(query.like || query.similar, query.limit);
|
||
|
|
case 'graph':
|
||
|
|
return this.graphTraversal(query.connected);
|
||
|
|
case 'field':
|
||
|
|
return this.fieldFilter(query.where);
|
||
|
|
default:
|
||
|
|
throw new Error(`Unknown signal type: ${signalType}`);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Expand graph connections from existing candidates
|
||
|
|
*/
|
||
|
|
async graphExpand(candidates) {
|
||
|
|
const expanded = [];
|
||
|
|
const visited = new Set();
|
||
|
|
// For each candidate, find its graph neighbors
|
||
|
|
for (const candidate of candidates) {
|
||
|
|
const id = candidate.id || candidate;
|
||
|
|
if (visited.has(id))
|
||
|
|
continue;
|
||
|
|
visited.add(id);
|
||
|
|
// Get connections for this node
|
||
|
|
const [sourceVerbs, targetVerbs] = await Promise.all([
|
||
|
|
this.api.getVerbsBySource(id),
|
||
|
|
this.api.getVerbsByTarget(id)
|
||
|
|
]);
|
||
|
|
// Add the original candidate
|
||
|
|
expanded.push(candidate);
|
||
|
|
// Add connected nodes
|
||
|
|
for (const verb of sourceVerbs) {
|
||
|
|
if (!visited.has(verb.targetId)) {
|
||
|
|
const entity = await this.api.getEntity(verb.targetId);
|
||
|
|
if (entity) {
|
||
|
|
expanded.push({
|
||
|
|
id: verb.targetId,
|
||
|
|
score: (candidate.score || 1.0) * 0.8, // Decay score by distance
|
||
|
|
entity,
|
||
|
|
metadata: entity.metadata
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
for (const verb of targetVerbs) {
|
||
|
|
if (!visited.has(verb.sourceId)) {
|
||
|
|
const entity = await this.api.getEntity(verb.sourceId);
|
||
|
|
if (entity) {
|
||
|
|
expanded.push({
|
||
|
|
id: verb.sourceId,
|
||
|
|
score: (candidate.score || 1.0) * 0.8, // Decay score by distance
|
||
|
|
entity,
|
||
|
|
metadata: entity.metadata
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return expanded;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Vector search within existing candidates
|
||
|
|
*/
|
||
|
|
async vectorSearchWithin(query, candidates) {
|
||
|
|
// Get the query vector
|
||
|
|
const queryVector = typeof query === 'string' ?
|
||
|
|
(await this.api.vectorSearch(query, 1))[0]?.entity?.vector :
|
||
|
|
query;
|
||
|
|
if (!queryVector)
|
||
|
|
return candidates;
|
||
|
|
// Score each candidate by vector similarity
|
||
|
|
const scored = [];
|
||
|
|
for (const candidate of candidates) {
|
||
|
|
const entity = candidate.entity || await this.api.getEntity(candidate.id);
|
||
|
|
if (entity && entity.vector) {
|
||
|
|
const similarity = this.cosineSimilarity(queryVector, entity.vector);
|
||
|
|
scored.push({
|
||
|
|
...candidate,
|
||
|
|
score: similarity,
|
||
|
|
entity
|
||
|
|
});
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Sort by similarity and return
|
||
|
|
return scored.sort((a, b) => b.score - a.score);
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Apply field filter to existing candidates
|
||
|
|
*/
|
||
|
|
applyFieldFilter(candidates, where) {
|
||
|
|
return candidates.filter(candidate => {
|
||
|
|
const metadata = candidate.metadata || candidate.entity?.metadata || {};
|
||
|
|
return this.matchesFilter(metadata, where);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Check if metadata matches filter conditions
|
||
|
|
*/
|
||
|
|
matchesFilter(metadata, where) {
|
||
|
|
for (const [field, condition] of Object.entries(where)) {
|
||
|
|
const value = metadata[field];
|
||
|
|
if (typeof condition === 'object' && condition !== null) {
|
||
|
|
// Handle operators
|
||
|
|
if ('$eq' in condition && value !== condition.$eq)
|
||
|
|
return false;
|
||
|
|
if ('$ne' in condition && value === condition.$ne)
|
||
|
|
return false;
|
||
|
|
if ('$gt' in condition && !(value > condition.$gt))
|
||
|
|
return false;
|
||
|
|
if ('$gte' in condition && !(value >= condition.$gte))
|
||
|
|
return false;
|
||
|
|
if ('$lt' in condition && !(value < condition.$lt))
|
||
|
|
return false;
|
||
|
|
if ('$lte' in condition && !(value <= condition.$lte))
|
||
|
|
return false;
|
||
|
|
if ('$in' in condition && !condition.$in.includes(value))
|
||
|
|
return false;
|
||
|
|
if ('$nin' in condition && condition.$nin.includes(value))
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
// Direct equality
|
||
|
|
if (value !== condition)
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Calculate cosine similarity between two vectors
|
||
|
|
*/
|
||
|
|
cosineSimilarity(a, b) {
|
||
|
|
let dotProduct = 0;
|
||
|
|
let normA = 0;
|
||
|
|
let normB = 0;
|
||
|
|
for (let i = 0; i < a.length; i++) {
|
||
|
|
dotProduct += a[i] * b[i];
|
||
|
|
normA += a[i] * a[i];
|
||
|
|
normB += b[i] * b[i];
|
||
|
|
}
|
||
|
|
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Fusion ranking using Reciprocal Rank Fusion (RRF)
|
||
|
|
* This is the same algorithm used by Google and Elasticsearch
|
||
|
|
*/
|
||
|
|
fusionRank(resultSets, query) {
|
||
|
|
// RRF constant - 60 is empirically proven optimal
|
||
|
|
const k = 60;
|
||
|
|
// Calculate dynamic weights based on query
|
||
|
|
const weights = this.calculateSignalWeights(query);
|
||
|
|
// Determine which result sets we have based on query
|
||
|
|
let vectorResultsIdx = -1;
|
||
|
|
let graphResultsIdx = -1;
|
||
|
|
let metadataResultsIdx = -1;
|
||
|
|
let currentIdx = 0;
|
||
|
|
if (query.like || query.similar) {
|
||
|
|
vectorResultsIdx = currentIdx++;
|
||
|
|
}
|
||
|
|
if (query.where) {
|
||
|
|
metadataResultsIdx = currentIdx++;
|
||
|
|
}
|
||
|
|
// If we have metadata filters AND other searches, apply intersection
|
||
|
|
if (metadataResultsIdx >= 0 && resultSets.length > 1) {
|
||
|
|
const metadataResults = resultSets[metadataResultsIdx];
|
||
|
|
// CRITICAL: If metadata filter returned no results, entire query should return empty
|
||
|
|
if (metadataResults.length === 0) {
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
const metadataIds = new Set(metadataResults.map(r => r.id || r));
|
||
|
|
// Filter ALL other result sets to only include items that match metadata
|
||
|
|
for (let i = 0; i < resultSets.length; i++) {
|
||
|
|
if (i !== metadataResultsIdx) {
|
||
|
|
resultSets[i] = resultSets[i].filter(r => metadataIds.has(r.id || r));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Build fusion scores using RRF
|
||
|
|
const fusionScores = new Map();
|
||
|
|
// Process each result set with RRF
|
||
|
|
resultSets.forEach((results, setIndex) => {
|
||
|
|
// Determine signal type
|
||
|
|
let signalType;
|
||
|
|
let weight = 1.0;
|
||
|
|
if (setIndex === vectorResultsIdx) {
|
||
|
|
signalType = 'vector';
|
||
|
|
weight = weights.vector;
|
||
|
|
}
|
||
|
|
else if (setIndex === graphResultsIdx) {
|
||
|
|
signalType = 'graph';
|
||
|
|
weight = weights.graph;
|
||
|
|
}
|
||
|
|
else if (setIndex === metadataResultsIdx) {
|
||
|
|
signalType = 'field';
|
||
|
|
weight = weights.field;
|
||
|
|
}
|
||
|
|
else {
|
||
|
|
return; // Skip unknown signal types
|
||
|
|
}
|
||
|
|
// Apply RRF to each result
|
||
|
|
results.forEach((result, rank) => {
|
||
|
|
const id = result.id || result;
|
||
|
|
// Calculate RRF score: 1 / (k + rank + 1)
|
||
|
|
const rrfScore = weight * (1.0 / (k + rank + 1));
|
||
|
|
if (!fusionScores.has(id)) {
|
||
|
|
fusionScores.set(id, {
|
||
|
|
entity: result.entity || result,
|
||
|
|
vectorScore: 0,
|
||
|
|
graphScore: 0,
|
||
|
|
fieldScore: 0,
|
||
|
|
rrfScore: 0,
|
||
|
|
fusionScore: 0,
|
||
|
|
metadata: result.metadata || {}
|
||
|
|
});
|
||
|
|
}
|
||
|
|
const fusion = fusionScores.get(id);
|
||
|
|
// Track individual signal scores
|
||
|
|
if (signalType === 'vector') {
|
||
|
|
fusion.vectorScore = result.score || 1.0;
|
||
|
|
}
|
||
|
|
else if (signalType === 'graph') {
|
||
|
|
fusion.graphScore = result.score || 1.0;
|
||
|
|
}
|
||
|
|
else if (signalType === 'field') {
|
||
|
|
fusion.fieldScore = result.score || 1.0;
|
||
|
|
}
|
||
|
|
// Accumulate RRF score
|
||
|
|
fusion.rrfScore += rrfScore;
|
||
|
|
});
|
||
|
|
});
|
||
|
|
// Convert to results array
|
||
|
|
const results = Array.from(fusionScores.entries()).map(([id, fusion]) => ({
|
||
|
|
id,
|
||
|
|
entity: fusion.entity,
|
||
|
|
score: fusion.rrfScore, // Use RRF score as primary score
|
||
|
|
vector: fusion.entity?.vector || new Float32Array(0), // Include vector for SearchResult compatibility
|
||
|
|
vectorScore: fusion.vectorScore,
|
||
|
|
graphScore: fusion.graphScore,
|
||
|
|
fieldScore: fusion.fieldScore,
|
||
|
|
fusionScore: fusion.rrfScore, // RRF score is the fusion score
|
||
|
|
metadata: fusion.metadata
|
||
|
|
}));
|
||
|
|
// Sort by fusion score (descending)
|
||
|
|
results.sort((a, b) => b.fusionScore - a.fusionScore);
|
||
|
|
// Apply offset and limit
|
||
|
|
let final = results;
|
||
|
|
if (query.offset && query.offset > 0) {
|
||
|
|
final = final.slice(query.offset);
|
||
|
|
}
|
||
|
|
if (query.limit) {
|
||
|
|
final = final.slice(0, query.limit);
|
||
|
|
}
|
||
|
|
return final;
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Calculate dynamic signal weights based on query characteristics
|
||
|
|
*/
|
||
|
|
calculateSignalWeights(query) {
|
||
|
|
const hasVector = !!(query.like || query.similar);
|
||
|
|
const hasGraph = !!(query.connected);
|
||
|
|
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||
|
|
// Count active signals
|
||
|
|
const activeSignals = [hasVector, hasGraph, hasField].filter(Boolean).length;
|
||
|
|
if (activeSignals === 1) {
|
||
|
|
// Single signal - full weight
|
||
|
|
return {
|
||
|
|
vector: hasVector ? 1.0 : 0,
|
||
|
|
graph: hasGraph ? 1.0 : 0,
|
||
|
|
field: hasField ? 1.0 : 0
|
||
|
|
};
|
||
|
|
}
|
||
|
|
// Multiple signals - adaptive weights
|
||
|
|
if (hasVector && hasGraph && hasField) {
|
||
|
|
// All three signals - balanced with slight vector preference
|
||
|
|
return {
|
||
|
|
vector: 0.4, // Semantic search is often most relevant
|
||
|
|
graph: 0.35, // Relationships are important
|
||
|
|
field: 0.25 // Metadata is supportive
|
||
|
|
};
|
||
|
|
}
|
||
|
|
else if (hasVector && hasGraph) {
|
||
|
|
// Vector + Graph - emphasize semantics
|
||
|
|
return {
|
||
|
|
vector: 0.6,
|
||
|
|
graph: 0.4,
|
||
|
|
field: 0
|
||
|
|
};
|
||
|
|
}
|
||
|
|
else if (hasVector && hasField) {
|
||
|
|
// Vector + Field - balanced
|
||
|
|
return {
|
||
|
|
vector: 0.5,
|
||
|
|
graph: 0,
|
||
|
|
field: 0.5
|
||
|
|
};
|
||
|
|
}
|
||
|
|
else if (hasGraph && hasField) {
|
||
|
|
// Graph + Field - emphasize relationships
|
||
|
|
return {
|
||
|
|
vector: 0,
|
||
|
|
graph: 0.6,
|
||
|
|
field: 0.4
|
||
|
|
};
|
||
|
|
}
|
||
|
|
// Default balanced weights
|
||
|
|
return {
|
||
|
|
vector: 0.34,
|
||
|
|
graph: 0.33,
|
||
|
|
field: 0.33
|
||
|
|
};
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Apply boost strategies
|
||
|
|
*/
|
||
|
|
applyBoosts(results, boost) {
|
||
|
|
return results.map(r => {
|
||
|
|
let boostFactor = 1.0;
|
||
|
|
switch (boost) {
|
||
|
|
case 'recent':
|
||
|
|
// Boost recent items
|
||
|
|
const age = Date.now() - (r.metadata?.timestamp || 0);
|
||
|
|
boostFactor = Math.exp(-age / (30 * 24 * 60 * 60 * 1000)); // 30-day half-life
|
||
|
|
break;
|
||
|
|
case 'popular':
|
||
|
|
// Boost by view count or connections
|
||
|
|
boostFactor = Math.log10((r.metadata?.views || 0) + 10) / 2;
|
||
|
|
break;
|
||
|
|
case 'verified':
|
||
|
|
// Boost verified content
|
||
|
|
boostFactor = r.metadata?.verified ? 1.5 : 1.0;
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
...r,
|
||
|
|
fusionScore: r.fusionScore * boostFactor
|
||
|
|
};
|
||
|
|
});
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Add query explanations for debugging
|
||
|
|
*/
|
||
|
|
addExplanations(results, plan, totalTime) {
|
||
|
|
return results.map(r => ({
|
||
|
|
...r,
|
||
|
|
explanation: {
|
||
|
|
plan: plan.steps.map(s => `${s.type}:${s.operation}`).join(' → '),
|
||
|
|
timing: {
|
||
|
|
total: totalTime,
|
||
|
|
...plan.steps.reduce((acc, step) => ({
|
||
|
|
...acc,
|
||
|
|
[step.type]: step.estimated
|
||
|
|
}), {})
|
||
|
|
},
|
||
|
|
boosts: []
|
||
|
|
}
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
// Query learning removed - unnecessary complexity
|
||
|
|
/**
|
||
|
|
* Optimize plan based on historical patterns
|
||
|
|
*/
|
||
|
|
// Query optimization from history removed
|
||
|
|
/**
|
||
|
|
* Clear query optimization cache
|
||
|
|
*/
|
||
|
|
clearCache() {
|
||
|
|
this.planCache.clear();
|
||
|
|
}
|
||
|
|
/**
|
||
|
|
* Get optimization statistics
|
||
|
|
*/
|
||
|
|
getStats() {
|
||
|
|
return {
|
||
|
|
cachedPlans: this.planCache.size,
|
||
|
|
historySize: 0 // Query history removed
|
||
|
|
};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
// Export a beautiful, simple API
|
||
|
|
export async function find(brain, query) {
|
||
|
|
const engine = new TripleIntelligenceEngine(brain);
|
||
|
|
return engine.find(query);
|
||
|
|
}
|
||
|
|
//# sourceMappingURL=TripleIntelligence.js.map
|