chore: recovery checkpoint - v3.0 API successfully recovered
CRITICAL CHECKPOINT - DO NOT PUSH TO GITHUB Recovery Status: - Successfully recovered brainy.ts from compiled JavaScript - All core v3.0 API methods functional (add, get, update, delete, relate, find, etc.) - Neural subsystem intact (562KB embedded patterns, NLP working) - Augmentation pipeline operational (20+ augmentations) - HNSW clustering system complete - Triple Intelligence compiled (needs constructor fix) - Test suite validates functionality Changes preserved: - 898 files with changes from last 3 days - 144,475 insertions - All augmentation improvements - All test coverage enhancements - Complete v3.0 feature set This is a LOCAL checkpoint only - contains recovered work after corruption incident. Created backup in .backups/brainy-full-20250910-151314.tar.gz Branch: recovery-checkpoint-20250910-151433 Date: Wed Sep 10 03:18:04 PM PDT 2025
This commit is contained in:
parent
f65455fb22
commit
8ff382ca3b
895 changed files with 143654 additions and 28268 deletions
|
|
@ -0,0 +1,814 @@
|
|||
/**
|
||||
* Triple Intelligence Engine
|
||||
* Revolutionary unified search combining Vector + Graph + Field intelligence
|
||||
*
|
||||
* This is Brainy's killer feature - no other database can do this!
|
||||
*/
|
||||
/**
|
||||
* The Triple Intelligence Engine
|
||||
* Unifies vector, graph, and field search into one beautiful API
|
||||
*/
|
||||
export class TripleIntelligenceEngine {
|
||||
constructor(brain) {
|
||||
this.planCache = new Map();
|
||||
this.brain = brain;
|
||||
this.api = brain.getTripleIntelligenceAPI();
|
||||
// Query history removed - unnecessary complexity for minimal gain
|
||||
}
|
||||
/**
|
||||
* The magic happens here - one query to rule them all
|
||||
*/
|
||||
async find(query) {
|
||||
const startTime = Date.now();
|
||||
// Generate optimal query plan
|
||||
const plan = await this.optimizeQuery(query);
|
||||
// Execute based on plan
|
||||
let results;
|
||||
if (plan.canParallelize) {
|
||||
// Run all three paths in parallel for maximum speed
|
||||
results = await this.parallelSearch(query, plan);
|
||||
}
|
||||
else {
|
||||
// Progressive filtering for efficiency
|
||||
results = await this.progressiveSearch(query, plan);
|
||||
}
|
||||
// Apply boosts if requested
|
||||
if (query.boost) {
|
||||
results = this.applyBoosts(results, query.boost);
|
||||
}
|
||||
// Add explanations if requested
|
||||
if (query.explain) {
|
||||
const timing = Date.now() - startTime;
|
||||
results = this.addExplanations(results, plan, timing);
|
||||
}
|
||||
// Query history removed - no learning needed
|
||||
// Apply limit
|
||||
if (query.limit) {
|
||||
results = results.slice(0, query.limit);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Generate optimal execution plan based on query shape and statistics
|
||||
*/
|
||||
async optimizeQuery(query) {
|
||||
// Short-circuit optimization for single-signal queries
|
||||
const hasVector = !!(query.like || query.similar);
|
||||
const hasGraph = !!(query.connected);
|
||||
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||||
const signalCount = [hasVector, hasGraph, hasField].filter(Boolean).length;
|
||||
// Single signal - skip fusion entirely!
|
||||
if (signalCount === 1) {
|
||||
const singleType = hasVector ? 'vector' : hasGraph ? 'graph' : 'field';
|
||||
return {
|
||||
startWith: singleType,
|
||||
canParallelize: false,
|
||||
estimatedCost: 1,
|
||||
steps: [{
|
||||
type: singleType,
|
||||
operation: 'direct', // Direct execution, no fusion
|
||||
estimated: 50
|
||||
}]
|
||||
};
|
||||
}
|
||||
// Check cache first
|
||||
const cacheKey = JSON.stringify(query);
|
||||
if (this.planCache.has(cacheKey)) {
|
||||
return this.planCache.get(cacheKey);
|
||||
}
|
||||
// Get real statistics for cost-based optimization
|
||||
const stats = await this.api.getStatistics();
|
||||
// Calculate costs for each operation
|
||||
const costs = await this.calculateOperationCosts(query, stats);
|
||||
// Build optimal plan based on actual costs
|
||||
const plan = this.buildOptimalPlan(query, costs, stats);
|
||||
this.planCache.set(cacheKey, plan);
|
||||
return plan;
|
||||
}
|
||||
/**
|
||||
* Calculate real costs for each operation based on statistics
|
||||
*/
|
||||
async calculateOperationCosts(query, stats) {
|
||||
const costs = {
|
||||
vector: Infinity,
|
||||
graph: Infinity,
|
||||
field: Infinity
|
||||
};
|
||||
// Vector search cost - O(log n) with HNSW
|
||||
if (query.like || query.similar) {
|
||||
// HNSW search complexity: O(log n) * ef
|
||||
const ef = 200; // exploration factor
|
||||
costs.vector = Math.log2(stats.totalCount) * ef;
|
||||
}
|
||||
// Graph traversal cost - depends on connectivity
|
||||
if (query.connected) {
|
||||
const depth = query.connected.maxDepth || query.connected.depth || 2;
|
||||
// Assume average branching factor of 10
|
||||
const branchingFactor = 10;
|
||||
costs.graph = Math.pow(branchingFactor, depth);
|
||||
}
|
||||
// Field filter cost - depends on selectivity
|
||||
if (query.where) {
|
||||
const selectivity = await this.estimateFieldSelectivity(query.where, stats);
|
||||
costs.field = stats.totalCount * selectivity;
|
||||
// If we have an index, cost is O(log n)
|
||||
if (this.api.hasMetadataIndex()) {
|
||||
costs.field = Math.log2(stats.totalCount) + costs.field;
|
||||
}
|
||||
}
|
||||
return costs;
|
||||
}
|
||||
/**
|
||||
* Estimate selectivity of field filters
|
||||
*/
|
||||
async estimateFieldSelectivity(where, stats) {
|
||||
let selectivity = 1.0;
|
||||
for (const [field, condition] of Object.entries(where)) {
|
||||
const fieldStats = stats.fieldStats[field];
|
||||
if (!fieldStats) {
|
||||
// Unknown field - assume 10% selectivity
|
||||
selectivity *= 0.1;
|
||||
continue;
|
||||
}
|
||||
if (typeof condition === 'object' && condition !== null) {
|
||||
// Handle operators
|
||||
if ('$eq' in condition) {
|
||||
// Equality - 1/cardinality
|
||||
selectivity *= 1.0 / (fieldStats.cardinality || 100);
|
||||
}
|
||||
else if ('$gt' in condition || '$gte' in condition) {
|
||||
// Range query - estimate based on distribution
|
||||
const threshold = condition.$gt || condition.$gte;
|
||||
if (typeof threshold === 'number' && fieldStats.type === 'number') {
|
||||
const range = fieldStats.max - fieldStats.min;
|
||||
const remainingRange = fieldStats.max - threshold;
|
||||
selectivity *= remainingRange / range;
|
||||
}
|
||||
else {
|
||||
selectivity *= 0.3; // Default for non-numeric
|
||||
}
|
||||
}
|
||||
else if ('$lt' in condition || '$lte' in condition) {
|
||||
// Range query - estimate based on distribution
|
||||
const threshold = condition.$lt || condition.$lte;
|
||||
if (typeof threshold === 'number' && fieldStats.type === 'number') {
|
||||
const range = fieldStats.max - fieldStats.min;
|
||||
const remainingRange = threshold - fieldStats.min;
|
||||
selectivity *= remainingRange / range;
|
||||
}
|
||||
else {
|
||||
selectivity *= 0.3; // Default for non-numeric
|
||||
}
|
||||
}
|
||||
else if ('$in' in condition) {
|
||||
// IN query
|
||||
selectivity *= condition.$in.length / (fieldStats.cardinality || 100);
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Direct equality
|
||||
selectivity *= 1.0 / (fieldStats.cardinality || 100);
|
||||
}
|
||||
}
|
||||
return Math.max(0.0001, Math.min(1.0, selectivity));
|
||||
}
|
||||
/**
|
||||
* Build optimal execution plan based on costs
|
||||
*/
|
||||
buildOptimalPlan(query, costs, stats) {
|
||||
const hasVector = !!(query.like || query.similar);
|
||||
const hasGraph = !!(query.connected);
|
||||
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||||
// Find the most selective operation
|
||||
const sortedOps = Object.entries(costs)
|
||||
.filter(([op]) => {
|
||||
return (op === 'vector' && hasVector) ||
|
||||
(op === 'graph' && hasGraph) ||
|
||||
(op === 'field' && hasField);
|
||||
})
|
||||
.sort((a, b) => a[1] - b[1]);
|
||||
// If the most selective operation filters out > 99%, start with it
|
||||
const mostSelective = sortedOps[0];
|
||||
if (mostSelective && mostSelective[1] < stats.totalCount * 0.01) {
|
||||
// Progressive plan - start with most selective
|
||||
return {
|
||||
startWith: mostSelective[0],
|
||||
canParallelize: false,
|
||||
estimatedCost: mostSelective[1],
|
||||
steps: this.buildProgressiveSteps(sortedOps, query)
|
||||
};
|
||||
}
|
||||
// If operations have similar costs, parallelize
|
||||
if (sortedOps.length > 1) {
|
||||
const ratio = sortedOps[1][1] / sortedOps[0][1];
|
||||
if (ratio < 10) {
|
||||
// Costs are within 10x - parallelize
|
||||
return {
|
||||
startWith: sortedOps[0][0],
|
||||
canParallelize: true,
|
||||
estimatedCost: Math.max(...sortedOps.map(op => op[1])),
|
||||
steps: this.buildParallelSteps(sortedOps, query)
|
||||
};
|
||||
}
|
||||
}
|
||||
// Default progressive plan
|
||||
return {
|
||||
startWith: sortedOps[0][0],
|
||||
canParallelize: false,
|
||||
estimatedCost: sortedOps.reduce((sum, op) => sum + op[1], 0),
|
||||
steps: this.buildProgressiveSteps(sortedOps, query)
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Build progressive execution steps
|
||||
*/
|
||||
buildProgressiveSteps(sortedOps, query) {
|
||||
const steps = [];
|
||||
for (const [op, cost] of sortedOps) {
|
||||
steps.push({
|
||||
type: op,
|
||||
operation: op === 'vector' ? 'search' : op === 'graph' ? 'traverse' : 'filter',
|
||||
estimated: Math.round(cost)
|
||||
});
|
||||
}
|
||||
// Add fusion step if multiple operations
|
||||
if (steps.length > 1) {
|
||||
steps.push({
|
||||
type: 'fusion',
|
||||
operation: 'rank',
|
||||
estimated: Math.round(sortedOps.length * 50)
|
||||
});
|
||||
}
|
||||
return steps;
|
||||
}
|
||||
/**
|
||||
* Build parallel execution steps
|
||||
*/
|
||||
buildParallelSteps(sortedOps, query) {
|
||||
const steps = sortedOps.map(([op, cost]) => ({
|
||||
type: op,
|
||||
operation: op === 'vector' ? 'search' : op === 'graph' ? 'traverse' : 'filter',
|
||||
estimated: Math.round(cost)
|
||||
}));
|
||||
// Always add fusion for parallel execution
|
||||
steps.push({
|
||||
type: 'fusion',
|
||||
operation: 'rank',
|
||||
estimated: Math.round(sortedOps.length * 100)
|
||||
});
|
||||
return steps;
|
||||
}
|
||||
/**
|
||||
* Execute searches in parallel for maximum speed
|
||||
*/
|
||||
async parallelSearch(query, plan) {
|
||||
// Check for single-signal optimization
|
||||
if (plan.steps.length === 1 && plan.steps[0].operation === 'direct') {
|
||||
// Skip fusion for single signal queries
|
||||
const results = await this.executeSingleSignal(query, plan.steps[0].type);
|
||||
return results.map(r => ({
|
||||
...r,
|
||||
fusionScore: r.score || 1.0,
|
||||
score: r.score || 1.0
|
||||
}));
|
||||
}
|
||||
const tasks = [];
|
||||
// Vector search
|
||||
if (query.like || query.similar) {
|
||||
tasks.push(this.vectorSearch(query.like || query.similar, query.limit));
|
||||
}
|
||||
// Graph traversal
|
||||
if (query.connected) {
|
||||
tasks.push(this.graphTraversal(query.connected));
|
||||
}
|
||||
// Field filtering
|
||||
if (query.where) {
|
||||
tasks.push(this.fieldFilter(query.where));
|
||||
}
|
||||
// Run all in parallel
|
||||
const results = await Promise.all(tasks);
|
||||
// Fusion ranking combines all signals
|
||||
return this.fusionRank(results, query);
|
||||
}
|
||||
/**
|
||||
* Progressive filtering for efficiency
|
||||
*/
|
||||
async progressiveSearch(query, plan) {
|
||||
let candidates = [];
|
||||
for (const step of plan.steps) {
|
||||
switch (step.type) {
|
||||
case 'field':
|
||||
if (candidates.length === 0) {
|
||||
// Initial field filter
|
||||
candidates = await this.fieldFilter(query.where);
|
||||
}
|
||||
else {
|
||||
// Filter existing candidates
|
||||
candidates = this.applyFieldFilter(candidates, query.where);
|
||||
}
|
||||
break;
|
||||
case 'vector':
|
||||
// CRITICAL: If we have a previous step that returned 0 candidates,
|
||||
// we must respect that and not do a fresh search
|
||||
if (candidates.length === 0 && plan.steps[0].type === 'vector') {
|
||||
// This is the first step - do initial vector search
|
||||
const results = await this.vectorSearch(query.like || query.similar, query.limit);
|
||||
candidates = results;
|
||||
}
|
||||
else if (candidates.length > 0) {
|
||||
// Vector search within existing candidates
|
||||
candidates = await this.vectorSearchWithin(query.like || query.similar, candidates);
|
||||
}
|
||||
// If candidates.length === 0 and this isn't the first step, keep empty candidates
|
||||
break;
|
||||
case 'graph':
|
||||
// CRITICAL: Same logic as vector - respect empty candidates from previous steps
|
||||
if (candidates.length === 0 && plan.steps[0].type === 'graph') {
|
||||
// This is the first step - do initial graph traversal
|
||||
candidates = await this.graphTraversal(query.connected);
|
||||
}
|
||||
else if (candidates.length > 0) {
|
||||
// Graph expansion from existing candidates
|
||||
candidates = await this.graphExpand(candidates);
|
||||
}
|
||||
// If candidates.length === 0 and this isn't the first step, keep empty candidates
|
||||
break;
|
||||
case 'fusion':
|
||||
// Final fusion ranking
|
||||
return this.fusionRank([candidates], query);
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
/**
|
||||
* Vector similarity search
|
||||
*/
|
||||
async vectorSearch(query, limit) {
|
||||
// Use clean internal vector search API to avoid circular dependency
|
||||
// This is the proper architecture: find() uses internal methods, not public search()
|
||||
return this.api.vectorSearch(query, limit || 100);
|
||||
}
|
||||
/**
|
||||
* Graph traversal
|
||||
*/
|
||||
async graphTraversal(connected) {
|
||||
// Get starting nodes
|
||||
const startNodes = connected.from ?
|
||||
(Array.isArray(connected.from) ? connected.from : [connected.from]) :
|
||||
connected.to ?
|
||||
(Array.isArray(connected.to) ? connected.to : [connected.to]) :
|
||||
[];
|
||||
// Use the API for graph traversal
|
||||
const options = {
|
||||
start: startNodes,
|
||||
type: connected.type,
|
||||
direction: connected.direction || 'both',
|
||||
maxDepth: connected.maxDepth || connected.depth || 2
|
||||
};
|
||||
const results = await this.api.graphTraversal(options);
|
||||
// Convert to expected format
|
||||
return results.map(r => ({
|
||||
id: r.id,
|
||||
type: connected.type || 'relates_to',
|
||||
score: r.score
|
||||
}));
|
||||
}
|
||||
/**
|
||||
* Field-based filtering using MetadataIndex for O(log n) performance
|
||||
* NO FALLBACKS - Requires proper where clause and MetadataIndex
|
||||
*/
|
||||
async fieldFilter(where) {
|
||||
// Require a valid where clause - no empty queries allowed
|
||||
if (!where || Object.keys(where).length === 0) {
|
||||
throw new Error('Field filter requires a where clause. ' +
|
||||
'For retrieving all items, use a different query type or specify explicit criteria.');
|
||||
}
|
||||
// Verify MetadataIndex is available
|
||||
if (!this.api.hasMetadataIndex || !this.api.hasMetadataIndex()) {
|
||||
throw new Error('MetadataIndex not available - cannot perform O(log n) field queries. ' +
|
||||
'Initialize Brainy with enableMetadataIndex: true');
|
||||
}
|
||||
// Use the MetadataIndex for O(log n) performance
|
||||
// This uses B-tree indexes for range queries and hash indexes for exact matches
|
||||
const startTime = performance.now();
|
||||
const matchingIds = await this.api.metadataQuery(where);
|
||||
// Verify we got results from the fast path
|
||||
if (!matchingIds) {
|
||||
throw new Error('MetadataIndex query failed - no fallback allowed');
|
||||
}
|
||||
// Track performance metrics
|
||||
const queryTime = performance.now() - startTime;
|
||||
const expectedTime = Math.log2(1000000) * 5; // Assume max 1M items, 5ms per log operation
|
||||
if (queryTime > expectedTime * 2) {
|
||||
console.warn(`Field filter performance warning: ${queryTime.toFixed(2)}ms > expected ${expectedTime.toFixed(2)}ms`);
|
||||
}
|
||||
// Convert matching IDs to result format with full entities
|
||||
const results = [];
|
||||
const idsArray = Array.from(matchingIds);
|
||||
// Process results in batches for efficiency
|
||||
const batchSize = 100;
|
||||
for (let i = 0; i < Math.min(idsArray.length, 1000); i += batchSize) {
|
||||
const batch = idsArray.slice(i, i + batchSize);
|
||||
const entities = await Promise.all(batch.map(id => this.api.getEntity(id)));
|
||||
for (let j = 0; j < entities.length; j++) {
|
||||
const entity = entities[j];
|
||||
if (entity) {
|
||||
results.push({
|
||||
id: batch[j],
|
||||
score: 1.0, // Field matches are binary
|
||||
entity,
|
||||
metadata: entity.metadata || {}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
/**
|
||||
* Execute a single signal query directly
|
||||
*/
|
||||
async executeSingleSignal(query, signalType) {
|
||||
switch (signalType) {
|
||||
case 'vector':
|
||||
return this.vectorSearch(query.like || query.similar, query.limit);
|
||||
case 'graph':
|
||||
return this.graphTraversal(query.connected);
|
||||
case 'field':
|
||||
return this.fieldFilter(query.where);
|
||||
default:
|
||||
throw new Error(`Unknown signal type: ${signalType}`);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Expand graph connections from existing candidates
|
||||
*/
|
||||
async graphExpand(candidates) {
|
||||
const expanded = [];
|
||||
const visited = new Set();
|
||||
// For each candidate, find its graph neighbors
|
||||
for (const candidate of candidates) {
|
||||
const id = candidate.id || candidate;
|
||||
if (visited.has(id))
|
||||
continue;
|
||||
visited.add(id);
|
||||
// Get connections for this node
|
||||
const [sourceVerbs, targetVerbs] = await Promise.all([
|
||||
this.api.getVerbsBySource(id),
|
||||
this.api.getVerbsByTarget(id)
|
||||
]);
|
||||
// Add the original candidate
|
||||
expanded.push(candidate);
|
||||
// Add connected nodes
|
||||
for (const verb of sourceVerbs) {
|
||||
if (!visited.has(verb.targetId)) {
|
||||
const entity = await this.api.getEntity(verb.targetId);
|
||||
if (entity) {
|
||||
expanded.push({
|
||||
id: verb.targetId,
|
||||
score: (candidate.score || 1.0) * 0.8, // Decay score by distance
|
||||
entity,
|
||||
metadata: entity.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const verb of targetVerbs) {
|
||||
if (!visited.has(verb.sourceId)) {
|
||||
const entity = await this.api.getEntity(verb.sourceId);
|
||||
if (entity) {
|
||||
expanded.push({
|
||||
id: verb.sourceId,
|
||||
score: (candidate.score || 1.0) * 0.8, // Decay score by distance
|
||||
entity,
|
||||
metadata: entity.metadata
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return expanded;
|
||||
}
|
||||
/**
|
||||
* Vector search within existing candidates
|
||||
*/
|
||||
async vectorSearchWithin(query, candidates) {
|
||||
// Get the query vector
|
||||
const queryVector = typeof query === 'string' ?
|
||||
(await this.api.vectorSearch(query, 1))[0]?.entity?.vector :
|
||||
query;
|
||||
if (!queryVector)
|
||||
return candidates;
|
||||
// Score each candidate by vector similarity
|
||||
const scored = [];
|
||||
for (const candidate of candidates) {
|
||||
const entity = candidate.entity || await this.api.getEntity(candidate.id);
|
||||
if (entity && entity.vector) {
|
||||
const similarity = this.cosineSimilarity(queryVector, entity.vector);
|
||||
scored.push({
|
||||
...candidate,
|
||||
score: similarity,
|
||||
entity
|
||||
});
|
||||
}
|
||||
}
|
||||
// Sort by similarity and return
|
||||
return scored.sort((a, b) => b.score - a.score);
|
||||
}
|
||||
/**
|
||||
* Apply field filter to existing candidates
|
||||
*/
|
||||
applyFieldFilter(candidates, where) {
|
||||
return candidates.filter(candidate => {
|
||||
const metadata = candidate.metadata || candidate.entity?.metadata || {};
|
||||
return this.matchesFilter(metadata, where);
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Check if metadata matches filter conditions
|
||||
*/
|
||||
matchesFilter(metadata, where) {
|
||||
for (const [field, condition] of Object.entries(where)) {
|
||||
const value = metadata[field];
|
||||
if (typeof condition === 'object' && condition !== null) {
|
||||
// Handle operators
|
||||
if ('$eq' in condition && value !== condition.$eq)
|
||||
return false;
|
||||
if ('$ne' in condition && value === condition.$ne)
|
||||
return false;
|
||||
if ('$gt' in condition && !(value > condition.$gt))
|
||||
return false;
|
||||
if ('$gte' in condition && !(value >= condition.$gte))
|
||||
return false;
|
||||
if ('$lt' in condition && !(value < condition.$lt))
|
||||
return false;
|
||||
if ('$lte' in condition && !(value <= condition.$lte))
|
||||
return false;
|
||||
if ('$in' in condition && !condition.$in.includes(value))
|
||||
return false;
|
||||
if ('$nin' in condition && condition.$nin.includes(value))
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
// Direct equality
|
||||
if (value !== condition)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
/**
|
||||
* Calculate cosine similarity between two vectors
|
||||
*/
|
||||
cosineSimilarity(a, b) {
|
||||
let dotProduct = 0;
|
||||
let normA = 0;
|
||||
let normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dotProduct += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
||||
}
|
||||
/**
|
||||
* Fusion ranking using Reciprocal Rank Fusion (RRF)
|
||||
* This is the same algorithm used by Google and Elasticsearch
|
||||
*/
|
||||
fusionRank(resultSets, query) {
|
||||
// RRF constant - 60 is empirically proven optimal
|
||||
const k = 60;
|
||||
// Calculate dynamic weights based on query
|
||||
const weights = this.calculateSignalWeights(query);
|
||||
// Determine which result sets we have based on query
|
||||
let vectorResultsIdx = -1;
|
||||
let graphResultsIdx = -1;
|
||||
let metadataResultsIdx = -1;
|
||||
let currentIdx = 0;
|
||||
if (query.like || query.similar) {
|
||||
vectorResultsIdx = currentIdx++;
|
||||
}
|
||||
if (query.where) {
|
||||
metadataResultsIdx = currentIdx++;
|
||||
}
|
||||
// If we have metadata filters AND other searches, apply intersection
|
||||
if (metadataResultsIdx >= 0 && resultSets.length > 1) {
|
||||
const metadataResults = resultSets[metadataResultsIdx];
|
||||
// CRITICAL: If metadata filter returned no results, entire query should return empty
|
||||
if (metadataResults.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const metadataIds = new Set(metadataResults.map(r => r.id || r));
|
||||
// Filter ALL other result sets to only include items that match metadata
|
||||
for (let i = 0; i < resultSets.length; i++) {
|
||||
if (i !== metadataResultsIdx) {
|
||||
resultSets[i] = resultSets[i].filter(r => metadataIds.has(r.id || r));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Build fusion scores using RRF
|
||||
const fusionScores = new Map();
|
||||
// Process each result set with RRF
|
||||
resultSets.forEach((results, setIndex) => {
|
||||
// Determine signal type
|
||||
let signalType;
|
||||
let weight = 1.0;
|
||||
if (setIndex === vectorResultsIdx) {
|
||||
signalType = 'vector';
|
||||
weight = weights.vector;
|
||||
}
|
||||
else if (setIndex === graphResultsIdx) {
|
||||
signalType = 'graph';
|
||||
weight = weights.graph;
|
||||
}
|
||||
else if (setIndex === metadataResultsIdx) {
|
||||
signalType = 'field';
|
||||
weight = weights.field;
|
||||
}
|
||||
else {
|
||||
return; // Skip unknown signal types
|
||||
}
|
||||
// Apply RRF to each result
|
||||
results.forEach((result, rank) => {
|
||||
const id = result.id || result;
|
||||
// Calculate RRF score: 1 / (k + rank + 1)
|
||||
const rrfScore = weight * (1.0 / (k + rank + 1));
|
||||
if (!fusionScores.has(id)) {
|
||||
fusionScores.set(id, {
|
||||
entity: result.entity || result,
|
||||
vectorScore: 0,
|
||||
graphScore: 0,
|
||||
fieldScore: 0,
|
||||
rrfScore: 0,
|
||||
fusionScore: 0,
|
||||
metadata: result.metadata || {}
|
||||
});
|
||||
}
|
||||
const fusion = fusionScores.get(id);
|
||||
// Track individual signal scores
|
||||
if (signalType === 'vector') {
|
||||
fusion.vectorScore = result.score || 1.0;
|
||||
}
|
||||
else if (signalType === 'graph') {
|
||||
fusion.graphScore = result.score || 1.0;
|
||||
}
|
||||
else if (signalType === 'field') {
|
||||
fusion.fieldScore = result.score || 1.0;
|
||||
}
|
||||
// Accumulate RRF score
|
||||
fusion.rrfScore += rrfScore;
|
||||
});
|
||||
});
|
||||
// Convert to results array
|
||||
const results = Array.from(fusionScores.entries()).map(([id, fusion]) => ({
|
||||
id,
|
||||
entity: fusion.entity,
|
||||
score: fusion.rrfScore, // Use RRF score as primary score
|
||||
vector: fusion.entity?.vector || new Float32Array(0), // Include vector for SearchResult compatibility
|
||||
vectorScore: fusion.vectorScore,
|
||||
graphScore: fusion.graphScore,
|
||||
fieldScore: fusion.fieldScore,
|
||||
fusionScore: fusion.rrfScore, // RRF score is the fusion score
|
||||
metadata: fusion.metadata
|
||||
}));
|
||||
// Sort by fusion score (descending)
|
||||
results.sort((a, b) => b.fusionScore - a.fusionScore);
|
||||
// Apply offset and limit
|
||||
let final = results;
|
||||
if (query.offset && query.offset > 0) {
|
||||
final = final.slice(query.offset);
|
||||
}
|
||||
if (query.limit) {
|
||||
final = final.slice(0, query.limit);
|
||||
}
|
||||
return final;
|
||||
}
|
||||
/**
|
||||
* Calculate dynamic signal weights based on query characteristics
|
||||
*/
|
||||
calculateSignalWeights(query) {
|
||||
const hasVector = !!(query.like || query.similar);
|
||||
const hasGraph = !!(query.connected);
|
||||
const hasField = !!(query.where && Object.keys(query.where).length > 0);
|
||||
// Count active signals
|
||||
const activeSignals = [hasVector, hasGraph, hasField].filter(Boolean).length;
|
||||
if (activeSignals === 1) {
|
||||
// Single signal - full weight
|
||||
return {
|
||||
vector: hasVector ? 1.0 : 0,
|
||||
graph: hasGraph ? 1.0 : 0,
|
||||
field: hasField ? 1.0 : 0
|
||||
};
|
||||
}
|
||||
// Multiple signals - adaptive weights
|
||||
if (hasVector && hasGraph && hasField) {
|
||||
// All three signals - balanced with slight vector preference
|
||||
return {
|
||||
vector: 0.4, // Semantic search is often most relevant
|
||||
graph: 0.35, // Relationships are important
|
||||
field: 0.25 // Metadata is supportive
|
||||
};
|
||||
}
|
||||
else if (hasVector && hasGraph) {
|
||||
// Vector + Graph - emphasize semantics
|
||||
return {
|
||||
vector: 0.6,
|
||||
graph: 0.4,
|
||||
field: 0
|
||||
};
|
||||
}
|
||||
else if (hasVector && hasField) {
|
||||
// Vector + Field - balanced
|
||||
return {
|
||||
vector: 0.5,
|
||||
graph: 0,
|
||||
field: 0.5
|
||||
};
|
||||
}
|
||||
else if (hasGraph && hasField) {
|
||||
// Graph + Field - emphasize relationships
|
||||
return {
|
||||
vector: 0,
|
||||
graph: 0.6,
|
||||
field: 0.4
|
||||
};
|
||||
}
|
||||
// Default balanced weights
|
||||
return {
|
||||
vector: 0.34,
|
||||
graph: 0.33,
|
||||
field: 0.33
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Apply boost strategies
|
||||
*/
|
||||
applyBoosts(results, boost) {
|
||||
return results.map(r => {
|
||||
let boostFactor = 1.0;
|
||||
switch (boost) {
|
||||
case 'recent':
|
||||
// Boost recent items
|
||||
const age = Date.now() - (r.metadata?.timestamp || 0);
|
||||
boostFactor = Math.exp(-age / (30 * 24 * 60 * 60 * 1000)); // 30-day half-life
|
||||
break;
|
||||
case 'popular':
|
||||
// Boost by view count or connections
|
||||
boostFactor = Math.log10((r.metadata?.views || 0) + 10) / 2;
|
||||
break;
|
||||
case 'verified':
|
||||
// Boost verified content
|
||||
boostFactor = r.metadata?.verified ? 1.5 : 1.0;
|
||||
break;
|
||||
}
|
||||
return {
|
||||
...r,
|
||||
fusionScore: r.fusionScore * boostFactor
|
||||
};
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Add query explanations for debugging
|
||||
*/
|
||||
addExplanations(results, plan, totalTime) {
|
||||
return results.map(r => ({
|
||||
...r,
|
||||
explanation: {
|
||||
plan: plan.steps.map(s => `${s.type}:${s.operation}`).join(' → '),
|
||||
timing: {
|
||||
total: totalTime,
|
||||
...plan.steps.reduce((acc, step) => ({
|
||||
...acc,
|
||||
[step.type]: step.estimated
|
||||
}), {})
|
||||
},
|
||||
boosts: []
|
||||
}
|
||||
}));
|
||||
}
|
||||
// Query learning removed - unnecessary complexity
|
||||
/**
|
||||
* Optimize plan based on historical patterns
|
||||
*/
|
||||
// Query optimization from history removed
|
||||
/**
|
||||
* Clear query optimization cache
|
||||
*/
|
||||
clearCache() {
|
||||
this.planCache.clear();
|
||||
}
|
||||
/**
|
||||
* Get optimization statistics
|
||||
*/
|
||||
getStats() {
|
||||
return {
|
||||
cachedPlans: this.planCache.size,
|
||||
historySize: 0 // Query history removed
|
||||
};
|
||||
}
|
||||
}
|
||||
// Export a beautiful, simple API
|
||||
export async function find(brain, query) {
|
||||
const engine = new TripleIntelligenceEngine(brain);
|
||||
return engine.find(query);
|
||||
}
|
||||
//# sourceMappingURL=TripleIntelligence.js.map
|
||||
Loading…
Add table
Add a link
Reference in a new issue