feat: implement incremental sorted indices and Triple Intelligence find()
- Add incremental sorted index updates during CRUD operations for consistent <5ms range queries - Implement parallel search optimization with vector, metadata, and graph intelligence fusion - Fix metadata-only query handling to properly return results without vector search - Fix NLP recursive call issue by using embed() instead of add() - Add cardinality tracking for smart index optimization - Store entity data in metadata for proper retrieval - Add comprehensive performance documentation This improves query performance from O(n) to O(log n) for range queries and ensures consistent fast performance without lazy loading delays.
This commit is contained in:
parent
0996c72468
commit
33c6b06649
9 changed files with 1023 additions and 181 deletions
|
|
@ -11,7 +11,6 @@
|
|||
import { Brainy } from '../brainy.js'
|
||||
import { BaseAugmentation } from './brainyAugmentation.js'
|
||||
import { CacheAugmentation } from './cacheAugmentation.js'
|
||||
import { IndexAugmentation } from './indexAugmentation.js'
|
||||
import { MetricsAugmentation } from './metricsAugmentation.js'
|
||||
import { MonitoringAugmentation } from './monitoringAugmentation.js'
|
||||
import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js'
|
||||
|
|
@ -26,7 +25,6 @@ import { UniversalDisplayAugmentation } from './universalDisplayAugmentation.js'
|
|||
export function createDefaultAugmentations(
|
||||
config: {
|
||||
cache?: boolean | Record<string, any>
|
||||
index?: boolean | Record<string, any>
|
||||
metrics?: boolean | Record<string, any>
|
||||
monitoring?: boolean | Record<string, any>
|
||||
display?: boolean | Record<string, any>
|
||||
|
|
@ -40,11 +38,7 @@ export function createDefaultAugmentations(
|
|||
augmentations.push(new CacheAugmentation(cacheConfig))
|
||||
}
|
||||
|
||||
// Index augmentation (was MetadataIndex)
|
||||
if (config.index !== false) {
|
||||
const indexConfig = typeof config.index === 'object' ? config.index : {}
|
||||
augmentations.push(new IndexAugmentation(indexConfig))
|
||||
}
|
||||
// Note: Index augmentation removed - metadata indexing is now core functionality
|
||||
|
||||
// Metrics augmentation (was StatisticsCollector)
|
||||
if (config.metrics !== false) {
|
||||
|
|
@ -94,11 +88,9 @@ export const AugmentationHelpers = {
|
|||
},
|
||||
|
||||
/**
|
||||
* Get index augmentation
|
||||
* Note: Index augmentation removed - metadata indexing is now core functionality
|
||||
* Use brain.metadataIndex directly instead
|
||||
*/
|
||||
getIndex(brain: Brainy): IndexAugmentation | null {
|
||||
return getAugmentation<IndexAugmentation>(brain, 'index')
|
||||
},
|
||||
|
||||
/**
|
||||
* Get metrics augmentation
|
||||
|
|
|
|||
503
src/brainy.ts
503
src/brainy.ts
|
|
@ -22,6 +22,7 @@ import { ImprovedNeuralAPI } from './neural/improvedNeuralAPI.js'
|
|||
import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js'
|
||||
import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js'
|
||||
import { MetadataIndexManager } from './utils/metadataIndex.js'
|
||||
import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js'
|
||||
import {
|
||||
Entity,
|
||||
Relation,
|
||||
|
|
@ -47,6 +48,8 @@ export class Brainy<T = any> {
|
|||
// Core components
|
||||
private index!: HNSWIndex | HNSWIndexOptimized
|
||||
private storage!: StorageAdapter
|
||||
private metadataIndex!: MetadataIndexManager
|
||||
private graphIndex!: GraphAdjacencyIndex
|
||||
private embedder: EmbeddingFunction
|
||||
private distance: DistanceFunction
|
||||
private augmentationRegistry: AugmentationRegistry
|
||||
|
|
@ -56,7 +59,6 @@ export class Brainy<T = any> {
|
|||
private _neural?: ImprovedNeuralAPI
|
||||
private _nlp?: NaturalLanguageProcessor
|
||||
private _tripleIntelligence?: TripleIntelligenceSystem
|
||||
private _metadataIndex?: MetadataIndexManager
|
||||
|
||||
// State
|
||||
private initialized = false
|
||||
|
|
@ -109,6 +111,15 @@ export class Brainy<T = any> {
|
|||
// Setup index now that we have storage
|
||||
this.index = this.setupIndex()
|
||||
|
||||
// Initialize core metadata index
|
||||
this.metadataIndex = new MetadataIndexManager(this.storage)
|
||||
|
||||
// Initialize core graph index
|
||||
this.graphIndex = new GraphAdjacencyIndex(this.storage)
|
||||
|
||||
// Rebuild indexes if needed for existing data
|
||||
await this.rebuildIndexesIfNeeded()
|
||||
|
||||
// Initialize augmentations
|
||||
await this.augmentationRegistry.initializeAll({
|
||||
brain: this,
|
||||
|
|
@ -180,21 +191,28 @@ export class Brainy<T = any> {
|
|||
// Add to index
|
||||
await this.index.addItem({ id, vector })
|
||||
|
||||
// Prepare metadata object with data field included
|
||||
const metadata = {
|
||||
...(typeof params.data === 'object' && params.data !== null && !Array.isArray(params.data) ? params.data : {}),
|
||||
...params.metadata,
|
||||
_data: params.data, // Store the raw data in metadata
|
||||
noun: params.type,
|
||||
service: params.service,
|
||||
createdAt: Date.now()
|
||||
}
|
||||
|
||||
// Save to storage
|
||||
await this.storage.saveNoun({
|
||||
id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
metadata: {
|
||||
...(typeof params.data === 'object' && params.data !== null ? params.data : {}),
|
||||
...params.metadata,
|
||||
noun: params.type,
|
||||
service: params.service,
|
||||
createdAt: Date.now()
|
||||
}
|
||||
metadata
|
||||
})
|
||||
|
||||
// Add to metadata index for fast filtering
|
||||
await this.metadataIndex.addToIndex(id, metadata)
|
||||
|
||||
return id
|
||||
})
|
||||
}
|
||||
|
|
@ -212,24 +230,36 @@ export class Brainy<T = any> {
|
|||
return null
|
||||
}
|
||||
|
||||
// Extract metadata - separate user metadata from system metadata
|
||||
const { noun: nounType, service, createdAt, updatedAt, ...userMetadata } = noun.metadata || {}
|
||||
|
||||
// Construct entity from noun
|
||||
const entity: Entity<T> = {
|
||||
id: noun.id,
|
||||
vector: noun.vector,
|
||||
type: (nounType as NounType) || NounType.Thing,
|
||||
metadata: userMetadata as T,
|
||||
service: service as string,
|
||||
createdAt: (createdAt as number) || Date.now(),
|
||||
updatedAt: updatedAt as number
|
||||
}
|
||||
|
||||
return entity
|
||||
// Use the common conversion method
|
||||
return this.convertNounToEntity(noun)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a noun from storage to an entity
|
||||
*/
|
||||
private async convertNounToEntity(noun: any): Promise<Entity<T>> {
|
||||
// Extract metadata - separate user metadata from system metadata
|
||||
const { noun: nounType, service, createdAt, updatedAt, _data, ...userMetadata } = noun.metadata || {}
|
||||
|
||||
const entity: Entity<T> = {
|
||||
id: noun.id,
|
||||
vector: noun.vector,
|
||||
type: (nounType as NounType) || NounType.Thing,
|
||||
metadata: userMetadata as T,
|
||||
service: service as string,
|
||||
createdAt: (createdAt as number) || Date.now(),
|
||||
updatedAt: updatedAt as number
|
||||
}
|
||||
|
||||
// Only add data field if it exists
|
||||
if (_data !== undefined) {
|
||||
entity.data = _data
|
||||
}
|
||||
|
||||
return entity
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an entity
|
||||
*/
|
||||
|
|
@ -262,24 +292,32 @@ export class Brainy<T = any> {
|
|||
: params.metadata || existing.metadata
|
||||
|
||||
// Merge data objects if both old and new are objects
|
||||
const dataFields = typeof params.data === 'object' && params.data !== null
|
||||
const dataFields = typeof params.data === 'object' && params.data !== null && !Array.isArray(params.data)
|
||||
? params.data
|
||||
: {}
|
||||
|
||||
// Prepare updated metadata object with data field
|
||||
const updatedMetadata = {
|
||||
...newMetadata,
|
||||
...dataFields,
|
||||
_data: params.data !== undefined ? params.data : existing.data, // Update the data field
|
||||
noun: params.type || existing.type,
|
||||
service: existing.service,
|
||||
createdAt: existing.createdAt,
|
||||
updatedAt: Date.now()
|
||||
}
|
||||
|
||||
await this.storage.saveNoun({
|
||||
id: params.id,
|
||||
vector,
|
||||
connections: new Map(),
|
||||
level: 0,
|
||||
metadata: {
|
||||
...newMetadata,
|
||||
...dataFields,
|
||||
noun: params.type || existing.type,
|
||||
service: existing.service,
|
||||
createdAt: existing.createdAt,
|
||||
updatedAt: Date.now()
|
||||
}
|
||||
metadata: updatedMetadata
|
||||
})
|
||||
|
||||
// Update metadata index - remove old entry and add new one
|
||||
await this.metadataIndex.removeFromIndex(params.id)
|
||||
await this.metadataIndex.addToIndex(params.id, updatedMetadata)
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -290,9 +328,12 @@ export class Brainy<T = any> {
|
|||
await this.ensureInitialized()
|
||||
|
||||
return this.augmentationRegistry.execute('delete', { id }, async () => {
|
||||
// Remove from index
|
||||
// Remove from vector index
|
||||
await this.index.removeItem(id)
|
||||
|
||||
// Remove from metadata index
|
||||
await this.metadataIndex.removeFromIndex(id)
|
||||
|
||||
// Delete from storage
|
||||
await this.storage.deleteNoun(id)
|
||||
|
||||
|
|
@ -365,6 +406,9 @@ export class Brainy<T = any> {
|
|||
|
||||
await this.storage.saveVerb(verb)
|
||||
|
||||
// Add to graph index for O(1) lookups
|
||||
await this.graphIndex.addVerb(verb)
|
||||
|
||||
// Create bidirectional if requested
|
||||
if (params.bidirectional) {
|
||||
const reverseId = uuidv4()
|
||||
|
|
@ -378,6 +422,8 @@ export class Brainy<T = any> {
|
|||
} as any
|
||||
|
||||
await this.storage.saveVerb(reverseVerb)
|
||||
// Add reverse relationship to graph index too
|
||||
await this.graphIndex.addVerb(reverseVerb)
|
||||
}
|
||||
|
||||
return id
|
||||
|
|
@ -391,6 +437,9 @@ export class Brainy<T = any> {
|
|||
await this.ensureInitialized()
|
||||
|
||||
return this.augmentationRegistry.execute('unrelate', { id }, async () => {
|
||||
// Remove from graph index
|
||||
await this.graphIndex.removeVerb(id)
|
||||
// Remove from storage
|
||||
await this.storage.deleteVerb(id)
|
||||
})
|
||||
}
|
||||
|
|
@ -437,6 +486,7 @@ export class Brainy<T = any> {
|
|||
|
||||
/**
|
||||
* Unified find method - supports natural language and structured queries
|
||||
* Implements Triple Intelligence with parallel search optimization
|
||||
*/
|
||||
async find(query: string | FindParams<T>): Promise<Result<T>[]> {
|
||||
await this.ensureInitialized()
|
||||
|
|
@ -448,55 +498,68 @@ export class Brainy<T = any> {
|
|||
return this.augmentationRegistry.execute('find', params, async () => {
|
||||
let results: Result<T>[] = []
|
||||
|
||||
// Vector search
|
||||
if (params.query || params.vector) {
|
||||
const vector = params.vector || (await this.embed(params.query!))
|
||||
const limit = params.limit || 10
|
||||
|
||||
// Search index - returns array of [id, score] tuples
|
||||
const searchResults = await this.index.search(vector, limit * 2) // Get extra for filtering
|
||||
|
||||
// Hydrate results
|
||||
for (const [id, score] of searchResults) {
|
||||
const entity = await this.get(id)
|
||||
if (entity) {
|
||||
// Handle empty query - return paginated results from storage
|
||||
const hasSearchCriteria = params.query || params.vector || params.where ||
|
||||
params.type || params.service || params.near || params.connected
|
||||
|
||||
if (!hasSearchCriteria) {
|
||||
const limit = params.limit || 20
|
||||
const offset = params.offset || 0
|
||||
|
||||
const storageResults = await this.storage.getNouns({
|
||||
pagination: { limit: limit + offset, offset: 0 }
|
||||
})
|
||||
|
||||
for (let i = offset; i < Math.min(offset + limit, storageResults.items.length); i++) {
|
||||
const noun = storageResults.items[i]
|
||||
if (noun) {
|
||||
const entity = await this.convertNounToEntity(noun)
|
||||
results.push({
|
||||
id,
|
||||
score,
|
||||
id: noun.id,
|
||||
score: 1.0, // All results equally relevant for empty query
|
||||
entity
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// Proximity search
|
||||
// Execute parallel searches for optimal performance
|
||||
const searchPromises: Promise<Result<T>[]>[] = []
|
||||
|
||||
// Vector search component
|
||||
if (params.query || params.vector) {
|
||||
searchPromises.push(this.executeVectorSearch(params))
|
||||
}
|
||||
|
||||
// Proximity search component
|
||||
if (params.near) {
|
||||
const nearEntity = await this.get(params.near.id)
|
||||
if (nearEntity) {
|
||||
const nearResults = await this.index.search(
|
||||
nearEntity.vector,
|
||||
params.limit || 10
|
||||
)
|
||||
|
||||
for (const [id, score] of nearResults) {
|
||||
if (score >= (params.near.threshold || 0.7)) {
|
||||
const entity = await this.get(id)
|
||||
if (entity) {
|
||||
results.push({
|
||||
id,
|
||||
score,
|
||||
entity
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
searchPromises.push(this.executeProximitySearch(params))
|
||||
}
|
||||
|
||||
// Execute searches in parallel
|
||||
if (searchPromises.length > 0) {
|
||||
const searchResults = await Promise.all(searchPromises)
|
||||
for (const batch of searchResults) {
|
||||
results.push(...batch)
|
||||
}
|
||||
}
|
||||
|
||||
// Apply O(log n) metadata filtering using MetadataIndexManager
|
||||
// Remove duplicate results from parallel searches
|
||||
if (results.length > 0) {
|
||||
const uniqueResults = new Map<string, Result<T>>()
|
||||
for (const result of results) {
|
||||
const existing = uniqueResults.get(result.id)
|
||||
if (!existing || result.score > existing.score) {
|
||||
uniqueResults.set(result.id, result)
|
||||
}
|
||||
}
|
||||
results = Array.from(uniqueResults.values())
|
||||
}
|
||||
|
||||
// Apply O(log n) metadata filtering using core MetadataIndexManager
|
||||
if (params.where || params.type || params.service) {
|
||||
const metadataIndex = await this.getMetadataIndex()
|
||||
|
||||
// Build filter object for metadata index
|
||||
const filter: any = {}
|
||||
if (params.where) Object.assign(filter, params.where)
|
||||
|
|
@ -506,19 +569,39 @@ export class Brainy<T = any> {
|
|||
}
|
||||
if (params.service) filter.service = params.service
|
||||
|
||||
const filteredIds = await metadataIndex.getIdsForFilter(filter)
|
||||
const filteredIds = await this.metadataIndex.getIdsForFilter(filter)
|
||||
|
||||
// Only keep results that match the metadata filter
|
||||
const filteredIdSet = new Set(filteredIds)
|
||||
results = results.filter((r) => filteredIdSet.has(r.id))
|
||||
// CRITICAL FIX: Handle both cases properly
|
||||
if (results.length > 0) {
|
||||
// Filter existing results (from vector search)
|
||||
const filteredIdSet = new Set(filteredIds)
|
||||
results = results.filter((r) => filteredIdSet.has(r.id))
|
||||
} else {
|
||||
// Create results from metadata matches (metadata-only query)
|
||||
for (const id of filteredIds) {
|
||||
const entity = await this.get(id)
|
||||
if (entity) {
|
||||
results.push({
|
||||
id,
|
||||
score: 1.0, // All metadata matches are equally relevant
|
||||
entity
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Graph constraints
|
||||
// Graph search component with O(1) traversal
|
||||
if (params.connected) {
|
||||
results = await this.applyGraphConstraints(results, params.connected)
|
||||
results = await this.executeGraphSearch(params, results)
|
||||
}
|
||||
|
||||
// Apply fusion scoring if requested
|
||||
if (params.fusion && results.length > 0) {
|
||||
results = this.applyFusionScoring(results, params.fusion)
|
||||
}
|
||||
|
||||
// Sort by score and limit
|
||||
// Sort by score and apply pagination
|
||||
results.sort((a, b) => b.score - a.score)
|
||||
const limit = params.limit || 10
|
||||
const offset = params.offset || 0
|
||||
|
|
@ -800,12 +883,11 @@ export class Brainy<T = any> {
|
|||
*/
|
||||
getTripleIntelligence(): TripleIntelligenceSystem {
|
||||
if (!this._tripleIntelligence) {
|
||||
// For now, pass minimal parameters to avoid errors
|
||||
// This will be properly initialized when needed
|
||||
// Use core components directly - no lazy loading needed
|
||||
this._tripleIntelligence = new TripleIntelligenceSystem(
|
||||
null as any, // metadataIndex
|
||||
this.index as any,
|
||||
null as any, // graphIndex
|
||||
this.metadataIndex,
|
||||
this.index,
|
||||
this.graphIndex,
|
||||
async (text: string) => this.embedder(text),
|
||||
this.storage
|
||||
)
|
||||
|
|
@ -813,16 +895,6 @@ export class Brainy<T = any> {
|
|||
return this._tripleIntelligence
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Metadata Index Manager for O(log n) filtering
|
||||
*/
|
||||
private async getMetadataIndex(): Promise<MetadataIndexManager> {
|
||||
if (!this._metadataIndex) {
|
||||
this._metadataIndex = new MetadataIndexManager(this.storage)
|
||||
}
|
||||
return this._metadataIndex
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a streaming pipeline
|
||||
*/
|
||||
|
|
@ -891,42 +963,229 @@ export class Brainy<T = any> {
|
|||
// ============= HELPER METHODS =============
|
||||
|
||||
/**
|
||||
* Parse natural language query
|
||||
* Parse natural language query using advanced NLP with 220+ patterns
|
||||
* The embedding model is always available as it's core to Brainy's functionality
|
||||
*/
|
||||
private async parseNaturalQuery(query: string): Promise<FindParams<T>> {
|
||||
// Initialize NLP processor if needed (lazy loading)
|
||||
if (!this._nlp) {
|
||||
this._nlp = new NaturalLanguageProcessor(this as any)
|
||||
await this._nlp.init() // Ensure pattern library is loaded
|
||||
}
|
||||
const parsed = await this._nlp.processNaturalQuery(query)
|
||||
return parsed as FindParams<T>
|
||||
|
||||
// Process with our advanced pattern library (220+ patterns with embeddings)
|
||||
const tripleQuery = await this._nlp.processNaturalQuery(query)
|
||||
|
||||
// Convert TripleQuery to FindParams
|
||||
const params: FindParams<T> = {}
|
||||
|
||||
// Handle vector search
|
||||
if (tripleQuery.like || tripleQuery.similar) {
|
||||
params.query = typeof tripleQuery.like === 'string' ? tripleQuery.like :
|
||||
typeof tripleQuery.similar === 'string' ? tripleQuery.similar : query
|
||||
} else if (!tripleQuery.where && !tripleQuery.connected) {
|
||||
// Default to vector search if no other criteria specified
|
||||
params.query = query
|
||||
}
|
||||
|
||||
// Handle metadata filtering
|
||||
if (tripleQuery.where) {
|
||||
params.where = tripleQuery.where as Partial<T>
|
||||
}
|
||||
|
||||
// Handle graph relationships
|
||||
if (tripleQuery.connected) {
|
||||
params.connected = {
|
||||
to: Array.isArray(tripleQuery.connected.to) ? tripleQuery.connected.to[0] : tripleQuery.connected.to,
|
||||
from: Array.isArray(tripleQuery.connected.from) ? tripleQuery.connected.from[0] : tripleQuery.connected.from,
|
||||
via: tripleQuery.connected.type as any,
|
||||
depth: tripleQuery.connected.depth,
|
||||
direction: tripleQuery.connected.direction
|
||||
}
|
||||
}
|
||||
|
||||
// Handle other options
|
||||
if (tripleQuery.limit) params.limit = tripleQuery.limit
|
||||
if (tripleQuery.offset) params.offset = tripleQuery.offset
|
||||
|
||||
return this.enhanceNLPResult(params, query)
|
||||
}
|
||||
|
||||
/**
|
||||
* Enhance NLP results with fusion scoring
|
||||
*/
|
||||
private enhanceNLPResult(params: FindParams<T>, _originalQuery: string): FindParams<T> {
|
||||
// Add fusion scoring for complex queries
|
||||
if (params.query && params.where && Object.keys(params.where).length > 0) {
|
||||
params.fusion = params.fusion || {
|
||||
strategy: 'adaptive',
|
||||
weights: {
|
||||
vector: 0.6,
|
||||
field: 0.3,
|
||||
graph: 0.1
|
||||
}
|
||||
}
|
||||
}
|
||||
return params
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply graph constraints to results
|
||||
* Execute vector search component
|
||||
*/
|
||||
private async executeVectorSearch(params: FindParams<T>): Promise<Result<T>[]> {
|
||||
const vector = params.vector || (await this.embed(params.query!))
|
||||
const limit = params.limit || 10
|
||||
|
||||
const searchResults = await this.index.search(vector, limit * 2)
|
||||
const results: Result<T>[] = []
|
||||
|
||||
for (const [id, distance] of searchResults) {
|
||||
const entity = await this.get(id)
|
||||
if (entity) {
|
||||
const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
|
||||
results.push({ id, score, entity })
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute proximity search component
|
||||
*/
|
||||
private async executeProximitySearch(params: FindParams<T>): Promise<Result<T>[]> {
|
||||
if (!params.near) return []
|
||||
|
||||
const nearEntity = await this.get(params.near.id)
|
||||
if (!nearEntity) return []
|
||||
|
||||
const nearResults = await this.index.search(
|
||||
nearEntity.vector,
|
||||
params.limit || 10
|
||||
)
|
||||
|
||||
const results: Result<T>[] = []
|
||||
for (const [id, distance] of nearResults) {
|
||||
const score = Math.max(0, Math.min(1, 1 / (1 + distance)))
|
||||
|
||||
if (score >= (params.near.threshold || 0.7)) {
|
||||
const entity = await this.get(id)
|
||||
if (entity) {
|
||||
results.push({ id, score, entity })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute graph search component with O(1) traversal
|
||||
*/
|
||||
private async executeGraphSearch(params: FindParams<T>, existingResults: Result<T>[]): Promise<Result<T>[]> {
|
||||
if (!params.connected) return existingResults
|
||||
|
||||
const { from, to, direction = 'both' } = params.connected
|
||||
const connectedIds: string[] = []
|
||||
|
||||
if (from) {
|
||||
const neighbors = await this.graphIndex.getNeighbors(from, direction)
|
||||
connectedIds.push(...neighbors)
|
||||
}
|
||||
|
||||
if (to) {
|
||||
const reverseDirection = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both'
|
||||
const neighbors = await this.graphIndex.getNeighbors(to, reverseDirection)
|
||||
connectedIds.push(...neighbors)
|
||||
}
|
||||
|
||||
// Filter existing results to only connected entities
|
||||
if (existingResults.length > 0) {
|
||||
const connectedIdSet = new Set(connectedIds)
|
||||
return existingResults.filter(r => connectedIdSet.has(r.id))
|
||||
}
|
||||
|
||||
// Create results from connected entities
|
||||
const results: Result<T>[] = []
|
||||
for (const id of connectedIds) {
|
||||
const entity = await this.get(id)
|
||||
if (entity) {
|
||||
results.push({
|
||||
id,
|
||||
score: 1.0,
|
||||
entity
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply fusion scoring for multi-source results
|
||||
*/
|
||||
private applyFusionScoring(results: Result<T>[], fusionType: any): Result<T>[] {
|
||||
// Implement different fusion strategies
|
||||
const strategy = typeof fusionType === 'string' ? fusionType : fusionType.strategy || 'weighted'
|
||||
|
||||
switch (strategy) {
|
||||
case 'max':
|
||||
// Use maximum score from any source
|
||||
return results
|
||||
|
||||
case 'average':
|
||||
// Average scores from multiple sources
|
||||
const scoreMap = new Map<string, number[]>()
|
||||
for (const result of results) {
|
||||
const scores = scoreMap.get(result.id) || []
|
||||
scores.push(result.score)
|
||||
scoreMap.set(result.id, scores)
|
||||
}
|
||||
|
||||
return results.map(r => ({
|
||||
...r,
|
||||
score: scoreMap.get(r.id)!.reduce((a, b) => a + b, 0) / scoreMap.get(r.id)!.length
|
||||
}))
|
||||
|
||||
case 'weighted':
|
||||
default:
|
||||
// Weighted combination based on source importance
|
||||
const weights = fusionType.weights || { vector: 0.7, metadata: 0.2, graph: 0.1 }
|
||||
return results.map(r => ({
|
||||
...r,
|
||||
score: r.score * (weights.vector || 1.0)
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply graph constraints using O(1) GraphAdjacencyIndex - TRUE Triple Intelligence!
|
||||
*/
|
||||
private async applyGraphConstraints(
|
||||
results: Result<T>[],
|
||||
constraints: any
|
||||
): Promise<Result<T>[]> {
|
||||
// Filter by graph connections
|
||||
// Filter by graph connections using fast graph index
|
||||
if (constraints.to || constraints.from) {
|
||||
const filtered: Result<T>[] = []
|
||||
|
||||
for (const result of results) {
|
||||
let hasConnection = false
|
||||
|
||||
if (constraints.to) {
|
||||
const verbs = await this.storage.getVerbsBySource(result.id)
|
||||
const hasConnection = verbs.some(v => v.targetId === constraints.to)
|
||||
if (hasConnection) {
|
||||
filtered.push(result)
|
||||
}
|
||||
// Check if this entity connects TO the target (O(1) lookup)
|
||||
const outgoingNeighbors = await this.graphIndex.getNeighbors(result.id, 'out')
|
||||
hasConnection = outgoingNeighbors.includes(constraints.to)
|
||||
}
|
||||
|
||||
if (constraints.from) {
|
||||
const verbs = await this.storage.getVerbsByTarget(result.id)
|
||||
const hasConnection = verbs.some(v => v.sourceId === constraints.from)
|
||||
if (hasConnection) {
|
||||
filtered.push(result)
|
||||
}
|
||||
if (constraints.from && !hasConnection) {
|
||||
// Check if this entity connects FROM the source (O(1) lookup)
|
||||
const incomingNeighbors = await this.graphIndex.getNeighbors(result.id, 'in')
|
||||
hasConnection = incomingNeighbors.includes(constraints.from)
|
||||
}
|
||||
|
||||
if (hasConnection) {
|
||||
filtered.push(result)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1059,6 +1318,34 @@ export class Brainy<T = any> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild indexes if there's existing data but empty indexes
|
||||
*/
|
||||
private async rebuildIndexesIfNeeded(): Promise<void> {
|
||||
try {
|
||||
// Check if storage has data
|
||||
const entities = await this.storage.getNouns({ pagination: { limit: 1 } })
|
||||
if (entities.totalCount === 0 || entities.items.length === 0) {
|
||||
// No data in storage, no rebuild needed
|
||||
return
|
||||
}
|
||||
|
||||
// Check if metadata index is empty
|
||||
const metadataStats = await this.metadataIndex.getStats()
|
||||
if (metadataStats.totalEntries === 0) {
|
||||
console.log('🔄 Rebuilding metadata index for existing data...')
|
||||
await this.metadataIndex.rebuild()
|
||||
const newStats = await this.metadataIndex.getStats()
|
||||
console.log(`✅ Metadata index rebuilt: ${newStats.totalEntries} entries`)
|
||||
}
|
||||
|
||||
// Note: GraphAdjacencyIndex will rebuild itself as relationships are added
|
||||
// Vector index should already be populated if storage has data
|
||||
} catch (error) {
|
||||
console.warn('Warning: Could not check or rebuild indexes:', error)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Close and cleanup
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
* 🧠 BRAINY EMBEDDED PATTERNS
|
||||
*
|
||||
* AUTO-GENERATED - DO NOT EDIT
|
||||
* Generated: 2025-09-09T17:55:32.179Z
|
||||
* Generated: 2025-09-12T17:15:05.777Z
|
||||
* Patterns: 220
|
||||
* Coverage: 94-98% of all queries
|
||||
*
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ export class NaturalLanguageProcessor {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get embedding using add/get/delete pattern
|
||||
* Get embedding directly using brain's embed method
|
||||
*/
|
||||
private async getEmbedding(text: string): Promise<Vector> {
|
||||
// Check cache first
|
||||
|
|
@ -64,17 +64,8 @@ export class NaturalLanguageProcessor {
|
|||
return this.embeddingCache.get(text)!
|
||||
}
|
||||
|
||||
// Use add/get/delete pattern to get embedding
|
||||
const id = await this.brain.add({
|
||||
data: text,
|
||||
type: 'document'
|
||||
})
|
||||
|
||||
const entity = await this.brain.get(id)
|
||||
const embedding = entity?.vector || []
|
||||
|
||||
// Clean up temporary entity
|
||||
await this.brain.delete(id)
|
||||
// Use brain's embed method directly to avoid recursion
|
||||
const embedding = await (this.brain as any).embed(text)
|
||||
|
||||
// Cache the embedding
|
||||
this.embeddingCache.set(text, embedding)
|
||||
|
|
@ -91,6 +82,13 @@ export class NaturalLanguageProcessor {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Public initialization method for external callers
|
||||
*/
|
||||
async init(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
}
|
||||
|
||||
/**
|
||||
* 🎯 MAIN METHOD: Convert natural language to Triple Intelligence query
|
||||
*/
|
||||
|
|
@ -136,21 +134,12 @@ export class NaturalLanguageProcessor {
|
|||
* Hybrid parse when pattern matching fails
|
||||
*/
|
||||
private async hybridParse(query: string, queryEmbedding: Vector): Promise<TripleQuery> {
|
||||
// Analyze intent using embeddings and keywords
|
||||
// Analyze intent using keywords only (no recursive searches)
|
||||
const intent = await this.analyzeIntent(query)
|
||||
|
||||
// Find similar successful queries from history
|
||||
const similar = await this.findSimilarQueries(queryEmbedding)
|
||||
if (similar.length > 0 && similar[0].similarity > 0.9) {
|
||||
// Adapt a very similar previous query (for future implementation)
|
||||
// return this.adaptQuery(query, similar[0].result)
|
||||
}
|
||||
|
||||
// Extract entities using Brainy's search
|
||||
const entities = await this.extractEntities(query)
|
||||
|
||||
// Build query based on intent and entities
|
||||
return this.buildQuery(query, intent, entities)
|
||||
// Build query based on intent alone - no entity extraction needed
|
||||
// The vector search will handle finding relevant entities
|
||||
return this.buildQuery(query, intent, [])
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -118,17 +118,8 @@ export class PatternLibrary {
|
|||
return this.embeddingCache.get(text)!
|
||||
}
|
||||
|
||||
// Use add/get/delete pattern to get embeddings
|
||||
const id = await this.brain.add({
|
||||
data: text,
|
||||
type: 'document'
|
||||
})
|
||||
|
||||
const entity = await this.brain.get(id)
|
||||
const embedding = entity?.vector || []
|
||||
|
||||
// Clean up temporary entity
|
||||
await this.brain.delete(id)
|
||||
// Use brain's embed method directly to avoid recursion
|
||||
const embedding = await (this.brain as any).embed(text)
|
||||
|
||||
this.embeddingCache.set(text, embedding)
|
||||
return embedding
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ export interface Entity<T = any> {
|
|||
id: string
|
||||
vector: Vector
|
||||
type: NounType
|
||||
data?: any
|
||||
metadata?: T
|
||||
service?: string
|
||||
createdAt: number
|
||||
|
|
@ -163,7 +164,9 @@ export interface GraphConstraints {
|
|||
to?: string // Connected to this entity
|
||||
from?: string // Connected from this entity
|
||||
via?: VerbType | VerbType[] // Via these relationship types
|
||||
type?: VerbType | VerbType[] // Alias for via
|
||||
depth?: number // Max traversal depth (default: 1)
|
||||
direction?: 'in' | 'out' | 'both' // Direction of traversal (default: 'both')
|
||||
bidirectional?: boolean // Consider both directions
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -107,10 +107,10 @@ export class MetadataIndexManager {
|
|||
if (loaded) {
|
||||
this.sortedIndices.set(field, loaded)
|
||||
} else {
|
||||
// Create new sorted index
|
||||
// Create new sorted index - NOT dirty since we maintain incrementally
|
||||
this.sortedIndices.set(field, {
|
||||
values: [],
|
||||
isDirty: true,
|
||||
isDirty: false, // Clean by default with incremental updates
|
||||
fieldType: 'mixed'
|
||||
})
|
||||
}
|
||||
|
|
@ -165,6 +165,127 @@ export class MetadataIndexManager {
|
|||
sortedIndex.isDirty = false
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect field type from value
|
||||
*/
|
||||
private detectFieldType(value: any): 'number' | 'date' | 'string' | 'mixed' {
|
||||
if (typeof value === 'number' && !isNaN(value)) return 'number'
|
||||
if (value instanceof Date) return 'date'
|
||||
return 'string'
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two values based on field type for sorting
|
||||
*/
|
||||
private compareValues(a: any, b: any, fieldType: string): number {
|
||||
switch (fieldType) {
|
||||
case 'number':
|
||||
return (a as number) - (b as number)
|
||||
case 'date':
|
||||
return (a as Date).getTime() - (b as Date).getTime()
|
||||
case 'string':
|
||||
default:
|
||||
const aStr = String(a)
|
||||
const bStr = String(b)
|
||||
return aStr < bStr ? -1 : aStr > bStr ? 1 : 0
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Binary search to find insertion position for a value
|
||||
* Returns the index where the value should be inserted to maintain sorted order
|
||||
*/
|
||||
private findInsertPosition(sortedArray: Array<[any, Set<string>]>, value: any, fieldType: string): number {
|
||||
if (sortedArray.length === 0) return 0
|
||||
|
||||
let left = 0
|
||||
let right = sortedArray.length - 1
|
||||
|
||||
while (left <= right) {
|
||||
const mid = Math.floor((left + right) / 2)
|
||||
const midVal = sortedArray[mid][0]
|
||||
|
||||
const comparison = this.compareValues(midVal, value, fieldType)
|
||||
|
||||
if (comparison < 0) {
|
||||
left = mid + 1
|
||||
} else if (comparison > 0) {
|
||||
right = mid - 1
|
||||
} else {
|
||||
return mid // Value already exists at this position
|
||||
}
|
||||
}
|
||||
|
||||
return left // Insert position
|
||||
}
|
||||
|
||||
/**
|
||||
* Incrementally update sorted index when adding an ID
|
||||
*/
|
||||
private updateSortedIndexAdd(field: string, value: any, id: string): void {
|
||||
// Ensure sorted index exists
|
||||
if (!this.sortedIndices.has(field)) {
|
||||
this.sortedIndices.set(field, {
|
||||
values: [],
|
||||
isDirty: false,
|
||||
fieldType: this.detectFieldType(value)
|
||||
})
|
||||
}
|
||||
|
||||
const sortedIndex = this.sortedIndices.get(field)!
|
||||
const normalizedValue = this.normalizeValue(value)
|
||||
|
||||
// Find where this value should be in the sorted array
|
||||
const insertPos = this.findInsertPosition(
|
||||
sortedIndex.values,
|
||||
normalizedValue,
|
||||
sortedIndex.fieldType
|
||||
)
|
||||
|
||||
if (insertPos < sortedIndex.values.length &&
|
||||
sortedIndex.values[insertPos][0] === normalizedValue) {
|
||||
// Value already exists, just add the ID to the existing Set
|
||||
sortedIndex.values[insertPos][1].add(id)
|
||||
} else {
|
||||
// New value, insert at the correct position
|
||||
sortedIndex.values.splice(insertPos, 0, [normalizedValue, new Set([id])])
|
||||
}
|
||||
|
||||
// Mark as clean since we're maintaining it incrementally
|
||||
sortedIndex.isDirty = false
|
||||
}
|
||||
|
||||
/**
|
||||
* Incrementally update sorted index when removing an ID
|
||||
*/
|
||||
private updateSortedIndexRemove(field: string, value: any, id: string): void {
|
||||
const sortedIndex = this.sortedIndices.get(field)
|
||||
if (!sortedIndex || sortedIndex.values.length === 0) return
|
||||
|
||||
const normalizedValue = this.normalizeValue(value)
|
||||
|
||||
// Binary search to find the value
|
||||
const pos = this.findInsertPosition(
|
||||
sortedIndex.values,
|
||||
normalizedValue,
|
||||
sortedIndex.fieldType
|
||||
)
|
||||
|
||||
if (pos < sortedIndex.values.length &&
|
||||
sortedIndex.values[pos][0] === normalizedValue) {
|
||||
// Remove the ID from the Set
|
||||
sortedIndex.values[pos][1].delete(id)
|
||||
|
||||
// If no IDs left for this value, remove the entire entry
|
||||
if (sortedIndex.values[pos][1].size === 0) {
|
||||
sortedIndex.values.splice(pos, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Keep it clean
|
||||
sortedIndex.isDirty = false
|
||||
}
|
||||
|
||||
/**
|
||||
* Binary search for range start (inclusive or exclusive)
|
||||
*/
|
||||
|
|
@ -221,11 +342,18 @@ export class MetadataIndexManager {
|
|||
includeMin: boolean = true,
|
||||
includeMax: boolean = true
|
||||
): Promise<string[]> {
|
||||
// Ensure sorted index exists and is up to date
|
||||
// Ensure sorted index exists
|
||||
await this.ensureSortedIndex(field)
|
||||
await this.buildSortedIndex(field)
|
||||
|
||||
const sortedIndex = this.sortedIndices.get(field)
|
||||
// With incremental updates, we should rarely need to rebuild
|
||||
// Only rebuild if it's marked dirty (e.g., after a bulk load or migration)
|
||||
let sortedIndex = this.sortedIndices.get(field)
|
||||
if (sortedIndex?.isDirty) {
|
||||
prodLog.warn(`MetadataIndex: Sorted index for field '${field}' was dirty, rebuilding...`)
|
||||
await this.buildSortedIndex(field)
|
||||
sortedIndex = this.sortedIndices.get(field)
|
||||
}
|
||||
|
||||
if (!sortedIndex || sortedIndex.values.length === 0) return []
|
||||
|
||||
const sorted = sortedIndex.values
|
||||
|
|
@ -369,13 +497,8 @@ export class MetadataIndexManager {
|
|||
async addToIndex(id: string, metadata: any, skipFlush: boolean = false): Promise<void> {
|
||||
const fields = this.extractIndexableFields(metadata)
|
||||
|
||||
// Mark sorted indices as dirty when adding new data
|
||||
for (const { field } of fields) {
|
||||
const sortedIndex = this.sortedIndices.get(field)
|
||||
if (sortedIndex) {
|
||||
sortedIndex.isDirty = true
|
||||
}
|
||||
}
|
||||
// Track which fields we're updating for incremental sorted index maintenance
|
||||
const updatedFields = new Set<string>()
|
||||
|
||||
for (let i = 0; i < fields.length; i++) {
|
||||
const { field, value } = fields[i]
|
||||
|
|
@ -399,6 +522,15 @@ export class MetadataIndexManager {
|
|||
entry.lastUpdated = Date.now()
|
||||
this.dirtyEntries.add(key)
|
||||
|
||||
// Incrementally update sorted index for this field
|
||||
if (!updatedFields.has(field)) {
|
||||
this.updateSortedIndexAdd(field, value, id)
|
||||
updatedFields.add(field)
|
||||
} else {
|
||||
// Multiple values for same field - still update
|
||||
this.updateSortedIndexAdd(field, value, id)
|
||||
}
|
||||
|
||||
// Update field index
|
||||
await this.updateFieldIndex(field, value, 1)
|
||||
|
||||
|
|
@ -488,6 +620,9 @@ export class MetadataIndexManager {
|
|||
entry.lastUpdated = Date.now()
|
||||
this.dirtyEntries.add(key)
|
||||
|
||||
// Incrementally update sorted index when removing
|
||||
this.updateSortedIndexRemove(field, value, id)
|
||||
|
||||
// Update field index
|
||||
await this.updateFieldIndex(field, value, -1)
|
||||
|
||||
|
|
@ -509,6 +644,9 @@ export class MetadataIndexManager {
|
|||
entry.lastUpdated = Date.now()
|
||||
this.dirtyEntries.add(key)
|
||||
|
||||
// Incrementally update sorted index
|
||||
this.updateSortedIndexRemove(entry.field, entry.value, id)
|
||||
|
||||
if (entry.ids.size === 0) {
|
||||
this.indexCache.delete(key)
|
||||
await this.deleteIndexEntry(key)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue