fix: flush all native providers on shutdown to prevent data loss
Shutdown/close/flush now properly flushes all 4 components in parallel: metadataIndex, graphIndex, HNSW dirty nodes, and storage counts. Previously only counts were flushed, causing native provider data loss on restart. Also: - Wire roaring, msgpack, entityIdMapper provider consumption from plugins - Fix allOf filter O(n²) intersection → O(n) Set-based - Fix ne/exists negation filter to use Set-based exclusion - Add setMsgpackImplementation() swap in SSTable for native msgpack - Add setRoaringImplementation() swap for native CRoaring bitmaps - Add getAllIntIds() to EntityIdMapper for bitmap operations - Remove TypeAwareHNSWIndex from default index creation path - Export memory detection utilities from internals - Clean up 26 permanently-skipped dead tests
This commit is contained in:
parent
b87426409d
commit
773c5171c3
24 changed files with 297 additions and 1268 deletions
362
src/brainy.ts
362
src/brainy.ts
|
|
@ -7,7 +7,8 @@
|
|||
|
||||
import { v4 as uuidv4 } from './universal/uuid.js'
|
||||
import { HNSWIndex } from './hnsw/hnswIndex.js'
|
||||
import { TypeAwareHNSWIndex } from './hnsw/typeAwareHNSWIndex.js'
|
||||
// TypeAwareHNSWIndex removed from default path — single unified HNSWIndex is faster
|
||||
// for the 99% of queries that don't filter by type (avoids searching 42 separate graphs)
|
||||
import { createStorage } from './storage/storageFactory.js'
|
||||
import { BaseStorage } from './storage/baseStorage.js'
|
||||
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb } from './coreTypes.js'
|
||||
|
|
@ -48,14 +49,12 @@ import {
|
|||
import {
|
||||
SaveNounMetadataOperation,
|
||||
SaveNounOperation,
|
||||
AddToTypeAwareHNSWOperation,
|
||||
AddToHNSWOperation,
|
||||
AddToMetadataIndexOperation,
|
||||
SaveVerbMetadataOperation,
|
||||
SaveVerbOperation,
|
||||
AddToGraphIndexOperation,
|
||||
RemoveFromHNSWOperation,
|
||||
RemoveFromTypeAwareHNSWOperation,
|
||||
RemoveFromMetadataIndexOperation,
|
||||
RemoveFromGraphIndexOperation,
|
||||
UpdateNounMetadataOperation,
|
||||
|
|
@ -135,8 +134,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
private static instances: Brainy[] = []
|
||||
|
||||
// Core components
|
||||
private index!: HNSWIndex | TypeAwareHNSWIndex
|
||||
private indexIsTypeAware = false // true when index supports addItem(item, type) / search(v, k, type)
|
||||
private index!: HNSWIndex
|
||||
private storage!: BaseStorage
|
||||
private metadataIndex!: MetadataIndexManager
|
||||
private graphIndex!: GraphAdjacencyIndex
|
||||
|
|
@ -275,7 +273,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
try {
|
||||
// Auto-detect and activate plugins BEFORE storage setup
|
||||
// so plugin-provided storage factories (e.g., mmap-filesystem) are available
|
||||
// so plugin-provided storage factories (e.g., filesystem override from cortex) are available
|
||||
await this.loadPlugins()
|
||||
|
||||
// Setup and initialize storage (checks plugin storage factories first)
|
||||
|
|
@ -301,6 +299,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
setGlobalCache(cacheProvider)
|
||||
}
|
||||
|
||||
// Provider: roaring bitmaps (native CRoaring replacement for WASM)
|
||||
const roaringProvider = this.pluginRegistry.getProvider<any>('roaring')
|
||||
if (roaringProvider) {
|
||||
const { setRoaringImplementation } = await import('./utils/roaring/index.js')
|
||||
setRoaringImplementation(roaringProvider)
|
||||
}
|
||||
|
||||
// Provider: msgpack (native replacement for JS @msgpack/msgpack)
|
||||
const msgpackProvider = this.pluginRegistry.getProvider<any>('msgpack')
|
||||
if (msgpackProvider) {
|
||||
const { setMsgpackImplementation } = await import('./graph/lsm/SSTable.js')
|
||||
setMsgpackImplementation(msgpackProvider)
|
||||
}
|
||||
|
||||
// Provider: distance function (resolve BEFORE setupIndex — index uses this.distance)
|
||||
const nativeDistance = this.pluginRegistry.getProvider<DistanceFunction>('distance')
|
||||
if (nativeDistance) {
|
||||
|
|
@ -309,14 +321,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Provider: HNSW index factory (plugin or JS fallback)
|
||||
this.index = this.createIndex()
|
||||
this.indexIsTypeAware = this.index instanceof TypeAwareHNSWIndex
|
||||
|| typeof (this.index as any).getIndexForType === 'function'
|
||||
|
||||
// Provider: metadata index factory
|
||||
const metadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex')
|
||||
this.metadataIndex = metadataFactory
|
||||
? metadataFactory(this.storage)
|
||||
: new MetadataIndexManager(this.storage)
|
||||
if (metadataFactory) {
|
||||
this.metadataIndex = metadataFactory(this.storage)
|
||||
} else {
|
||||
// JS fallback — inject native EntityIdMapper if cortex provides one
|
||||
const entityIdMapperFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('entityIdMapper')
|
||||
this.metadataIndex = new MetadataIndexManager(this.storage, {}, {
|
||||
entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined,
|
||||
})
|
||||
}
|
||||
|
||||
// Provider: graph index factory
|
||||
const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex')
|
||||
|
|
@ -440,21 +456,42 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
*/
|
||||
private registerShutdownHooks(): void {
|
||||
const flushOnShutdown = async () => {
|
||||
console.log('⚠️ Shutdown signal received - flushing pending counts...')
|
||||
console.log('Shutdown signal received - flushing pending data...')
|
||||
try {
|
||||
// Flush counts for all Brainy instances
|
||||
let flushedCount = 0
|
||||
for (const instance of Brainy.instances) {
|
||||
if (instance.storage && typeof (instance.storage as any).flushCounts === 'function') {
|
||||
await (instance.storage as any).flushCounts()
|
||||
if (instance.initialized) {
|
||||
// Full flush: counts + metadata index + graph index + HNSW dirty nodes
|
||||
await Promise.all([
|
||||
(async () => {
|
||||
if (instance.storage && typeof (instance.storage as any).flushCounts === 'function') {
|
||||
await (instance.storage as any).flushCounts()
|
||||
}
|
||||
})(),
|
||||
(async () => {
|
||||
if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') {
|
||||
await instance.metadataIndex.flush()
|
||||
}
|
||||
})(),
|
||||
(async () => {
|
||||
if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') {
|
||||
await instance.graphIndex.flush()
|
||||
}
|
||||
})(),
|
||||
(async () => {
|
||||
if (instance.index && typeof (instance.index as any).flush === 'function') {
|
||||
await (instance.index as any).flush()
|
||||
}
|
||||
})()
|
||||
])
|
||||
flushedCount++
|
||||
}
|
||||
}
|
||||
if (flushedCount > 0) {
|
||||
console.log(`✅ Counts flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`)
|
||||
console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to flush counts on shutdown:', error)
|
||||
console.error('Failed to flush on shutdown:', error)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -755,20 +792,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
)
|
||||
|
||||
// Operation 3: Add to HNSW index (after entity saved)
|
||||
if (this.indexIsTypeAware) {
|
||||
tx.addOperation(
|
||||
new AddToTypeAwareHNSWOperation(
|
||||
this.index as any,
|
||||
id,
|
||||
vector,
|
||||
params.type as any
|
||||
)
|
||||
)
|
||||
} else {
|
||||
tx.addOperation(
|
||||
new AddToHNSWOperation(this.index as any, id, vector)
|
||||
)
|
||||
}
|
||||
tx.addOperation(
|
||||
new AddToHNSWOperation(this.index as any, id, vector)
|
||||
)
|
||||
|
||||
// Operation 4: Add to metadata index
|
||||
tx.addOperation(
|
||||
|
|
@ -1182,31 +1208,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Operation 3-4: Update HNSW index (remove and re-add if reindexing needed)
|
||||
if (needsReindexing) {
|
||||
if (this.indexIsTypeAware) {
|
||||
tx.addOperation(
|
||||
new RemoveFromTypeAwareHNSWOperation(
|
||||
this.index as any,
|
||||
params.id,
|
||||
existing.vector,
|
||||
existing.type as any
|
||||
)
|
||||
)
|
||||
tx.addOperation(
|
||||
new AddToTypeAwareHNSWOperation(
|
||||
this.index as any,
|
||||
params.id,
|
||||
vector,
|
||||
newType as any
|
||||
)
|
||||
)
|
||||
} else {
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index as any, params.id, existing.vector)
|
||||
)
|
||||
tx.addOperation(
|
||||
new AddToHNSWOperation(this.index as any, params.id, vector)
|
||||
)
|
||||
}
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index as any, params.id, existing.vector)
|
||||
)
|
||||
tx.addOperation(
|
||||
new AddToHNSWOperation(this.index as any, params.id, vector)
|
||||
)
|
||||
}
|
||||
|
||||
// Operation 5-6: Update metadata index (remove old, add new)
|
||||
|
|
@ -1264,21 +1271,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Execute atomically with transaction system
|
||||
await this.transactionManager.executeTransaction(async (tx) => {
|
||||
// Operation 1: Remove from vector index
|
||||
if (noun && metadata) {
|
||||
if (this.indexIsTypeAware && metadata.noun) {
|
||||
tx.addOperation(
|
||||
new RemoveFromTypeAwareHNSWOperation(
|
||||
this.index as any,
|
||||
id,
|
||||
noun.vector,
|
||||
metadata.noun as any
|
||||
)
|
||||
)
|
||||
} else {
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index as any, id, noun.vector)
|
||||
)
|
||||
}
|
||||
if (noun) {
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index as any, id, noun.vector)
|
||||
)
|
||||
}
|
||||
|
||||
// Operation 2: Remove from metadata index
|
||||
|
|
@ -1975,6 +1971,42 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return results
|
||||
}
|
||||
|
||||
// Metadata-first optimization: pre-resolve filter IDs before vector search.
|
||||
// This enables HNSW to search only within matching candidates instead of
|
||||
// doing expensive post-filtering. Native HNSW uses searchWithCandidates (Rust
|
||||
// bitmap), JS HNSW converts to a Set-based filter function.
|
||||
let preResolvedMetadataIds: string[] | null = null
|
||||
let preResolvedFilter: any = null
|
||||
|
||||
if (params.where || params.type || params.service || params.excludeVFS) {
|
||||
preResolvedFilter = {}
|
||||
if (params.where) Object.assign(preResolvedFilter, params.where)
|
||||
if (params.service) preResolvedFilter.service = params.service
|
||||
if (params.excludeVFS === true) {
|
||||
preResolvedFilter.vfsType = { exists: false }
|
||||
preResolvedFilter.isVFSEntity = { ne: true }
|
||||
}
|
||||
if (params.type) {
|
||||
const types = Array.isArray(params.type) ? params.type : [params.type]
|
||||
if (types.length === 1) {
|
||||
preResolvedFilter.noun = types[0]
|
||||
} else {
|
||||
preResolvedFilter = {
|
||||
anyOf: types.map(type => ({
|
||||
noun: type,
|
||||
...preResolvedFilter
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter)
|
||||
|
||||
// Short-circuit: if metadata filter matches nothing, skip expensive vector search
|
||||
if (preResolvedMetadataIds.length === 0) {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
// Zero-Config Hybrid Search
|
||||
// Determine search mode: auto (default) combines text + semantic for query searches
|
||||
const searchMode = params.searchMode || 'auto'
|
||||
|
|
@ -1986,14 +2018,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
// Handle semantic-only query (user explicitly wants vector search)
|
||||
else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) {
|
||||
results = await this.executeVectorSearch(params)
|
||||
results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
|
||||
}
|
||||
// Handle explicit hybrid or auto mode with query
|
||||
else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) {
|
||||
// Zero-config hybrid: combine text + semantic search with RRF fusion
|
||||
const [textResults, semanticResults] = await Promise.all([
|
||||
this.executeTextSearch(params.query, limit * 2),
|
||||
this.executeVectorSearch(params)
|
||||
this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
|
||||
])
|
||||
|
||||
// Use user-specified alpha or auto-detect based on query length
|
||||
|
|
@ -2007,7 +2039,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
// Handle direct vector search (no query text) - no hybrid needed
|
||||
else if (params.vector && !params.query) {
|
||||
results = await this.executeVectorSearch(params)
|
||||
results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined)
|
||||
}
|
||||
// Handle proximity search
|
||||
else if (params.near) {
|
||||
|
|
@ -2032,42 +2064,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
results = Array.from(uniqueResults.values())
|
||||
}
|
||||
|
||||
// Apply O(log n) metadata filtering using core MetadataIndexManager
|
||||
if (params.where || params.type || params.service || params.excludeVFS) {
|
||||
// Build filter object for metadata index
|
||||
let filter: any = {}
|
||||
// Apply metadata filtering using pre-resolved IDs (metadata-first optimization).
|
||||
// When vector search was performed, HNSW already filtered by candidateIds —
|
||||
// this block handles pagination, entity loading, and metadata-only queries.
|
||||
if (preResolvedMetadataIds && preResolvedFilter) {
|
||||
const filteredIds = preResolvedMetadataIds
|
||||
|
||||
// Base filter from where and service
|
||||
if (params.where) Object.assign(filter, params.where)
|
||||
if (params.service) filter.service = params.service
|
||||
|
||||
// ExcludeVFS helper - exclude VFS infrastructure entities
|
||||
// VFS files/folders have vfsType set, extracted entities do NOT
|
||||
if (params.excludeVFS === true) {
|
||||
filter.vfsType = { exists: false }
|
||||
filter.isVFSEntity = { ne: true }
|
||||
}
|
||||
|
||||
if (params.type) {
|
||||
const types = Array.isArray(params.type) ? params.type : [params.type]
|
||||
if (types.length === 1) {
|
||||
filter.noun = types[0]
|
||||
} else {
|
||||
// For multiple types, create separate filter for each type with all conditions
|
||||
filter = {
|
||||
anyOf: types.map(type => ({
|
||||
noun: type,
|
||||
...filter
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const filteredIds = await this.metadataIndex.getIdsForFilter(filter)
|
||||
|
||||
// CRITICAL FIX: Handle both cases properly
|
||||
if (results.length > 0) {
|
||||
// OPTIMIZED: Filter existing results (from vector search) efficiently
|
||||
// Filter results by pre-resolved metadata IDs.
|
||||
// With metadata-first HNSW, most results already match — this is a safety net
|
||||
// for text search results and proximity results that weren't pre-filtered.
|
||||
const filteredIdSet = new Set(filteredIds)
|
||||
results = results.filter((r) => filteredIdSet.has(r.id))
|
||||
|
||||
|
|
@ -2120,7 +2126,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// Apply sorting if requested for metadata-only queries
|
||||
if (params.orderBy) {
|
||||
const sortedIds = await this.metadataIndex.getSortedIdsForFilter(
|
||||
filter,
|
||||
preResolvedFilter,
|
||||
params.orderBy,
|
||||
params.order || 'asc'
|
||||
)
|
||||
|
|
@ -2529,21 +2535,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const allVerbs = [...verbs, ...targetVerbs]
|
||||
|
||||
// Add delete operations to transaction
|
||||
if (noun && metadata) {
|
||||
if (this.indexIsTypeAware && metadata.noun) {
|
||||
tx.addOperation(
|
||||
new RemoveFromTypeAwareHNSWOperation(
|
||||
this.index as any,
|
||||
id,
|
||||
noun.vector,
|
||||
metadata.noun as any
|
||||
)
|
||||
)
|
||||
} else {
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index as any, id, noun.vector)
|
||||
)
|
||||
}
|
||||
if (noun) {
|
||||
tx.addOperation(
|
||||
new RemoveFromHNSWOperation(this.index as any, id, noun.vector)
|
||||
)
|
||||
}
|
||||
|
||||
if (metadata) {
|
||||
|
|
@ -2948,9 +2943,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Create HNSW index (uses plugin factory when available)
|
||||
clone.index = this.createIndex()
|
||||
clone.indexIsTypeAware = this.indexIsTypeAware
|
||||
|
||||
// Enable COW (handle both HNSWIndex and TypeAwareHNSWIndex)
|
||||
// Enable COW
|
||||
if ('enableCOW' in clone.index && typeof clone.index.enableCOW === 'function') {
|
||||
(clone.index as any).enableCOW(this.index)
|
||||
}
|
||||
|
|
@ -4188,7 +4182,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* 1. Storage counts (entity/verb counts by type)
|
||||
* 2. Metadata index (field indexes + EntityIdMapper)
|
||||
* 3. Graph adjacency index (relationship cache)
|
||||
* 4. HNSW vector index (no flush needed - saves directly)
|
||||
* 4. HNSW vector index (deferred dirty nodes)
|
||||
*
|
||||
* @example
|
||||
* // Flush after bulk operations
|
||||
|
|
@ -4204,7 +4198,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
async flush(): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
console.log('🔄 Flushing Brainy indexes and caches to disk...')
|
||||
console.log('Flushing Brainy indexes and caches to disk...')
|
||||
|
||||
const startTime = Date.now()
|
||||
|
||||
|
|
@ -4220,24 +4214,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// 2. Flush metadata index (field indexes + EntityIdMapper)
|
||||
this.metadataIndex.flush(),
|
||||
|
||||
// 3. Flush graph adjacency index (relationship cache)
|
||||
// Note: Graph structure is already persisted via storage.saveVerb() calls
|
||||
// This just flushes the in-memory cache for performance
|
||||
this.graphIndex.flush()
|
||||
// 3. Flush graph adjacency index (relationship cache + LSM trees)
|
||||
this.graphIndex.flush(),
|
||||
|
||||
// Note: Write-through cache (storage.writeCache) is NOT cleared here
|
||||
// Cache persists indefinitely for read-after-write consistency
|
||||
// Provides safety net for:
|
||||
// - Cloud storage eventual consistency (S3, GCS, Azure, R2)
|
||||
// - Filesystem buffer cache timing
|
||||
// - Type cache warming period (nounTypeCache population)
|
||||
// Cache entries are removed only when explicitly deleted (deleteObjectFromBranch)
|
||||
// Memory footprint is negligible for typical workloads (<10MB for 100k entities)
|
||||
// 4. Flush HNSW dirty nodes (deferred persistence mode)
|
||||
(async () => {
|
||||
if (this.index && typeof (this.index as any).flush === 'function') {
|
||||
await (this.index as any).flush()
|
||||
}
|
||||
})()
|
||||
])
|
||||
|
||||
const elapsed = Date.now() - startTime
|
||||
|
||||
console.log(`✅ All indexes flushed to disk in ${elapsed}ms`)
|
||||
console.log(`All indexes flushed to disk in ${elapsed}ms`)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -5619,15 +5609,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
/**
|
||||
* Execute vector search component
|
||||
*
|
||||
* @param params Find parameters
|
||||
* @param candidateIds Optional pre-resolved metadata filter IDs for metadata-first search.
|
||||
* When provided, HNSW search is restricted to these candidates:
|
||||
* - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering)
|
||||
* - JS HNSWIndex: converts to filter function with O(1) Set lookups
|
||||
*/
|
||||
private async executeVectorSearch(params: FindParams<T>): Promise<Result<T>[]> {
|
||||
private async executeVectorSearch(params: FindParams<T>, candidateIds?: string[]): Promise<Result<T>[]> {
|
||||
const vector = params.vector || (await this.embed(params.query!))
|
||||
const limit = params.limit || 10
|
||||
|
||||
// Pass type for type-aware indexes (10x faster for type-specific queries)
|
||||
const searchResults: [string, number][] = this.indexIsTypeAware && params.type
|
||||
? await (this.index as any).search(vector, limit * 2, params.type as any)
|
||||
: await this.index.search(vector, limit * 2)
|
||||
// Build search options for metadata-first candidate filtering
|
||||
const searchOptions = candidateIds ? { candidateIds } : undefined
|
||||
|
||||
// HNSW search with optional metadata-first candidate filtering
|
||||
const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions)
|
||||
|
||||
// Batch-load entities for 10-50x faster cloud storage performance
|
||||
// GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster)
|
||||
|
|
@ -5655,10 +5652,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const nearEntity = await this.get(params.near.id)
|
||||
if (!nearEntity) return []
|
||||
|
||||
// Pass type for type-aware indexes
|
||||
const nearResults: [string, number][] = this.indexIsTypeAware && params.type
|
||||
? await (this.index as any).search(nearEntity.vector, params.limit || 10, params.type as any)
|
||||
: await this.index.search(nearEntity.vector, params.limit || 10)
|
||||
const nearResults: [string, number][] = await this.index.search(nearEntity.vector, params.limit || 10)
|
||||
|
||||
// Filter by threshold first to minimize batch fetch
|
||||
const threshold = params.near.threshold || 0.7
|
||||
|
|
@ -6252,7 +6246,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const storageConfig = (this.config.storage || {}) as Record<string, unknown>
|
||||
const storageType = (storageConfig.type as string) || 'auto'
|
||||
|
||||
// Check plugin-provided storage factories (e.g., 'mmap-filesystem' from cortex)
|
||||
// Check plugin-provided storage factories (e.g., 'filesystem' override from cortex)
|
||||
const pluginFactory = this.pluginRegistry.getStorageFactory(storageType)
|
||||
if (pluginFactory) {
|
||||
const adapter = await pluginFactory.create(storageConfig)
|
||||
|
|
@ -6290,18 +6284,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
/**
|
||||
* Setup index
|
||||
* Setup index — single unified HNSW graph.
|
||||
*
|
||||
* Phase 2: Uses TypeAwareHNSWIndex for billion-scale optimization
|
||||
* - 87% memory reduction through separate graphs per entity type
|
||||
* - 10x faster type-specific queries
|
||||
* - Automatic type routing
|
||||
*
|
||||
* Smart defaults for HNSW persistence mode
|
||||
* - Cloud storage (GCS/S3/R2/Azure): 'deferred' for 30-50× faster adds
|
||||
* Smart defaults for HNSW persistence mode:
|
||||
* - Cloud storage (GCS/S3/R2/Azure): 'deferred' for 30-50x faster adds
|
||||
* - Local storage (FileSystem/Memory/OPFS): 'immediate' (already fast)
|
||||
*/
|
||||
private setupIndex(): HNSWIndex | TypeAwareHNSWIndex {
|
||||
private setupIndex(): HNSWIndex {
|
||||
const indexConfig = {
|
||||
...this.config.index,
|
||||
distanceFunction: this.distance,
|
||||
|
|
@ -6316,24 +6305,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
const persistMode = this.resolveHNSWPersistMode()
|
||||
|
||||
// Use TypeAwareHNSWIndex for non-memory storage (billion-scale optimization)
|
||||
const detectedStorageType = this.config.storage?.type || this.getStorageType()
|
||||
if (detectedStorageType !== 'memory') {
|
||||
return new TypeAwareHNSWIndex(indexConfig, this.distance, {
|
||||
storage: this.storage,
|
||||
useParallelization: true,
|
||||
persistMode
|
||||
})
|
||||
}
|
||||
|
||||
return new HNSWIndex(indexConfig as any, this.distance, { persistMode })
|
||||
return new HNSWIndex(indexConfig as any, this.distance, {
|
||||
storage: this.storage,
|
||||
useParallelization: true,
|
||||
persistMode
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an HNSW index, using plugin factory when available.
|
||||
* Shared by init(), fork(), checkout(), and clear() to avoid duplication.
|
||||
*/
|
||||
private createIndex(): HNSWIndex | TypeAwareHNSWIndex {
|
||||
private createIndex(): HNSWIndex {
|
||||
const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw')
|
||||
if (hnswFactory) {
|
||||
const persistMode = this.resolveHNSWPersistMode()
|
||||
|
|
@ -6719,13 +6702,36 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* This ensures deferred persistence mode data is saved
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
// Flush HNSW dirty nodes before closing
|
||||
// In deferred persistence mode, this persists all pending HNSW graph data
|
||||
if (this.index && typeof this.index.flush === 'function') {
|
||||
await this.index.flush()
|
||||
}
|
||||
// Flush ALL components before closing to prevent data loss
|
||||
// This is critical when cortex native providers buffer data in Rust memory
|
||||
await Promise.all([
|
||||
// Flush HNSW dirty nodes (deferred persistence mode)
|
||||
(async () => {
|
||||
if (this.index && typeof (this.index as any).flush === 'function') {
|
||||
await (this.index as any).flush()
|
||||
}
|
||||
})(),
|
||||
// Flush metadata index (field indexes + EntityIdMapper)
|
||||
(async () => {
|
||||
if (this.metadataIndex && typeof this.metadataIndex.flush === 'function') {
|
||||
await this.metadataIndex.flush()
|
||||
}
|
||||
})(),
|
||||
// Flush graph adjacency index (LSM trees)
|
||||
(async () => {
|
||||
if (this.graphIndex && typeof this.graphIndex.flush === 'function') {
|
||||
await this.graphIndex.flush()
|
||||
}
|
||||
})(),
|
||||
// Flush storage adapter counts
|
||||
(async () => {
|
||||
if (this.storage && typeof (this.storage as any).flushCounts === 'function') {
|
||||
await (this.storage as any).flushCounts()
|
||||
}
|
||||
})()
|
||||
])
|
||||
|
||||
// Deactivate plugins
|
||||
// Deactivate plugins (safe — all data flushed above)
|
||||
await this.pluginRegistry.deactivateAll()
|
||||
|
||||
// Restore console methods if silent mode was enabled
|
||||
|
|
|
|||
|
|
@ -14,9 +14,23 @@
|
|||
* - Footer: checksum, stats
|
||||
*/
|
||||
|
||||
import { encode, decode } from '@msgpack/msgpack'
|
||||
import { encode as defaultEncode, decode as defaultDecode } from '@msgpack/msgpack'
|
||||
import { BloomFilter, SerializedBloomFilter } from './BloomFilter.js'
|
||||
|
||||
// Swappable msgpack implementation — defaults to @msgpack/msgpack JS,
|
||||
// can be replaced with native msgpack (e.g., cortex's Rust-backed encoder)
|
||||
let _encode: (data: unknown) => Uint8Array = defaultEncode as (data: unknown) => Uint8Array
|
||||
let _decode: (data: Uint8Array) => unknown = defaultDecode as (data: Uint8Array) => unknown
|
||||
|
||||
/**
|
||||
* Replace the msgpack encode/decode implementation at runtime.
|
||||
* Called by brainy.ts when a cortex 'msgpack' provider is registered.
|
||||
*/
|
||||
export function setMsgpackImplementation(impl: { encode: (data: unknown) => Uint8Array, decode: (data: Uint8Array) => unknown }) {
|
||||
_encode = impl.encode
|
||||
_decode = impl.decode
|
||||
}
|
||||
|
||||
/**
|
||||
* Entry in the SSTable
|
||||
* Maps a source node to its target nodes
|
||||
|
|
@ -299,7 +313,7 @@ export class SSTable {
|
|||
checksum: this.calculateChecksum(this.entries)
|
||||
}
|
||||
|
||||
const serialized = encode(data)
|
||||
const serialized = _encode(data)
|
||||
|
||||
// Update size in metadata
|
||||
this.metadata.sizeBytes = serialized.length
|
||||
|
|
@ -331,7 +345,7 @@ export class SSTable {
|
|||
* @returns SSTable instance
|
||||
*/
|
||||
static deserialize(data: Uint8Array): SSTable {
|
||||
const decoded = decode(data) as SerializedSSTable
|
||||
const decoded = _decode(data) as SerializedSSTable
|
||||
|
||||
// Verify checksum
|
||||
const sstable = new SSTable(
|
||||
|
|
|
|||
|
|
@ -681,12 +681,18 @@ export class HNSWIndex {
|
|||
queryVector: Vector,
|
||||
k: number = 10,
|
||||
filter?: (id: string) => Promise<boolean>,
|
||||
options?: { rerank?: { multiplier: number } }
|
||||
options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
|
||||
): Promise<Array<[string, number]>> {
|
||||
if (this.nouns.size === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Metadata-first: convert candidateIds to filter function if no explicit filter
|
||||
if (!filter && options?.candidateIds && options.candidateIds.length > 0) {
|
||||
const candidateSet = new Set(options.candidateIds)
|
||||
filter = async (id: string) => candidateSet.has(id)
|
||||
}
|
||||
|
||||
// Check if query vector is defined
|
||||
if (!queryVector) {
|
||||
throw new Error('Query vector is undefined or null')
|
||||
|
|
|
|||
|
|
@ -257,7 +257,7 @@ export class TypeAwareHNSWIndex {
|
|||
k: number = 10,
|
||||
type?: NounType | NounType[],
|
||||
filter?: (id: string) => Promise<boolean>,
|
||||
options?: { rerank?: { multiplier: number } }
|
||||
options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
|
||||
): Promise<Array<[string, number]>> {
|
||||
// Single-type search (fast path)
|
||||
if (type && typeof type === 'string') {
|
||||
|
|
@ -288,7 +288,7 @@ export class TypeAwareHNSWIndex {
|
|||
k: number,
|
||||
types: NounType[],
|
||||
filter?: (id: string) => Promise<boolean>,
|
||||
options?: { rerank?: { multiplier: number } }
|
||||
options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
|
||||
): Promise<Array<[string, number]>> {
|
||||
const allResults: Array<[string, number]> = []
|
||||
|
||||
|
|
@ -323,7 +323,7 @@ export class TypeAwareHNSWIndex {
|
|||
queryVector: Vector,
|
||||
k: number,
|
||||
filter?: (id: string) => Promise<boolean>,
|
||||
options?: { rerank?: { multiplier: number } }
|
||||
options?: { rerank?: { multiplier: number }; candidateIds?: string[] }
|
||||
): Promise<Array<[string, number]>> {
|
||||
const allResults: Array<[string, number]> = []
|
||||
|
||||
|
|
|
|||
|
|
@ -9,3 +9,5 @@ export { FieldTypeInference, FieldType } from './utils/fieldTypeInference.js'
|
|||
export type { FieldTypeInfo } from './utils/fieldTypeInference.js'
|
||||
export { EntityIdMapper } from './utils/entityIdMapper.js'
|
||||
export type { EntityIdMapperOptions, EntityIdMapperData } from './utils/entityIdMapper.js'
|
||||
export { getRecommendedCacheConfig, formatBytes, checkMemoryPressure } from './utils/memoryDetection.js'
|
||||
export type { MemoryInfo, CacheAllocationStrategy } from './utils/memoryDetection.js'
|
||||
|
|
|
|||
|
|
@ -2,8 +2,7 @@
|
|||
* Index Operations with Rollback Support
|
||||
*
|
||||
* Provides transactional operations for all indexes:
|
||||
* - TypeAwareHNSWIndex (per-type vector indexes)
|
||||
* - HNSWIndex (legacy/fallback vector index)
|
||||
* - HNSWIndex (unified vector index)
|
||||
* - MetadataIndexManager (roaring bitmap filtering)
|
||||
* - GraphAdjacencyIndex (LSM-tree graph storage)
|
||||
*
|
||||
|
|
@ -11,10 +10,8 @@
|
|||
*/
|
||||
|
||||
import type { HNSWIndex } from '../../hnsw/hnswIndex.js'
|
||||
import type { TypeAwareHNSWIndex } from '../../hnsw/typeAwareHNSWIndex.js'
|
||||
import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
|
||||
import type { GraphAdjacencyIndex } from '../../graph/graphAdjacencyIndex.js'
|
||||
import type { NounType } from '../../types/graphTypes.js'
|
||||
import type { GraphVerb } from '../../coreTypes.js'
|
||||
import type { Operation, RollbackAction } from '../types.js'
|
||||
|
||||
|
|
@ -67,55 +64,6 @@ export class AddToHNSWOperation implements Operation {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add to TypeAwareHNSW index with rollback support
|
||||
*
|
||||
* Rollback strategy:
|
||||
* - Remove item from type-specific index
|
||||
*/
|
||||
export class AddToTypeAwareHNSWOperation implements Operation {
|
||||
readonly name = 'AddToTypeAwareHNSW'
|
||||
|
||||
constructor(
|
||||
private readonly index: TypeAwareHNSWIndex,
|
||||
private readonly id: string,
|
||||
private readonly vector: number[],
|
||||
private readonly nounType: NounType
|
||||
) {}
|
||||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
// Check if item already exists
|
||||
const existed = await this.itemExists(this.id, this.nounType)
|
||||
|
||||
// Add to type-specific index
|
||||
await this.index.addItem({ id: this.id, vector: this.vector }, this.nounType)
|
||||
|
||||
// Return rollback action
|
||||
return async () => {
|
||||
if (!existed) {
|
||||
// Remove newly added item from type-specific index
|
||||
await this.index.removeItem(this.id, this.nounType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if item exists in type-specific index
|
||||
*/
|
||||
private async itemExists(id: string, type: NounType): Promise<boolean> {
|
||||
try {
|
||||
const index = (this.index as any).getIndexForType?.(type)
|
||||
if (index) {
|
||||
await index.getItem?.(id)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove from HNSW index with rollback support
|
||||
*
|
||||
|
|
@ -145,34 +93,6 @@ export class RemoveFromHNSWOperation implements Operation {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove from TypeAwareHNSW index with rollback support
|
||||
*
|
||||
* Rollback strategy:
|
||||
* - Re-add item to type-specific index with original vector
|
||||
*/
|
||||
export class RemoveFromTypeAwareHNSWOperation implements Operation {
|
||||
readonly name = 'RemoveFromTypeAwareHNSW'
|
||||
|
||||
constructor(
|
||||
private readonly index: TypeAwareHNSWIndex,
|
||||
private readonly id: string,
|
||||
private readonly vector: number[], // Required for rollback
|
||||
private readonly nounType: NounType
|
||||
) {}
|
||||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
// Remove from type-specific index
|
||||
await this.index.removeItem(this.id, this.nounType)
|
||||
|
||||
// Return rollback action
|
||||
return async () => {
|
||||
// Re-add item with original vector to type-specific index
|
||||
await this.index.addItem({ id: this.id, vector: this.vector }, this.nounType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add to metadata index with rollback support
|
||||
*
|
||||
|
|
|
|||
|
|
@ -22,9 +22,7 @@ export {
|
|||
// Index Operations
|
||||
export {
|
||||
AddToHNSWOperation,
|
||||
AddToTypeAwareHNSWOperation,
|
||||
RemoveFromHNSWOperation,
|
||||
RemoveFromTypeAwareHNSWOperation,
|
||||
AddToMetadataIndexOperation,
|
||||
RemoveFromMetadataIndexOperation,
|
||||
AddToGraphIndexOperation,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
*/
|
||||
|
||||
import { HNSWIndex } from '../hnsw/hnswIndex.js'
|
||||
import { TypeAwareHNSWIndex } from '../hnsw/typeAwareHNSWIndex.js'
|
||||
import { MetadataIndexManager } from '../utils/metadataIndex.js'
|
||||
import { Vector } from '../coreTypes.js'
|
||||
import { NounType } from '../types/graphTypes.js'
|
||||
|
|
@ -231,7 +230,7 @@ class QueryPlanner {
|
|||
*/
|
||||
export class TripleIntelligenceSystem {
|
||||
private metadataIndex: MetadataIndexManager
|
||||
private hnswIndex: HNSWIndex | TypeAwareHNSWIndex
|
||||
private hnswIndex: HNSWIndex
|
||||
private graphIndex: GraphAdjacencyIndex
|
||||
private metrics: PerformanceMetrics
|
||||
private planner: QueryPlanner
|
||||
|
|
@ -240,7 +239,7 @@ export class TripleIntelligenceSystem {
|
|||
|
||||
constructor(
|
||||
metadataIndex: MetadataIndexManager,
|
||||
hnswIndex: HNSWIndex | TypeAwareHNSWIndex,
|
||||
hnswIndex: HNSWIndex,
|
||||
graphIndex: GraphAdjacencyIndex,
|
||||
embedder: (text: string) => Promise<Vector>,
|
||||
storage: any
|
||||
|
|
@ -318,10 +317,8 @@ export class TripleIntelligenceSystem {
|
|||
? await this.embedder(query)
|
||||
: query
|
||||
|
||||
// Phase 3: Pass types to TypeAwareHNSWIndex for optimized search
|
||||
const searchResults = this.hnswIndex instanceof TypeAwareHNSWIndex
|
||||
? await this.hnswIndex.search(vector, limit, types)
|
||||
: await this.hnswIndex.search(vector, limit)
|
||||
// Single unified HNSW search (type filtering handled by metadata-first optimization)
|
||||
const searchResults = await this.hnswIndex.search(vector, limit)
|
||||
|
||||
// Convert to result format
|
||||
const results: TripleResult[] = []
|
||||
|
|
|
|||
|
|
@ -150,6 +150,15 @@ export class EntityIdMapper {
|
|||
return this.uuidToInt.size
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all mapped integer IDs without storage reads.
|
||||
* Used for bitmap negation operations (ne, exists:false, missing:true)
|
||||
* to avoid full-table getAllIds() scans.
|
||||
*/
|
||||
getAllIntIds(): number[] {
|
||||
return Array.from(this.intToUuid.keys())
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert array of UUIDs to array of integers
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -56,6 +56,10 @@ export interface MetadataIndexConfig {
|
|||
excludeFields?: string[] // Never index these fields
|
||||
}
|
||||
|
||||
export interface MetadataIndexOptions {
|
||||
entityIdMapper?: EntityIdMapper // Optional pre-configured EntityIdMapper (e.g., native from cortex)
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages metadata indexes for fast filtering
|
||||
* Maintains inverted indexes: field+value -> list of IDs
|
||||
|
|
@ -143,7 +147,7 @@ export class MetadataIndexManager {
|
|||
// Replaces unreliable pattern matching with DuckDB-inspired value analysis
|
||||
private fieldTypeInference: FieldTypeInference
|
||||
|
||||
constructor(storage: StorageAdapter, config: MetadataIndexConfig = {}) {
|
||||
constructor(storage: StorageAdapter, config: MetadataIndexConfig = {}, options: MetadataIndexOptions = {}) {
|
||||
this.storage = storage
|
||||
this.config = {
|
||||
maxIndexSize: config.maxIndexSize ?? 10000,
|
||||
|
|
@ -185,8 +189,8 @@ export class MetadataIndexManager {
|
|||
// Get global unified cache for coordinated memory management
|
||||
this.unifiedCache = getGlobalCache()
|
||||
|
||||
// Initialize EntityIdMapper for roaring bitmap UUID ↔ integer mapping
|
||||
this.idMapper = new EntityIdMapper({
|
||||
// Use injected EntityIdMapper (e.g., native from cortex) or create JS fallback
|
||||
this.idMapper = options.entityIdMapper ?? new EntityIdMapper({
|
||||
storage,
|
||||
storageKey: 'brainy:entityIdMapper'
|
||||
})
|
||||
|
|
@ -1868,11 +1872,15 @@ export class MetadataIndexManager {
|
|||
|
||||
if (allIds.length === 0) return []
|
||||
if (allIds.length === 1) return allIds[0]
|
||||
|
||||
// Intersection of all sets
|
||||
return allIds.reduce((intersection, currentSet) =>
|
||||
intersection.filter(id => currentSet.includes(id))
|
||||
)
|
||||
|
||||
// Set-based intersection O(n) — start with smallest set for optimal perf
|
||||
const sorted = allIds.sort((a, b) => a.length - b.length)
|
||||
let result = new Set(sorted[0])
|
||||
for (let i = 1; i < sorted.length; i++) {
|
||||
const current = new Set(sorted[i])
|
||||
result = new Set([...result].filter(id => current.has(id)))
|
||||
}
|
||||
return Array.from(result)
|
||||
}
|
||||
|
||||
if (filter.anyOf && Array.isArray(filter.anyOf)) {
|
||||
|
|
@ -1930,21 +1938,22 @@ export class MetadataIndexManager {
|
|||
// Canonical: 'ne' | Alias: 'notEquals' | Deprecated: 'isNot'
|
||||
case 'isNot': // DEPRECATED: Use 'ne' instead
|
||||
case 'notEquals': // Alias for 'ne'
|
||||
case 'ne':
|
||||
case 'ne': {
|
||||
// For notEquals, we need all IDs EXCEPT those matching the value
|
||||
// This is especially important for soft delete: deleted !== true
|
||||
// should include items without a deleted field
|
||||
|
||||
// First, get all IDs in the database
|
||||
const allItemIds = await this.getAllIds()
|
||||
// Use EntityIdMapper universe (in-memory) instead of getAllIds() storage scan
|
||||
const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
|
||||
|
||||
// Then get IDs that match the value we want to exclude
|
||||
const excludeIds = await this.getIds(field, operand)
|
||||
const excludeSet = new Set(excludeIds)
|
||||
|
||||
// Return all IDs except those to exclude
|
||||
fieldResults = allItemIds.filter(id => !excludeSet.has(id))
|
||||
fieldResults = allKnownIds.filter(id => !excludeSet.has(id))
|
||||
break
|
||||
}
|
||||
|
||||
// ===== MULTI-VALUE OPERATORS =====
|
||||
// Canonical: 'in' | Alias: 'oneOf'
|
||||
|
|
@ -2034,8 +2043,7 @@ export class MetadataIndexManager {
|
|||
fieldResults = this.idMapper.intsIterableToUuids(allIntIds)
|
||||
} else {
|
||||
// exists: false - Get all IDs that DON'T have this field
|
||||
// Fixed excludeVFS bug (was returning empty array)
|
||||
const allItemIds = await this.getAllIds()
|
||||
// Uses EntityIdMapper universe (in-memory) instead of getAllIds() storage scan
|
||||
const existsIntIds = new Set<number>()
|
||||
|
||||
// Get IDs that HAVE this field
|
||||
|
|
@ -2053,10 +2061,11 @@ export class MetadataIndexManager {
|
|||
}
|
||||
}
|
||||
|
||||
// Convert to UUIDs and subtract from all IDs
|
||||
// Use EntityIdMapper universe and subtract IDs that have this field
|
||||
const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
|
||||
const existsUuids = this.idMapper.intsIterableToUuids(existsIntIds)
|
||||
const existsSet = new Set(existsUuids)
|
||||
fieldResults = allItemIds.filter(id => !existsSet.has(id))
|
||||
fieldResults = allKnownIds.filter(id => !existsSet.has(id))
|
||||
}
|
||||
break
|
||||
|
||||
|
|
@ -2068,7 +2077,7 @@ export class MetadataIndexManager {
|
|||
// Added for API consistency with in-memory metadataFilter
|
||||
if (operand) {
|
||||
// missing: true - field does NOT exist (same as exists: false)
|
||||
const allItemIds = await this.getAllIds()
|
||||
// Uses EntityIdMapper universe (in-memory) instead of getAllIds() storage scan
|
||||
const existsIntIds = new Set<number>()
|
||||
|
||||
const sparseIndex = await this.loadSparseIndex(field)
|
||||
|
|
@ -2085,9 +2094,11 @@ export class MetadataIndexManager {
|
|||
}
|
||||
}
|
||||
|
||||
// Use EntityIdMapper universe and subtract IDs that have this field
|
||||
const allKnownIds = this.idMapper.intsIterableToUuids(this.idMapper.getAllIntIds())
|
||||
const existsUuids = this.idMapper.intsIterableToUuids(existsIntIds)
|
||||
const existsSet = new Set(existsUuids)
|
||||
fieldResults = allItemIds.filter(id => !existsSet.has(id))
|
||||
fieldResults = allKnownIds.filter(id => !existsSet.has(id))
|
||||
} else {
|
||||
// missing: false - field DOES exist (same as exists: true)
|
||||
const allIntIds = new Set<number>()
|
||||
|
|
@ -2126,11 +2137,15 @@ export class MetadataIndexManager {
|
|||
|
||||
if (idSets.length === 0) return []
|
||||
if (idSets.length === 1) return idSets[0]
|
||||
|
||||
// Intersection of all field criteria (implicit AND)
|
||||
return idSets.reduce((intersection, currentSet) =>
|
||||
intersection.filter(id => currentSet.includes(id))
|
||||
)
|
||||
|
||||
// Set-based intersection O(n) — start with smallest set for optimal perf
|
||||
const sortedSets = idSets.sort((a, b) => a.length - b.length)
|
||||
let resultSet = new Set(sortedSets[0])
|
||||
for (let i = 1; i < sortedSets.length; i++) {
|
||||
const current = new Set(sortedSets[i])
|
||||
resultSet = new Set([...resultSet].filter(id => current.has(id)))
|
||||
}
|
||||
return Array.from(resultSet)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
*/
|
||||
|
||||
import {
|
||||
RoaringBitmap32,
|
||||
RoaringBitmap32 as WasmRoaringBitmap32,
|
||||
RoaringBitmap32Iterator,
|
||||
roaringLibraryInitialize,
|
||||
roaringLibraryIsReady,
|
||||
|
|
@ -24,9 +24,35 @@ import {
|
|||
// This ensures RoaringBitmap32 is ready to use immediately after import
|
||||
await roaringLibraryInitialize()
|
||||
|
||||
// Re-export all types and values
|
||||
// Swappable implementation — defaults to WASM, can be replaced with native CRoaring
|
||||
// Uses a wrapper class that delegates to the active implementation so that the exported
|
||||
// RoaringBitmap32 remains a class (usable as both value and type).
|
||||
let _impl: typeof WasmRoaringBitmap32 = WasmRoaringBitmap32
|
||||
|
||||
/**
|
||||
* Replace the RoaringBitmap32 implementation at runtime.
|
||||
* Called by brainy.ts when a cortex 'roaring' provider is registered.
|
||||
*/
|
||||
export function setRoaringImplementation(impl: typeof WasmRoaringBitmap32) {
|
||||
_impl = impl
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current RoaringBitmap32 implementation (WASM or native).
|
||||
* Use this instead of direct `new RoaringBitmap32()` when constructing bitmaps
|
||||
* in hot paths that should benefit from native replacement.
|
||||
*/
|
||||
export function getRoaringBitmap32(): typeof WasmRoaringBitmap32 {
|
||||
return _impl
|
||||
}
|
||||
|
||||
// Re-export the original WASM class as RoaringBitmap32 for type compatibility
|
||||
// Consumers use this as both a type (Map<string, RoaringBitmap32>) and value (new RoaringBitmap32())
|
||||
// The WASM and native implementations are API-compatible
|
||||
export { WasmRoaringBitmap32 as RoaringBitmap32 }
|
||||
|
||||
// Re-export remaining types and values
|
||||
export {
|
||||
RoaringBitmap32,
|
||||
RoaringBitmap32Iterator,
|
||||
roaringLibraryInitialize,
|
||||
roaringLibraryIsReady,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue