fix: flush all native providers on shutdown to prevent data loss

Shutdown/close/flush now properly flushes all 4 components in parallel:
metadataIndex, graphIndex, HNSW dirty nodes, and storage counts. Previously
only counts were flushed, causing native provider data loss on restart.

Also:
- Wire roaring, msgpack, entityIdMapper provider consumption from plugins
- Fix allOf filter O(n²) intersection → O(n) Set-based
- Fix ne/exists negation filter to use Set-based exclusion
- Add setMsgpackImplementation() swap in SSTable for native msgpack
- Add setRoaringImplementation() swap for native CRoaring bitmaps
- Add getAllIntIds() to EntityIdMapper for bitmap operations
- Remove TypeAwareHNSWIndex from default index creation path
- Export memory detection utilities from internals
- Clean up 26 permanently-skipped dead tests
This commit is contained in:
David Snelling 2026-02-01 16:23:49 -08:00
parent b87426409d
commit 773c5171c3
24 changed files with 297 additions and 1268 deletions

View file

@ -14,7 +14,6 @@
*/
import { HNSWIndex } from '../hnsw/hnswIndex.js'
import { TypeAwareHNSWIndex } from '../hnsw/typeAwareHNSWIndex.js'
import { MetadataIndexManager } from '../utils/metadataIndex.js'
import { Vector } from '../coreTypes.js'
import { NounType } from '../types/graphTypes.js'
@ -231,7 +230,7 @@ class QueryPlanner {
*/
export class TripleIntelligenceSystem {
private metadataIndex: MetadataIndexManager
private hnswIndex: HNSWIndex | TypeAwareHNSWIndex
private hnswIndex: HNSWIndex
private graphIndex: GraphAdjacencyIndex
private metrics: PerformanceMetrics
private planner: QueryPlanner
@ -240,7 +239,7 @@ export class TripleIntelligenceSystem {
constructor(
metadataIndex: MetadataIndexManager,
hnswIndex: HNSWIndex | TypeAwareHNSWIndex,
hnswIndex: HNSWIndex,
graphIndex: GraphAdjacencyIndex,
embedder: (text: string) => Promise<Vector>,
storage: any
@ -318,10 +317,8 @@ export class TripleIntelligenceSystem {
? await this.embedder(query)
: query
// Phase 3: Pass types to TypeAwareHNSWIndex for optimized search
const searchResults = this.hnswIndex instanceof TypeAwareHNSWIndex
? await this.hnswIndex.search(vector, limit, types)
: await this.hnswIndex.search(vector, limit)
// Single unified HNSW search (type filtering handled by metadata-first optimization)
const searchResults = await this.hnswIndex.search(vector, limit)
// Convert to result format
const results: TripleResult[] = []