fix: flush all native providers on shutdown to prevent data loss
Shutdown/close/flush now properly flushes all 4 components in parallel: metadataIndex, graphIndex, HNSW dirty nodes, and storage counts. Previously only counts were flushed, causing native provider data loss on restart. Also: - Wire roaring, msgpack, entityIdMapper provider consumption from plugins - Fix allOf filter O(n²) intersection → O(n) Set-based - Fix ne/exists negation filter to use Set-based exclusion - Add setMsgpackImplementation() swap in SSTable for native msgpack - Add setRoaringImplementation() swap for native CRoaring bitmaps - Add getAllIntIds() to EntityIdMapper for bitmap operations - Remove TypeAwareHNSWIndex from default index creation path - Export memory detection utilities from internals - Clean up 26 permanently-skipped dead tests
This commit is contained in:
parent
b87426409d
commit
773c5171c3
24 changed files with 297 additions and 1268 deletions
|
|
@ -2,8 +2,7 @@
|
|||
* Index Operations with Rollback Support
|
||||
*
|
||||
* Provides transactional operations for all indexes:
|
||||
* - TypeAwareHNSWIndex (per-type vector indexes)
|
||||
* - HNSWIndex (legacy/fallback vector index)
|
||||
* - HNSWIndex (unified vector index)
|
||||
* - MetadataIndexManager (roaring bitmap filtering)
|
||||
* - GraphAdjacencyIndex (LSM-tree graph storage)
|
||||
*
|
||||
|
|
@ -11,10 +10,8 @@
|
|||
*/
|
||||
|
||||
import type { HNSWIndex } from '../../hnsw/hnswIndex.js'
|
||||
import type { TypeAwareHNSWIndex } from '../../hnsw/typeAwareHNSWIndex.js'
|
||||
import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
|
||||
import type { GraphAdjacencyIndex } from '../../graph/graphAdjacencyIndex.js'
|
||||
import type { NounType } from '../../types/graphTypes.js'
|
||||
import type { GraphVerb } from '../../coreTypes.js'
|
||||
import type { Operation, RollbackAction } from '../types.js'
|
||||
|
||||
|
|
@ -67,55 +64,6 @@ export class AddToHNSWOperation implements Operation {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add to TypeAwareHNSW index with rollback support
|
||||
*
|
||||
* Rollback strategy:
|
||||
* - Remove item from type-specific index
|
||||
*/
|
||||
export class AddToTypeAwareHNSWOperation implements Operation {
|
||||
readonly name = 'AddToTypeAwareHNSW'
|
||||
|
||||
constructor(
|
||||
private readonly index: TypeAwareHNSWIndex,
|
||||
private readonly id: string,
|
||||
private readonly vector: number[],
|
||||
private readonly nounType: NounType
|
||||
) {}
|
||||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
// Check if item already exists
|
||||
const existed = await this.itemExists(this.id, this.nounType)
|
||||
|
||||
// Add to type-specific index
|
||||
await this.index.addItem({ id: this.id, vector: this.vector }, this.nounType)
|
||||
|
||||
// Return rollback action
|
||||
return async () => {
|
||||
if (!existed) {
|
||||
// Remove newly added item from type-specific index
|
||||
await this.index.removeItem(this.id, this.nounType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if item exists in type-specific index
|
||||
*/
|
||||
private async itemExists(id: string, type: NounType): Promise<boolean> {
|
||||
try {
|
||||
const index = (this.index as any).getIndexForType?.(type)
|
||||
if (index) {
|
||||
await index.getItem?.(id)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove from HNSW index with rollback support
|
||||
*
|
||||
|
|
@ -145,34 +93,6 @@ export class RemoveFromHNSWOperation implements Operation {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove from TypeAwareHNSW index with rollback support
|
||||
*
|
||||
* Rollback strategy:
|
||||
* - Re-add item to type-specific index with original vector
|
||||
*/
|
||||
export class RemoveFromTypeAwareHNSWOperation implements Operation {
|
||||
readonly name = 'RemoveFromTypeAwareHNSW'
|
||||
|
||||
constructor(
|
||||
private readonly index: TypeAwareHNSWIndex,
|
||||
private readonly id: string,
|
||||
private readonly vector: number[], // Required for rollback
|
||||
private readonly nounType: NounType
|
||||
) {}
|
||||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
// Remove from type-specific index
|
||||
await this.index.removeItem(this.id, this.nounType)
|
||||
|
||||
// Return rollback action
|
||||
return async () => {
|
||||
// Re-add item with original vector to type-specific index
|
||||
await this.index.addItem({ id: this.id, vector: this.vector }, this.nounType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add to metadata index with rollback support
|
||||
*
|
||||
|
|
|
|||
|
|
@ -22,9 +22,7 @@ export {
|
|||
// Index Operations
|
||||
export {
|
||||
AddToHNSWOperation,
|
||||
AddToTypeAwareHNSWOperation,
|
||||
RemoveFromHNSWOperation,
|
||||
RemoveFromTypeAwareHNSWOperation,
|
||||
AddToMetadataIndexOperation,
|
||||
RemoveFromMetadataIndexOperation,
|
||||
AddToGraphIndexOperation,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue