fix: flush graph LSM-trees on close to prevent data loss across restarts
GraphAdjacencyIndex.flush() was a no-op — LSM MemTables were never written to SSTables for datasets under the 100K auto-flush threshold. This caused readdir, getRelations, and getDescendants to return empty results after close + reopen. Three fixes: - LSMTree.get(): merge MemTable + SSTable results (data spans both after flush, old early-return missed SSTable data) - GraphAdjacencyIndex.flush(): actually flush all 4 LSM-trees - GraphAdjacencyIndex.close(): close all 4 trees (was only closing 2) Also: brain.close() and shutdown hooks now call close() on graphIndex, HNSW index, and metadataIndex to release timers and file handles.
This commit is contained in:
parent
0add0af45a
commit
ab2493af02
4 changed files with 213 additions and 22 deletions
|
|
@ -800,10 +800,21 @@ export class GraphAdjacencyIndex {
|
|||
|
||||
const startTime = Date.now()
|
||||
|
||||
// Flush both LSM-trees
|
||||
// Note: LSMTree.close() will handle flushing MemTable
|
||||
// For now, we don't have an explicit flush method in LSMTree
|
||||
// The MemTable will be flushed automatically when threshold is reached
|
||||
// Flush all 4 LSM-trees in parallel (MemTables → SSTables on disk)
|
||||
await Promise.all([
|
||||
this.lsmTreeSource.flush().then(() => {
|
||||
prodLog.debug(`GraphAdjacencyIndex: Flushed source tree`)
|
||||
}),
|
||||
this.lsmTreeTarget.flush().then(() => {
|
||||
prodLog.debug(`GraphAdjacencyIndex: Flushed target tree`)
|
||||
}),
|
||||
this.lsmTreeVerbsBySource.flush().then(() => {
|
||||
prodLog.debug(`GraphAdjacencyIndex: Flushed verbs-by-source tree`)
|
||||
}),
|
||||
this.lsmTreeVerbsByTarget.flush().then(() => {
|
||||
prodLog.debug(`GraphAdjacencyIndex: Flushed verbs-by-target tree`)
|
||||
}),
|
||||
])
|
||||
|
||||
const elapsed = Date.now() - startTime
|
||||
|
||||
|
|
@ -819,10 +830,14 @@ export class GraphAdjacencyIndex {
|
|||
this.flushTimer = undefined
|
||||
}
|
||||
|
||||
// Close LSM-trees (will flush MemTables)
|
||||
// Close all 4 LSM-trees (will flush MemTables to SSTables)
|
||||
if (this.initialized) {
|
||||
await this.lsmTreeSource.close()
|
||||
await this.lsmTreeTarget.close()
|
||||
await Promise.all([
|
||||
this.lsmTreeSource.close(),
|
||||
this.lsmTreeTarget.close(),
|
||||
this.lsmTreeVerbsBySource.close(),
|
||||
this.lsmTreeVerbsByTarget.close(),
|
||||
])
|
||||
}
|
||||
|
||||
prodLog.info('GraphAdjacencyIndex: Shutdown complete')
|
||||
|
|
|
|||
|
|
@ -296,18 +296,21 @@ export class LSMTree {
|
|||
async get(sourceId: string): Promise<string[] | null> {
|
||||
const startTime = performance.now()
|
||||
|
||||
// Check MemTable first (hot data)
|
||||
// Merge results from MemTable AND SSTables
|
||||
// Data can span both after a flush (old data in SSTables, new in MemTable)
|
||||
const allTargets = new Set<string>()
|
||||
|
||||
// Check MemTable (hot data)
|
||||
const memResult = this.memTable.get(sourceId)
|
||||
if (memResult !== null) {
|
||||
return memResult
|
||||
for (const target of memResult) {
|
||||
allTargets.add(target)
|
||||
}
|
||||
}
|
||||
|
||||
// Check SSTables from newest to oldest
|
||||
// Newer levels (L0, L1, L2) checked first for better cache locality
|
||||
const maxLevel = Math.max(...Array.from(this.sstablesByLevel.keys()), 0)
|
||||
|
||||
const allTargets = new Set<string>()
|
||||
|
||||
for (let level = 0; level <= maxLevel; level++) {
|
||||
const sstables = this.sstablesByLevel.get(level) || []
|
||||
|
||||
|
|
@ -608,16 +611,20 @@ export class LSMTree {
|
|||
}
|
||||
|
||||
/**
|
||||
* Flush MemTable and stop compaction
|
||||
* Called during shutdown
|
||||
* Flush MemTable to SSTables without closing
|
||||
* Called by GraphAdjacencyIndex.flush() and brain.close()
|
||||
*/
|
||||
async flush(): Promise<void> {
|
||||
if (!this.memTable.isEmpty()) {
|
||||
await this.flushMemTable()
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.stopCompactionTimer()
|
||||
|
||||
// Final MemTable flush
|
||||
if (!this.memTable.isEmpty()) {
|
||||
await this.flushMemTable()
|
||||
}
|
||||
await this.flush()
|
||||
|
||||
prodLog.info('LSMTree: Closed successfully')
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue