perf: defer HNSW persistence during addMany() batch operations
addMany() now temporarily switches the HNSW index to deferred persist mode during the batch loop. Previously, each add() triggered ~16-20 saveHNSWData calls for modified neighbors (each a read→gzip→atomic-write cycle). For 450 items this was ~8,100 individual storage writes. With deferred mode, dirty node IDs are collected in a Set during the batch and flushed once at the end — deduplicating repeated neighbor updates. A node modified by insert #3 and again by insert #47 is written only once. Also adds setPersistMode() to TypeAwareHNSWIndex, propagating mode changes to all existing type-specific sub-indexes.
This commit is contained in:
parent
c054c41943
commit
973b6aaa39
2 changed files with 80 additions and 42 deletions
|
|
@ -2691,6 +2691,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OPTIMIZATION: Defer HNSW persistence during batch insert.
|
||||||
|
// Without this, each add() triggers ~16-20 neighbor saveHNSWData calls
|
||||||
|
// (each a read→gzip→atomic-write cycle). For 450 items that's ~8,100
|
||||||
|
// individual storage writes. Deferred mode collects dirty node IDs and
|
||||||
|
// flushes once at the end — deduplicating repeated neighbor updates.
|
||||||
|
const index = this.index as any
|
||||||
|
const prevPersistMode = typeof index.getPersistMode === 'function'
|
||||||
|
? index.getPersistMode()
|
||||||
|
: null
|
||||||
|
const canDefer = prevPersistMode === 'immediate'
|
||||||
|
&& typeof index.setPersistMode === 'function'
|
||||||
|
&& typeof index.flush === 'function'
|
||||||
|
|
||||||
|
if (canDefer) {
|
||||||
|
index.setPersistMode('deferred')
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
// Process in batches
|
// Process in batches
|
||||||
for (let i = 0; i < params.items.length; i += batchSize) {
|
for (let i = 0; i < params.items.length; i += batchSize) {
|
||||||
const chunk = params.items.slice(i, i + batchSize)
|
const chunk = params.items.slice(i, i + batchSize)
|
||||||
|
|
@ -2742,6 +2760,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
lastBatchTime = Date.now()
|
lastBatchTime = Date.now()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} finally {
|
||||||
|
// Restore persist mode and flush all dirty nodes in one pass
|
||||||
|
if (canDefer) {
|
||||||
|
await index.flush()
|
||||||
|
index.setPersistMode(prevPersistMode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
result.duration = Date.now() - startTime
|
result.duration = Date.now() - startTime
|
||||||
return result
|
return result
|
||||||
|
|
|
||||||
|
|
@ -163,6 +163,19 @@ export class TypeAwareHNSWIndex {
|
||||||
return this.persistMode
|
return this.persistMode
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Switch persist mode at runtime. Propagates to all existing type-specific indexes.
|
||||||
|
* Used by addMany() to defer persistence during batch operations.
|
||||||
|
*/
|
||||||
|
public setPersistMode(mode: 'immediate' | 'deferred'): void {
|
||||||
|
this.persistMode = mode
|
||||||
|
for (const index of this.indexes.values()) {
|
||||||
|
if (typeof (index as any).setPersistMode === 'function') {
|
||||||
|
(index as any).setPersistMode(mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get or create HNSW index for a specific type (lazy initialization)
|
* Get or create HNSW index for a specific type (lazy initialization)
|
||||||
*
|
*
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue