feat: mmap-vector backend wiring — HNSWIndex consumes vectorStore:mmap (2.4.0 #2)
Cortex already registers the vectorStore:mmap provider (its Rust NativeMmapVectorStore), but brainy has never consumed it — preloadVectors and getVectorSafe still go straight to storage.getNounVector for every id, even when an mmap layer is available. This wires the consumer end. Architecture: - NEW MmapVectorBackend (src/hnsw/mmapVectorBackend.ts) — bridges brainy's UUID-keyed vector reads to a int-slot mmap file via the vectorStore:mmap provider. Slots are addressed by the stable int id from the post-2.4.0 #1 EntityIdMapper (the foundation this depends on). Auto-grows the file (doubling) when a write lands beyond capacity, so HNSWIndex never has to think about sizing. The class never touches per-entity storage — it owns only the mmap layer. - HNSWIndex changes — adds a vectorBackend field + a setVectorBackend setter. The vector read paths (preloadVectors, getVectorSafe) try the mmap layer first; on a storage fallback hit, they LAZILY write back into the mmap slot. An upgraded install converges to the zero-copy fast path under live traffic — no big-bang migration step. The legacy per-entity path is preserved and still used when no backend is set. - brainy.ts wiring — a new private wireMmapVectorBackend() runs once during init, after plugin activation + metadataIndex setup. It activates the backend only when (a) the vectorStore:mmap provider is registered, (b) the storage adapter resolves a real local path via getBinaryBlobPath(), and (c) the metadata index exposes its idMapper. Cloud adapters return null on (b) and the backend is silently skipped; HNSWIndex's behaviour is then identical to pre-2.4.0. - Provider interfaces in plugin.ts — VectorStoreMmapProvider and VectorStoreMmapInstance document the contract cortex's class fulfils (the class IS the provider — static factory methods). Brainy depends on the interfaces, not on cortex; the structural match is verified when cortex 2.4.0 picks up this brainy release. Tests (1428 total, +11 vs pre-2.4.0): - tests/unit/hnsw/mmap-vector-backend.test.ts — 6 unit tests with an in-memory mock provider. Covers round-trip, batch reads with interleaved misses, slot stability (no re-slotting on overwrite), file growth without data loss, idempotent open, and null returns for unwritten slots. The real perf integration with cortex's NativeMmapVectorStore is exercised when cortex 2.4.0 wires this in. - tests/unit/utils/entity-id-mapper-stability.test.ts — moved here from tests/regression/ (which is NOT in the unit-config include glob, so the five #23 tests were not actually being run by npm test). The unit config matches tests/unit/**/*.test.ts. The 2.4.0 #2 follow-up will be the chunked-segment layout for remote storage adapters (S3 / R2 / GCS) where a single growing file doesn't fit immutable objects. For 2.4.0 release: local-FS only.
This commit is contained in:
parent
b2408cb5a6
commit
d4cb26c604
6 changed files with 640 additions and 4 deletions
|
|
@ -1,161 +0,0 @@
|
|||
/**
|
||||
* Regression test: EntityIdMapper stability across rebuild.
|
||||
*
|
||||
* The foundation 2.4.0 (vector mmap store, graph link compression, column-store
|
||||
* JS↔native interchange) all key off UUID→int mappings that **must not change**
|
||||
* across a metadata-index rebuild. Previously `metadataIndex.rebuild()` called
|
||||
* `idMapper.clear()` which reset `nextId` to 1 and renumbered every UUID by
|
||||
* re-insertion order, silently invalidating any consumer that had persisted
|
||||
* int-keyed data against the old map.
|
||||
*
|
||||
* This test pins down the stability contract:
|
||||
*
|
||||
* 1. UUID→int mappings persist across a single rebuild.
|
||||
* 2. Mappings persist across many consecutive rebuilds.
|
||||
* 3. New entities added after rebuild get fresh ints greater than any prior
|
||||
* assignment — no collisions with existing UUIDs' ints.
|
||||
* 4. Removed entities leave a permanent hole — new entities don't recycle the
|
||||
* gap, even across a rebuild.
|
||||
* 5. `clearAllIndexData()` is the explicit, intentional nuclear path — it DOES
|
||||
* renumber. This is the only documented way to invalidate the int space, and
|
||||
* a warning is logged so consumers know persisted int-keyed data is now stale.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
|
||||
const DIM = 384
|
||||
const makeVec = (seed = 1) =>
|
||||
new Float32Array(DIM).map((_, i) => ((i + seed) % DIM) / DIM)
|
||||
|
||||
describe('EntityIdMapper stability (foundation for 2.4.0)', () => {
|
||||
let brain: Brainy
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({ storage: { type: 'memory' }, silent: true })
|
||||
await brain.init()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
async function addEntity(name: string, seed: number): Promise<string> {
|
||||
return brain.add({
|
||||
data: name,
|
||||
vector: makeVec(seed),
|
||||
type: 'thing' as any,
|
||||
metadata: { name }
|
||||
})
|
||||
}
|
||||
|
||||
function getInt(uuid: string): number | undefined {
|
||||
return (brain as any).metadataIndex.idMapper.getInt(uuid)
|
||||
}
|
||||
|
||||
async function rebuild(): Promise<void> {
|
||||
await (brain as any).metadataIndex.rebuild()
|
||||
}
|
||||
|
||||
it('UUID→int mappings persist across a single metadata-index rebuild', async () => {
|
||||
const ids = [
|
||||
await addEntity('a', 1),
|
||||
await addEntity('b', 2),
|
||||
await addEntity('c', 3),
|
||||
await addEntity('d', 4),
|
||||
await addEntity('e', 5)
|
||||
]
|
||||
const before = ids.map(id => getInt(id))
|
||||
expect(before.every(i => typeof i === 'number' && (i as number) > 0)).toBe(true)
|
||||
|
||||
await rebuild()
|
||||
|
||||
const after = ids.map(id => getInt(id))
|
||||
expect(after).toEqual(before)
|
||||
})
|
||||
|
||||
it('mappings stay byte-for-byte stable across many consecutive rebuilds', async () => {
|
||||
const ids = [
|
||||
await addEntity('a', 1),
|
||||
await addEntity('b', 2),
|
||||
await addEntity('c', 3)
|
||||
]
|
||||
const before = ids.map(id => getInt(id))
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await rebuild()
|
||||
const after = ids.map(id => getInt(id))
|
||||
expect(after).toEqual(before)
|
||||
}
|
||||
})
|
||||
|
||||
it('entities added after rebuild get fresh monotonic ints (no collision with existing)', async () => {
|
||||
const priorIds = [
|
||||
await addEntity('a', 1),
|
||||
await addEntity('b', 2),
|
||||
await addEntity('c', 3)
|
||||
]
|
||||
const priorInts = priorIds.map(id => getInt(id) as number)
|
||||
const maxPrior = Math.max(...priorInts)
|
||||
|
||||
await rebuild()
|
||||
|
||||
const newId = await addEntity('d', 4)
|
||||
const newInt = getInt(newId) as number
|
||||
expect(newInt).toBeGreaterThan(maxPrior)
|
||||
// Prior entities' ints didn't drift.
|
||||
expect(priorIds.map(id => getInt(id))).toEqual(priorInts)
|
||||
})
|
||||
|
||||
it('removed entities leave a permanent hole — new entities never recycle the gap', async () => {
|
||||
const ids = [
|
||||
await addEntity('a', 1),
|
||||
await addEntity('b', 2),
|
||||
await addEntity('c', 3),
|
||||
await addEntity('d', 4),
|
||||
await addEntity('e', 5)
|
||||
]
|
||||
const beforeInts = ids.map(id => getInt(id) as number)
|
||||
const deletedId = ids[2]
|
||||
const deletedInt = beforeInts[2]
|
||||
const maxBefore = Math.max(...beforeInts)
|
||||
|
||||
await brain.delete(deletedId)
|
||||
expect(getInt(deletedId)).toBeUndefined()
|
||||
|
||||
const newId = await addEntity('f', 6)
|
||||
const newInt = getInt(newId) as number
|
||||
expect(newInt).not.toBe(deletedInt)
|
||||
expect(newInt).toBeGreaterThan(maxBefore)
|
||||
|
||||
// Surviving ids keep their ints across the deletion + the add.
|
||||
const survivors = ids.filter((_, i) => i !== 2)
|
||||
const survivorIntsBefore = beforeInts.filter((_, i) => i !== 2)
|
||||
expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore)
|
||||
|
||||
// Survivors' ints also survive a rebuild after the delete.
|
||||
await rebuild()
|
||||
expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore)
|
||||
// The deleted id is still gone after rebuild (no resurrection).
|
||||
expect(getInt(deletedId)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('clearAllIndexData() is the explicit nuclear path that DOES renumber', async () => {
|
||||
const id1 = await addEntity('a', 1)
|
||||
const id2 = await addEntity('b', 2)
|
||||
const priorInts = [getInt(id1) as number, getInt(id2) as number]
|
||||
expect(priorInts.every(i => i >= 1)).toBe(true)
|
||||
|
||||
// Nuclear recovery: explicit destructive op. The warning logged here is
|
||||
// the only documented way to invalidate the canonical int space.
|
||||
await (brain as any).metadataIndex.clearAllIndexData()
|
||||
|
||||
// Both UUIDs are gone from the mapper.
|
||||
expect(getInt(id1)).toBeUndefined()
|
||||
expect(getInt(id2)).toBeUndefined()
|
||||
|
||||
// The int counter restarted from 1: the next add() gets int 1.
|
||||
const idAfter = await addEntity('c', 3)
|
||||
expect(getInt(idAfter)).toBe(1)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue