From 62f6472fa09b00df9e310002fe453e58f232681a Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 10 Jun 2026 09:40:38 -0700 Subject: [PATCH] =?UTF-8?q?chore(8.0):=20delete=20vectorStore:mmap=20wirin?= =?UTF-8?q?g=20=E2=80=94=20dead=20in=20the=208.0=20provider=20world?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mmap-vector backend was a 2.x-era accelerator for the JS HNSW read path: a native provider registered under 'vectorStore:mmap' supplied a single-file mmap'd vector store, and JsHnswVectorIndex tried it before per-entity storage reads with lazy write-back migration. In the 8.0 provider world nothing registers that key — the native plugin's 3.0 line replaces the entire vector index under the 'vector' provider key (vectors live inside its own index file), and the open-core path never had an mmap provider. The wiring is unreachable; per the no-unwired-code rule it goes away: - src/hnsw/mmapVectorBackend.ts deleted (175 LOC) - wireMmapVectorBackend() + its init call site removed from brainy.ts - VectorStoreMmapProvider + VectorStoreMmapInstance contracts removed from plugin.ts - JsHnswVectorIndex loses the vectorBackend field, setVectorBackend(), and the mmap-first branches in getVectorSafe/preloadVectors — the storage + UnifiedCache path is now the single read path - tests/unit/hnsw/mmap-vector-backend.test.ts deleted The 7.x line keeps the wiring and got the capacity-NaN bugfix as 7.31.3 (see the platform handoff mmap thread). EntityIdMapperProvider stays — the connections codec and the id mapper still implement it. 1403/1403 tests, build clean. --- src/brainy.ts | 79 +------ src/hnsw/hnswIndex.ts | 98 +------- src/hnsw/mmapVectorBackend.ts | 174 -------------- src/plugin.ts | 49 ---- tests/unit/hnsw/mmap-vector-backend.test.ts | 242 -------------------- 5 files changed, 2 insertions(+), 640 deletions(-) delete mode 100644 src/hnsw/mmapVectorBackend.ts delete mode 100644 tests/unit/hnsw/mmap-vector-backend.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 21556db0..5248e217 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -40,10 +40,8 @@ import { PluginRegistry } from './plugin.js' import type { BrainyPlugin, BrainyPluginContext, - GraphCompressionProvider, - VectorStoreMmapProvider + GraphCompressionProvider } from './plugin.js' -import { MmapVectorBackend } from './hnsw/mmapVectorBackend.js' import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' import { RevisionConflictError } from './transaction/RevisionConflictError.js' @@ -604,15 +602,6 @@ export class Brainy implements BrainyInterface { this.graphIndex = graphIndex } - // Wire the mmap-vector backend (2.4.0 #2). When the vectorStore:mmap - // provider is registered AND the storage adapter resolves a real local - // path via getBinaryBlobPath(), open the mmap file there and inject the - // backend into JsHnswVectorIndex. Cloud adapters return null and skip silently; - // JsHnswVectorIndex's behaviour is then identical to pre-2.4.0 (per-entity reads - // from canonical storage). Failures are non-fatal — the index still - // works, just without the zero-copy fast path. - await this.wireMmapVectorBackend() - // Wire the connections codec (2.4.0 #3). When the graph:compression // provider is registered AND the metadata index exposes a stable // idMapper, inject a codec that encodes HNSW connections as @@ -9379,72 +9368,6 @@ export class Brainy implements BrainyInterface { await this.lazyRebuildPromise } - /** - * @description Open the mmap-vector backend and inject it into JsHnswVectorIndex. - * Called once during init after plugin activation, metadataIndex init, and - * graphIndex setup — so the idMapper is available and providers are wired. - * - * Activation conditions (ALL must hold; otherwise this is a no-op and the - * index falls back to per-entity storage reads, the pre-2.4.0 behaviour): - * 1. The `vectorStore:mmap` provider is registered (cortex registers this). - * 2. The storage adapter resolves a real local path via `getBinaryBlobPath()` - * — cloud adapters return null and the mmap layer is skipped silently. - * 3. The metadata index exposes its idMapper (a stable UUID↔int map). The - * mmap layout is keyed by these ints, so 2.4.0 #1 is the prerequisite. - * - * Vector dimensionality: read from `this.dimensions` if set; otherwise the - * default for the standard embedding model (384). The dim is fixed at - * file-creation time and cannot change later, so a wrong default would only - * matter on the very first init of a brand-new instance. After the first - * `add()` the in-memory `this.dimensions` is set, and any subsequent init - * (or a re-open) sees the correct value. - * - * Failures here are non-fatal: a debug-level log records the reason, and - * JsHnswVectorIndex continues without the mmap fast path. - */ - private async wireMmapVectorBackend(): Promise { - const provider = this.pluginRegistry.getProvider('vectorStore:mmap') - if (!provider) return - - const storageWithBlob = this.storage as unknown as { - getBinaryBlobPath?: (key: string) => string | null - } - const vectorPath = storageWithBlob.getBinaryBlobPath?.('_vectors/main') ?? null - if (!vectorPath) return - - const idMapper = this.metadataIndex.getIdMapper?.() - if (!idMapper) return - - const dim = this.dimensions ?? 384 - const initialCapacity = Math.max(idMapper.size * 2, 1024) - - try { - const backend = await MmapVectorBackend.open( - provider, - vectorPath, - dim, - initialCapacity, - idMapper - ) - this.index.setVectorBackend(backend) - if (!this.config.silent) { - console.log( - `[brainy] vector mmap backend wired (path=${vectorPath}, dim=${dim}, capacity=${initialCapacity})` - ) - } - } catch (error) { - // Common cases: dim mismatch from a prior init at a different dimension, - // permission errors, disk full, file corruption. Index keeps working via - // the per-entity storage path; flag the cause for diagnosis. - if (!this.config.silent) { - console.warn( - `[brainy] mmap-vector backend not wired (${(error as Error).message}); ` + - `falling back to per-entity vector reads` - ) - } - } - } - /** * @description Wire the HNSW connections codec (2.4.0 #3). Activates when * BOTH (a) the `graph:compression` provider is registered (cortex registers diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 0869b9de..f4d76876 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -17,7 +17,6 @@ import { prodLog } from '../utils/logger.js' import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js' import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js' import type { VectorIndexProvider } from '../plugin.js' -import { MmapVectorBackend } from './mmapVectorBackend.js' import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js' // Default HNSW parameters @@ -50,13 +49,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes // Always-adaptive caching - no "mode" concept, system adapts automatically - // Optional mmap-vector backend (2.4.0 #2). When set, the read paths - // (preloadVectors, getVectorSafe) try the mmap layer before storage, and - // on a storage hit they write back into the mmap slot — a lazy migration - // that converges upgraded installs to the fast path under live traffic. - // Null on cloud storage adapters (no local path) and pre-injection init. - private vectorBackend: MmapVectorBackend | null = null - // Optional connections codec (2.4.0 #3). When set, node-persist writes the // node's per-level connection sets as a single delta-varint-compressed // binary blob (typically ~4× smaller than the legacy JSON-UUID-array @@ -119,22 +111,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { this.useParallelization = useParallelization } - /** - * @description Inject (or detach) the mmap-vector backend. When set, the - * vector read paths (`preloadVectors`, `getVectorSafe`) try the mmap layer - * before storage and lazily write back on storage hits — converging an - * upgraded install to the zero-copy fast path without a migration step. - * Setting to null reverts to the existing per-entity read path. - * - * Lifecycle: brainy.ts wires this after plugin activation when (a) the - * `vectorStore:mmap` provider is registered AND (b) the storage adapter - * resolves a real local path via `getBinaryBlobPath()`. Cloud adapters - * leave it null; JsHnswVectorIndex's behaviour is then identical to pre-2.4.0. - */ - public setVectorBackend(backend: MmapVectorBackend | null): void { - this.vectorBackend = backend - } - /** * @description Inject (or detach) the connections codec. When set, node * persistence writes a delta-varint-compressed binary blob alongside an @@ -1180,23 +1156,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { const cacheKey = `hnsw:vector:${noun.id}` const vector = await this.unifiedCache.get(cacheKey, async () => { - // Try the mmap-vector backend first when wired — zero-copy slot read, - // no per-entity object hydration. - if (this.vectorBackend) { - const fromMmap = this.vectorBackend.readByUuid(noun.id) - if (fromMmap) { - this.unifiedCache.set( - cacheKey, - fromMmap, - 'vectors', - fromMmap.length * 4, - 50 - ) - return fromMmap - } - } - - // Storage fallback — the canonical source of truth. if (!this.storage) { throw new Error('Storage not available for vector loading') } @@ -1206,18 +1165,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { throw new Error(`Vector not found for noun ${noun.id}`) } - // Lazy migration: write back into the mmap slot so the next read is - // mmap-fast. Idempotent + resumable; a write failure is non-fatal — - // the field still serves this read, and the next storage hit re-tries - // the migration. (See mmapVectorBackend.ts for the contract.) - if (this.vectorBackend) { - try { - this.vectorBackend.writeByUuid(noun.id, loaded) - } catch (error) { - prodLog.debug(`MmapVectorBackend write-back failed for ${noun.id}`, error) - } - } - // Add to UnifiedCache with cost-aware eviction // This competes fairly with Graph and Metadata indexes this.unifiedCache.set( @@ -1275,50 +1222,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider { private async preloadVectors(nodeIds: string[]): Promise { if (nodeIds.length === 0) return - // mmap-vector backend fast path: batched O(1) slot reads + lazy - // write-back migration on miss. Cache populated from both layers so - // subsequent in-session reads are O(1) memory. - if (this.vectorBackend) { - // Filter out ids already cached so we never re-read. - const uncachedIds: string[] = [] - for (const id of nodeIds) { - if (this.unifiedCache.getSync(`hnsw:vector:${id}`) === undefined) { - uncachedIds.push(id) - } - } - if (uncachedIds.length === 0) return - - const fromMmap = this.vectorBackend.readBatchByUuid(uncachedIds) - const misses: string[] = [] - for (let i = 0; i < uncachedIds.length; i++) { - const id = uncachedIds[i] - const v = fromMmap[i] - if (v) { - this.unifiedCache.set(`vector:${id}`, v, 'vectors', v.length * 4, 50) - } else { - misses.push(id) - } - } - if (misses.length === 0) return - - // Storage fallback for the misses, with concurrent lazy write-back. - await Promise.all(misses.map(async (id) => { - if (!this.storage) return - const vector = await this.storage.getNounVector(id) - if (!vector) return - if (this.vectorBackend) { - try { - this.vectorBackend.writeByUuid(id, vector) - } catch (error) { - prodLog.debug(`MmapVectorBackend write-back failed for ${id}`, error) - } - } - this.unifiedCache.set(`vector:${id}`, vector, 'vectors', vector.length * 4, 50) - })) - return - } - - // Legacy path: per-entity storage load via UnifiedCache request coalescing. + // Per-entity storage load via UnifiedCache request coalescing. const promises = nodeIds.map(async (id) => { const cacheKey = `hnsw:vector:${id}` return this.unifiedCache.get(cacheKey, async () => { diff --git a/src/hnsw/mmapVectorBackend.ts b/src/hnsw/mmapVectorBackend.ts deleted file mode 100644 index 0e5cba71..00000000 --- a/src/hnsw/mmapVectorBackend.ts +++ /dev/null @@ -1,174 +0,0 @@ -/** - * @module hnsw/mmapVectorBackend - * @description Disk-resident vector cache backing JsHnswVectorIndex, wrapping the - * `vectorStore:mmap` provider (cortex's NativeMmapVectorStore in production). - * - * **Why this exists.** Brainy's per-entity vector storage hits two ceilings as - * datasets grow: (1) every `JsHnswVectorIndex.preloadVectors` does a sequential - * `storage.getNounVector` per id (no batch), and (2) cloud and FS storage both - * serialize each vector through JSON / msgpack, blocking zero-copy reads. An - * mmap-backed single-file vector store, addressed by the stable int from - * `EntityIdMapper`, gives O(1) batch reads + OS-page-cache prefetch and is the - * foundation 2.4.0 #2 needs. - * - * **Lazy migration on miss.** First read of an unmigrated UUID falls back to - * `storage.getNounVector` and writes the bytes into the mmap slot before - * returning — idempotent, resumable, and lets an upgraded install converge to - * the fast path under live traffic without a big-bang migration step. The - * per-entity store remains the canonical source of truth; nothing in the per- - * entity layer is rewritten or deleted here. - * - * **Local-FS only in 2.4.0.** Activated when the storage adapter exposes a - * real local path via `getBinaryBlobPath()`. Cloud adapters (S3 / R2 / GCS) - * return null and the backend is not constructed, so JsHnswVectorIndex falls back to - * the existing per-entity read path. Chunked vector segments for remote - * storage are the planned 2.4.0 #2 follow-up. - */ - -import { mkdir } from 'node:fs/promises' -import { dirname } from 'node:path' -import type { Vector } from '../coreTypes.js' -import type { - EntityIdMapperProvider, - VectorStoreMmapInstance, - VectorStoreMmapProvider -} from '../plugin.js' - -/** - * @description Bridge between brainy's UUID-keyed vector API and cortex's - * int-slot mmap vector store. Translates UUIDs to stable int slots via the - * `EntityIdMapper`, grows the file on demand, and handles read misses by - * surfacing `null` (JsHnswVectorIndex's caller does the lazy storage fallback + - * write-back). The class never touches per-entity storage directly — it owns - * only the mmap layer. - */ -export class MmapVectorBackend { - private readonly store: VectorStoreMmapInstance - private readonly idMapper: EntityIdMapperProvider - - private constructor(store: VectorStoreMmapInstance, idMapper: EntityIdMapperProvider) { - this.store = store - this.idMapper = idMapper - } - - /** - * @description Open or create the mmap vector file at `path`. If the file - * does not exist, it is created with the given dim + initial capacity. If - * the directory containing `path` does not exist, it is created recursively - * (the cortex provider does not auto-create parent directories). - * @param provider - The `vectorStore:mmap` provider (cortex - * `NativeMmapVectorStore` in production). - * @param path - Absolute path to the vector file. Derived by the caller - * from `storage.getBinaryBlobPath()` on local-FS adapters; cloud adapters - * return null and this method is not called. - * @param dim - Vector dimensionality. Must match the embedding model. - * @param initialCapacity - Slot capacity to allocate when creating a new - * file. The file grows automatically on writes beyond capacity. - * @param idMapper - Stable UUID↔int mapper (post-2.4.0 #1 semantics — never - * renumbers across rebuild). The mmap slots are addressed by these ints, - * so id stability is the load-bearing invariant of the layout. - * @returns A ready-to-use backend. - */ - static async open( - provider: VectorStoreMmapProvider, - path: string, - dim: number, - initialCapacity: number, - idMapper: EntityIdMapperProvider - ): Promise { - await mkdir(dirname(path), { recursive: true }) - let store: VectorStoreMmapInstance - try { - store = provider.open(path) - } catch { - store = provider.create(path, dim, Math.max(initialCapacity, 16)) - } - return new MmapVectorBackend(store, idMapper) - } - - /** Vector dimensionality of the underlying mmap file. */ - get dim(): number { - return this.store.dim - } - - /** - * @description Read a vector by UUID. Returns `null` for any of: - * (1) UUID not yet assigned an int (entity never written through this - * backend or storage), (2) int beyond the highest written slot, or - * (3) the underlying mmap read throwing (corrupt / out-of-bounds). - * The null path is the caller's signal to fall back to per-entity storage - * and trigger a write-back migration. - */ - readByUuid(uuid: string): Vector | null { - const intId = this.idMapper.getInt(uuid) - if (intId === undefined || intId >= this.store.count) return null - try { - return this.store.readVector(intId) as unknown as Vector - } catch { - return null - } - } - - /** - * @description Batch read vectors by UUID. Returns an array aligned with - * `uuids` where each entry is the vector or `null` for a miss. Order- - * preserving. Internally collapses to a single batched mmap read of only the - * indices that resolve to a written slot, then redistributes the flat output - * back to the caller's order — so missing slots cost only a couple of map - * lookups, not a full N-element read. - */ - readBatchByUuid(uuids: string[]): (Vector | null)[] { - const indices: number[] = [] - const slotByUuid = new Map() - for (const uuid of uuids) { - const intId = this.idMapper.getInt(uuid) - if (intId !== undefined && intId < this.store.count) { - slotByUuid.set(uuid, indices.length) - indices.push(intId) - } - } - if (indices.length === 0) return uuids.map(() => null) - const flat = this.store.readVectorsBatch(indices) - const dim = this.store.dim - return uuids.map(uuid => { - const slot = slotByUuid.get(uuid) - if (slot === undefined) return null - return flat.slice(slot * dim, (slot + 1) * dim) as unknown as Vector - }) - } - - /** - * @description Write a vector at the slot for `uuid`. Assigns a new int if - * the UUID is unseen (stable, append-only). Grows the underlying file (slot - * capacity doubled until the target index fits) before writing past - * capacity, so the caller need not worry about file sizing. - */ - writeByUuid(uuid: string, vector: Vector): void { - const intId = this.idMapper.getOrAssign(uuid) - if (intId >= this.store.capacity) { - let newCap = Math.max(this.store.capacity, 16) - while (intId >= newCap) newCap *= 2 - this.store.resize(newCap) - } - this.store.writeVector(intId, Array.from(vector as ArrayLike)) - } - - /** - * @description madvise(WILLNEED) hint for the slots backing `uuids`. Cheap; - * silently drops UUIDs not yet in the mmap. Used by HNSW search to overlap - * disk I/O with graph traversal. - */ - prefetchByUuid(uuids: string[]): void { - const indices: number[] = [] - for (const uuid of uuids) { - const intId = this.idMapper.getInt(uuid) - if (intId !== undefined && intId < this.store.count) indices.push(intId) - } - if (indices.length > 0) this.store.prefetch(indices) - } - - /** msync — flush dirty pages to disk. */ - flush(): void { - this.store.flush() - } -} diff --git a/src/plugin.ts b/src/plugin.ts index fedfcb8b..89a57d0f 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -285,55 +285,6 @@ export interface GraphCompressionProvider { decode(data: Buffer): number[] } -/** - * The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed - * vector file. Brainy's JsHnswVectorIndex uses the static `.create()` / `.open()` - * factories to open a single-file store at a derived path on disk-resident - * storage adapters; non-FS adapters with no local-path support skip the mmap - * layer and fall back to per-entity vector reads. - * - * Cortex registers `NativeMmapVectorStore` as this provider — a Rust mmap'd - * f32 array with a header, batch read/write APIs, and madvise prefetch. Slot - * indices are the stable ints from `EntityIdMapper`; lazy migration on read - * miss writes back from per-entity storage into the slot, converging an - * upgraded install to the fast path without a big-bang step. - */ -export interface VectorStoreMmapProvider { - /** Create a new vector file with pre-allocated slot capacity. */ - create(path: string, dim: number, capacity: number): VectorStoreMmapInstance - /** Open an existing vector file for read + write. */ - open(path: string): VectorStoreMmapInstance - /** Open an existing vector file for read-only access. */ - openReadOnly(path: string): VectorStoreMmapInstance -} - -/** - * An open mmap-vector-store instance — the surface brainy's JsHnswVectorIndex consumes - * for batch vector reads + lazy migration from per-entity storage on miss. - */ -export interface VectorStoreMmapInstance { - /** Number of vectors written (highest written index + 1). */ - readonly count: number - /** Maximum slot capacity before resize is needed. */ - readonly capacity: number - /** Vector dimensionality. */ - readonly dim: number - /** Write a single vector at `index`. Vector is f64 in JS, stored as f32. */ - writeVector(index: number, vector: number[]): void - /** Write a flat batch starting at `startIndex`. Returns vectors written. */ - writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number - /** Read a single vector at `index`. Returns f64[]. */ - readVector(index: number): number[] - /** Read multiple vectors by index. Returns flat f64[] of length n × dim. */ - readVectorsBatch(indices: number[]): number[] - /** Prefetch via madvise(WILLNEED); no-op on unsupported platforms. */ - prefetch(indices: number[]): void - /** Grow the vector file to a larger slot capacity. Re-maps the file. */ - resize(newCapacity: number): void - /** msync — flush pending writes to disk. */ - flush(): void -} - /** * Storage adapter factory — plugins register these to provide * new storage backends that users reference by name. diff --git a/tests/unit/hnsw/mmap-vector-backend.test.ts b/tests/unit/hnsw/mmap-vector-backend.test.ts deleted file mode 100644 index 257d50c4..00000000 --- a/tests/unit/hnsw/mmap-vector-backend.test.ts +++ /dev/null @@ -1,242 +0,0 @@ -/** - * @module hnsw/mmapVectorBackend.test - * @description Unit tests for the `MmapVectorBackend` bridge — the brainy-side - * wrapper that translates UUID-keyed vector reads/writes into stable int slot - * ops against an `vectorStore:mmap` provider. - * - * Mocks the provider so the tests run without cortex installed (cortex is a - * downstream consumer of brainy, not a dev dep). The real integration with - * cortex's `NativeMmapVectorStore` is exercised when cortex 2.4.0 picks up - * this brainy release and re-runs its cross-language parity suite. - * - * Coverage: - * 1. Open-then-write-then-read round-trips for a single vector. - * 2. Batch reads return an array aligned with the input UUIDs, with `null` - * entries for misses interleaved among hits — order preserved. - * 3. Slot assignment is stable across multiple writes for the same UUID - * (no re-slot, no overwrite of an adjacent slot). - * 4. Writes beyond the initial capacity grow the file (doubling) without - * losing the vectors already written. - * 5. `readByUuid` returns `null` for both unknown UUIDs and UUIDs in the map - * whose slot has not yet been written. - * 6. Open is idempotent — opening an already-existing file reuses it. - */ - -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { tmpdir } from 'node:os' -import { mkdtemp, rm } from 'node:fs/promises' -import { join } from 'node:path' -import { MmapVectorBackend } from '../../../src/hnsw/mmapVectorBackend.js' -import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js' -import type { - VectorStoreMmapInstance, - VectorStoreMmapProvider -} from '../../../src/plugin.js' - -/** - * Minimal storage stub for the EntityIdMapper. The mapper only touches storage - * in init/flush; for these tests the mapper starts empty and is never flushed. - */ -const stubStorage = { - getMetadata: async () => undefined, - saveMetadata: async () => {}, - getNouns: async () => ({ totalCount: 0, items: [] }) -} as any - -/** - * Pure in-memory mmap store. Mirrors cortex's NativeMmapVectorStore surface - * just closely enough to exercise the backend's contract — no real mmap, no - * file I/O, no f32 round-trip narrowing (the precision check is out of scope - * here; that's covered by cortex's parity suite). - */ -class MockMmapStore implements VectorStoreMmapInstance { - private readonly vectors: Array = [] - private highestWritten = -1 - constructor( - public readonly dim: number, - private _capacity: number - ) {} - get count(): number { - return this.highestWritten + 1 - } - get capacity(): number { - return this._capacity - } - writeVector(index: number, vector: number[]): void { - if (index >= this._capacity) { - throw new Error(`Slot ${index} >= capacity ${this._capacity}`) - } - if (vector.length !== this.dim) { - throw new Error(`Dim mismatch: expected ${this.dim}, got ${vector.length}`) - } - this.vectors[index] = [...vector] - if (index > this.highestWritten) this.highestWritten = index - } - writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number { - if (vectorsFlat.length % this.dim !== 0) { - throw new Error('vectorsFlat length not a multiple of dim') - } - const n = vectorsFlat.length / this.dim - for (let i = 0; i < n; i++) { - this.writeVector(startIndex + i, vectorsFlat.slice(i * this.dim, (i + 1) * this.dim)) - } - return n - } - readVector(index: number): number[] { - const v = this.vectors[index] - if (!v) throw new Error(`Slot ${index} not written`) - return [...v] - } - readVectorsBatch(indices: number[]): number[] { - const flat: number[] = [] - for (const i of indices) { - const v = this.vectors[i] - if (!v) throw new Error(`Slot ${i} not written`) - for (let k = 0; k < this.dim; k++) flat.push(v[k]) - } - return flat - } - prefetch(_indices: number[]): void { - /* no-op in mock */ - } - resize(newCapacity: number): void { - if (newCapacity < this._capacity) throw new Error('Cannot shrink') - this._capacity = newCapacity - } - flush(): void { - /* no-op in mock */ - } -} - -/** - * Mock provider — keeps one MockMmapStore per path. open() throws if the path - * doesn't exist yet (matches cortex semantics: open() is for existing files - * only); create() throws if it does (cortex doesn't, but the brainy backend - * does open-first-then-create, so the throw path is exercised). - */ -class MockMmapProvider implements VectorStoreMmapProvider { - private files = new Map() - create(path: string, dim: number, capacity: number): VectorStoreMmapInstance { - if (this.files.has(path)) throw new Error(`File exists at ${path}`) - const store = new MockMmapStore(dim, capacity) - this.files.set(path, store) - return store - } - open(path: string): VectorStoreMmapInstance { - const store = this.files.get(path) - if (!store) throw new Error(`No file at ${path}`) - return store - } - openReadOnly(path: string): VectorStoreMmapInstance { - return this.open(path) - } - /** Test helper — peek at the underlying store. */ - _peek(path: string): MockMmapStore | undefined { - return this.files.get(path) - } -} - -describe('MmapVectorBackend (2.4.0 #2 — wraps vectorStore:mmap provider)', () => { - let dir: string - let path: string - let idMapper: EntityIdMapper - let provider: MockMmapProvider - - beforeEach(async () => { - dir = await mkdtemp(join(tmpdir(), 'brainy-mmap-vec-')) - path = join(dir, 'vectors.bin') - idMapper = new EntityIdMapper({ storage: stubStorage }) - await idMapper.init() - provider = new MockMmapProvider() - }) - - afterEach(async () => { - await rm(dir, { recursive: true, force: true }).catch(() => {}) - }) - - it('open creates a new file when none exists, then round-trips a vector by UUID', async () => { - const backend = await MmapVectorBackend.open(provider, path, 4, 16, idMapper) - expect(backend.dim).toBe(4) - - backend.writeByUuid('alpha', [1, 2, 3, 4]) - expect(backend.readByUuid('alpha')).toEqual([1, 2, 3, 4]) - }) - - it('reads return null for unknown UUIDs and for UUIDs in the map but not yet written', async () => { - const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) - - // Unknown — never seen by the mapper. - expect(backend.readByUuid('ghost')).toBeNull() - - // Known to the mapper but no slot ever written. We assign through the - // mapper directly so the backend itself has not touched the slot. - idMapper.getOrAssign('reserved') - expect(backend.readByUuid('reserved')).toBeNull() - }) - - it('batch read returns an array aligned to the input UUIDs (nulls preserved in place)', async () => { - const backend = await MmapVectorBackend.open(provider, path, 3, 16, idMapper) - backend.writeByUuid('a', [1, 1, 1]) - backend.writeByUuid('b', [2, 2, 2]) - backend.writeByUuid('c', [3, 3, 3]) - - // Interleave hits + a never-seen UUID + a duplicate hit. Order preserved. - const result = backend.readBatchByUuid(['b', 'missing', 'a', 'c', 'missing-too', 'b']) - expect(result).toEqual([ - [2, 2, 2], - null, - [1, 1, 1], - [3, 3, 3], - null, - [2, 2, 2] - ]) - }) - - it('writes for the same UUID land in the same slot (stable id, no re-slotting)', async () => { - const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) - backend.writeByUuid('persistent', [1, 1]) - const slotAfterFirstWrite = idMapper.getInt('persistent') - - backend.writeByUuid('persistent', [9, 9]) // overwrite the same slot - expect(idMapper.getInt('persistent')).toBe(slotAfterFirstWrite) - expect(backend.readByUuid('persistent')).toEqual([9, 9]) - - // Another UUID gets a different slot, and the first vector is undisturbed. - backend.writeByUuid('other', [5, 5]) - expect(idMapper.getInt('other')).not.toBe(slotAfterFirstWrite) - expect(backend.readByUuid('persistent')).toEqual([9, 9]) - expect(backend.readByUuid('other')).toEqual([5, 5]) - }) - - it('grows the file (doubling) when a write lands beyond capacity, without losing prior data', async () => { - // Start at the smallest sane initial capacity (clamped to 16 by the backend). - const backend = await MmapVectorBackend.open(provider, path, 2, 1, idMapper) - const store = provider._peek(path)! - expect(store.capacity).toBe(16) // backend floor - - // Write 20 vectors so capacity must double at least once (16 → 32). - const uuids: string[] = [] - for (let i = 0; i < 20; i++) { - const uuid = `u-${i}` - uuids.push(uuid) - backend.writeByUuid(uuid, [i, i * 2]) - } - expect(store.capacity).toBeGreaterThanOrEqual(32) - - // Every vector survived the growth — no slot got overwritten or lost. - for (let i = 0; i < uuids.length; i++) { - expect(backend.readByUuid(uuids[i])).toEqual([i, i * 2]) - } - }) - - it('opens an existing file idempotently when the second open hits the same path', async () => { - // First open creates. - const backend1 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) - backend1.writeByUuid('persisted', [7, 7]) - - // Second open against the same path. The provider's create() throws on - // collision; the backend's open-first behaviour means we reuse the file. - const backend2 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) - expect(backend2.readByUuid('persisted')).toEqual([7, 7]) - }) -})