From 01cf4b1b7eb5957f8817350a58f6310181c7433b Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 28 May 2026 09:45:22 -0700 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20stable=20EntityIdMapper=20=E2=80=94?= =?UTF-8?q?=20rebuild()=20no=20longer=20renumbers=20UUID=E2=86=92int?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously metadataIndex.rebuild() called idMapper.clear() which reset nextId to 1 and renumbered every UUID by re-insertion order. Any consumer that had persisted int-keyed data against the old map was silently invalidated — and 2.4.0's vector mmap store (#20), graph link compression (#21), and column-store JS↔native interchange all need persisted int indices that survive a rebuild. Remove the unconditional clear() in rebuild(). The rebuild already re-iterates every entity via idMapper.getOrAssign(uuid), which returns the existing int unchanged for known UUIDs. Stale UUID→int entries for entities no longer in storage persist as harmless memory overhead; a dedicated prune step can be added if it ever matters. clearAllIndexData() — the explicit nuclear recovery path — keeps its existing idMapper.clear() call (renumbering is intentional there), and now logs a prodLog.warn making it explicit that any persisted int-keyed data is invalidated and must be rebuilt from canonical sources. Strengthened the EntityIdMapper class JSDoc to document the stability guarantee as a contract — append-only getOrAssign, monotonic nextId, remove() leaves permanent holes, rebuild() never renumbers, only clear() does. Added tests/regression/entity-id-mapper-stability.test.ts pinning down the five-point contract: (1) single-rebuild stability; (2) many-rebuild stability; (3) post-rebuild adds get fresh monotonic ints; (4) removes leave permanent holes — new entities never recycle; (5) clearAllIndexData() explicitly renumbers (the documented destructive path). Foundation for 2.4.0 #2-#4. Full test suite (62 files, 1417 tests) green. --- src/utils/entityIdMapper.ts | 34 +++- src/utils/metadataIndex.ts | 24 ++- .../entity-id-mapper-stability.test.ts | 161 ++++++++++++++++++ 3 files changed, 209 insertions(+), 10 deletions(-) create mode 100644 tests/regression/entity-id-mapper-stability.test.ts diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index ff67a4e9..5333c025 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -1,14 +1,34 @@ /** - * EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs for roaring bitmaps + * EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs. * - * Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings as entity IDs. - * This class provides efficient bidirectional mapping with persistence support. + * Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings + * as canonical entity IDs. This class provides efficient O(1) bidirectional + * mapping with persistence — and, importantly, a stability guarantee that any + * persisted int-keyed data can rely on. + * + * **Stability guarantee (the foundation 2.4.0 vector-mmap, graph-link-compression, + * and column-store interchange all key off):** + * + * - `getOrAssign(uuid)` is **append-only**: once a UUID is assigned an int, the + * mapping never changes. Subsequent `getOrAssign` calls for the same UUID + * return the same int. + * - `nextId` is **monotonically increasing**. New UUIDs always get an int greater + * than any previously assigned, so a removed-then-re-added UUID is treated as + * a fresh entity (and gets a fresh int — there is no automatic "revive"). + * - `remove(uuid)` removes the mapping but does **not** decrement `nextId` or + * recycle the int. The removed int becomes a permanent hole in `intToUuid` — + * downstream consumers seeing `getUuid(int) === undefined` know the entity + * was deleted. + * - A metadata-index `rebuild()` does **not** clear the mapper (the rebuild path + * re-iterates entities via `getOrAssign`, which returns existing ints unchanged). + * Only the explicit `clear()` method renumbers — used by `clearAllIndexData()` + * as the nuclear recovery path with a documented warning. * * Features: - * - O(1) lookup in both directions - * - Persistent storage via storage adapter - * - Atomic counter for next ID - * - Serialization/deserialization support + * - O(1) lookup in both directions. + * - Persistent storage via storage adapter. + * - Atomic, monotonic, append-only int counter. + * - Serialization/deserialization support. * * @module utils/entityIdMapper */ diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index a83d2e2a..5be89566 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2635,13 +2635,20 @@ export class MetadataIndexManager implements MetadataIndexProvider { this.verbCountsByTypeFixed.fill(0) this.typeFieldAffinity.clear() - // Clear EntityIdMapper + // Clear EntityIdMapper. This is the explicit destructive path: the caller + // asked for nuclear recovery of a corrupted index, so renumbering UUIDs is + // intentional. Persisted int-keyed data (vector-mmap slots, graph + // link-compression encodings) is invalidated by this op — the warning + // below makes that explicit. Rebuild on its own does NOT clear the mapper. await this.idMapper.clear() // Clear chunk manager cache this.chunkManager.clearCache() prodLog.info(`✅ Cleared ${deletedCount} field indexes and all in-memory state`) + prodLog.warn('⚠️ EntityIdMapper was cleared — any persisted int-keyed data ' + + '(vector mmap slots, graph link-compression encodings, etc.) is now stale ' + + 'and must be rebuilt from canonical sources.') prodLog.info('⚠️ Run brain.index.rebuild() to recreate the index from entity data') } @@ -3032,8 +3039,19 @@ export class MetadataIndexManager implements MetadataIndexProvider { prodLog.info(`Cleared ${existingFields.length} field indexes from storage`) } - // Clear EntityIdMapper to start fresh - await this.idMapper.clear() + // EntityIdMapper is intentionally NOT cleared here. Rebuild re-iterates + // every entity in storage and calls idMapper.getOrAssign(uuid), which + // returns the existing int for known UUIDs (no renumbering). This is the + // foundational stability guarantee — vector-mmap slot indices, graph + // link-compression encodings, and any other persisted int-keyed data + // remain valid across a rebuild. Previously this line reset nextId to 1 + // and renumbered every UUID by re-insertion order, silently breaking + // any consumer that had persisted int-keyed data against the old map. + // Stale entries for UUIDs no longer in storage persist (harmless memory + // overhead); a dedicated prune step can be added if it ever matters. + // The destructive wipe is still available via clearAllIndexData() → + // idMapper.clear(), which is the explicit "recovery" path with the + // appropriate warning about invalidating persisted int-keyed data. // Clear chunk manager cache this.chunkManager.clearCache() diff --git a/tests/regression/entity-id-mapper-stability.test.ts b/tests/regression/entity-id-mapper-stability.test.ts new file mode 100644 index 00000000..4ba2bda6 --- /dev/null +++ b/tests/regression/entity-id-mapper-stability.test.ts @@ -0,0 +1,161 @@ +/** + * Regression test: EntityIdMapper stability across rebuild. + * + * The foundation 2.4.0 (vector mmap store, graph link compression, column-store + * JS↔native interchange) all key off UUID→int mappings that **must not change** + * across a metadata-index rebuild. Previously `metadataIndex.rebuild()` called + * `idMapper.clear()` which reset `nextId` to 1 and renumbered every UUID by + * re-insertion order, silently invalidating any consumer that had persisted + * int-keyed data against the old map. + * + * This test pins down the stability contract: + * + * 1. UUID→int mappings persist across a single rebuild. + * 2. Mappings persist across many consecutive rebuilds. + * 3. New entities added after rebuild get fresh ints greater than any prior + * assignment — no collisions with existing UUIDs' ints. + * 4. Removed entities leave a permanent hole — new entities don't recycle the + * gap, even across a rebuild. + * 5. `clearAllIndexData()` is the explicit, intentional nuclear path — it DOES + * renumber. This is the only documented way to invalidate the int space, and + * a warning is logged so consumers know persisted int-keyed data is now stale. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' + +const DIM = 384 +const makeVec = (seed = 1) => + new Float32Array(DIM).map((_, i) => ((i + seed) % DIM) / DIM) + +describe('EntityIdMapper stability (foundation for 2.4.0)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ storage: { type: 'memory' }, silent: true }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + async function addEntity(name: string, seed: number): Promise { + return brain.add({ + data: name, + vector: makeVec(seed), + type: 'thing' as any, + metadata: { name } + }) + } + + function getInt(uuid: string): number | undefined { + return (brain as any).metadataIndex.idMapper.getInt(uuid) + } + + async function rebuild(): Promise { + await (brain as any).metadataIndex.rebuild() + } + + it('UUID→int mappings persist across a single metadata-index rebuild', async () => { + const ids = [ + await addEntity('a', 1), + await addEntity('b', 2), + await addEntity('c', 3), + await addEntity('d', 4), + await addEntity('e', 5) + ] + const before = ids.map(id => getInt(id)) + expect(before.every(i => typeof i === 'number' && (i as number) > 0)).toBe(true) + + await rebuild() + + const after = ids.map(id => getInt(id)) + expect(after).toEqual(before) + }) + + it('mappings stay byte-for-byte stable across many consecutive rebuilds', async () => { + const ids = [ + await addEntity('a', 1), + await addEntity('b', 2), + await addEntity('c', 3) + ] + const before = ids.map(id => getInt(id)) + + for (let i = 0; i < 5; i++) { + await rebuild() + const after = ids.map(id => getInt(id)) + expect(after).toEqual(before) + } + }) + + it('entities added after rebuild get fresh monotonic ints (no collision with existing)', async () => { + const priorIds = [ + await addEntity('a', 1), + await addEntity('b', 2), + await addEntity('c', 3) + ] + const priorInts = priorIds.map(id => getInt(id) as number) + const maxPrior = Math.max(...priorInts) + + await rebuild() + + const newId = await addEntity('d', 4) + const newInt = getInt(newId) as number + expect(newInt).toBeGreaterThan(maxPrior) + // Prior entities' ints didn't drift. + expect(priorIds.map(id => getInt(id))).toEqual(priorInts) + }) + + it('removed entities leave a permanent hole — new entities never recycle the gap', async () => { + const ids = [ + await addEntity('a', 1), + await addEntity('b', 2), + await addEntity('c', 3), + await addEntity('d', 4), + await addEntity('e', 5) + ] + const beforeInts = ids.map(id => getInt(id) as number) + const deletedId = ids[2] + const deletedInt = beforeInts[2] + const maxBefore = Math.max(...beforeInts) + + await brain.delete(deletedId) + expect(getInt(deletedId)).toBeUndefined() + + const newId = await addEntity('f', 6) + const newInt = getInt(newId) as number + expect(newInt).not.toBe(deletedInt) + expect(newInt).toBeGreaterThan(maxBefore) + + // Surviving ids keep their ints across the deletion + the add. + const survivors = ids.filter((_, i) => i !== 2) + const survivorIntsBefore = beforeInts.filter((_, i) => i !== 2) + expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore) + + // Survivors' ints also survive a rebuild after the delete. + await rebuild() + expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore) + // The deleted id is still gone after rebuild (no resurrection). + expect(getInt(deletedId)).toBeUndefined() + }) + + it('clearAllIndexData() is the explicit nuclear path that DOES renumber', async () => { + const id1 = await addEntity('a', 1) + const id2 = await addEntity('b', 2) + const priorInts = [getInt(id1) as number, getInt(id2) as number] + expect(priorInts.every(i => i >= 1)).toBe(true) + + // Nuclear recovery: explicit destructive op. The warning logged here is + // the only documented way to invalidate the canonical int space. + await (brain as any).metadataIndex.clearAllIndexData() + + // Both UUIDs are gone from the mapper. + expect(getInt(id1)).toBeUndefined() + expect(getInt(id2)).toBeUndefined() + + // The int counter restarted from 1: the next add() gets int 1. + const idAfter = await addEntity('c', 3) + expect(getInt(idAfter)).toBe(1) + }) +}) -- 2.49.1 From c18ccbb4979e03c2ddcdb3d08b60cfe9bfe4d678 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 28 May 2026 10:10:05 -0700 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20mmap-vector=20backend=20wiring=20?= =?UTF-8?q?=E2=80=94=20HNSWIndex=20consumes=20vectorStore:mmap=20(2.4.0=20?= =?UTF-8?q?#2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cortex already registers the vectorStore:mmap provider (its Rust NativeMmapVectorStore), but brainy has never consumed it — preloadVectors and getVectorSafe still go straight to storage.getNounVector for every id, even when an mmap layer is available. This wires the consumer end. Architecture: - NEW MmapVectorBackend (src/hnsw/mmapVectorBackend.ts) — bridges brainy's UUID-keyed vector reads to a int-slot mmap file via the vectorStore:mmap provider. Slots are addressed by the stable int id from the post-2.4.0 #1 EntityIdMapper (the foundation this depends on). Auto-grows the file (doubling) when a write lands beyond capacity, so HNSWIndex never has to think about sizing. The class never touches per-entity storage — it owns only the mmap layer. - HNSWIndex changes — adds a vectorBackend field + a setVectorBackend setter. The vector read paths (preloadVectors, getVectorSafe) try the mmap layer first; on a storage fallback hit, they LAZILY write back into the mmap slot. An upgraded install converges to the zero-copy fast path under live traffic — no big-bang migration step. The legacy per-entity path is preserved and still used when no backend is set. - brainy.ts wiring — a new private wireMmapVectorBackend() runs once during init, after plugin activation + metadataIndex setup. It activates the backend only when (a) the vectorStore:mmap provider is registered, (b) the storage adapter resolves a real local path via getBinaryBlobPath(), and (c) the metadata index exposes its idMapper. Cloud adapters return null on (b) and the backend is silently skipped; HNSWIndex's behaviour is then identical to pre-2.4.0. - Provider interfaces in plugin.ts — VectorStoreMmapProvider and VectorStoreMmapInstance document the contract cortex's class fulfils (the class IS the provider — static factory methods). Brainy depends on the interfaces, not on cortex; the structural match is verified when cortex 2.4.0 picks up this brainy release. Tests (1428 total, +11 vs pre-2.4.0): - tests/unit/hnsw/mmap-vector-backend.test.ts — 6 unit tests with an in-memory mock provider. Covers round-trip, batch reads with interleaved misses, slot stability (no re-slotting on overwrite), file growth without data loss, idempotent open, and null returns for unwritten slots. The real perf integration with cortex's NativeMmapVectorStore is exercised when cortex 2.4.0 wires this in. - tests/unit/utils/entity-id-mapper-stability.test.ts — moved here from tests/regression/ (which is NOT in the unit-config include glob, so the five #23 tests were not actually being run by npm test). The unit config matches tests/unit/**/*.test.ts. The 2.4.0 #2 follow-up will be the chunked-segment layout for remote storage adapters (S3 / R2 / GCS) where a single growing file doesn't fit immutable objects. For 2.4.0 release: local-FS only. --- src/brainy.ts | 78 +++++- src/hnsw/hnswIndex.ts | 99 ++++++- src/hnsw/mmapVectorBackend.ts | 174 +++++++++++++ src/plugin.ts | 49 ++++ tests/unit/hnsw/mmap-vector-backend.test.ts | 242 ++++++++++++++++++ .../utils}/entity-id-mapper-stability.test.ts | 2 +- 6 files changed, 640 insertions(+), 4 deletions(-) create mode 100644 src/hnsw/mmapVectorBackend.ts create mode 100644 tests/unit/hnsw/mmap-vector-backend.test.ts rename tests/{regression => unit/utils}/entity-id-mapper-stability.test.ts (99%) diff --git a/src/brainy.ts b/src/brainy.ts index 8819824e..7b18013d 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -37,7 +37,8 @@ import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' import { PluginRegistry } from './plugin.js' -import type { BrainyPlugin, BrainyPluginContext } from './plugin.js' +import type { BrainyPlugin, BrainyPluginContext, VectorStoreMmapProvider } from './plugin.js' +import { MmapVectorBackend } from './hnsw/mmapVectorBackend.js' import { TransactionManager } from './transaction/TransactionManager.js' import { ValidationConfig, @@ -559,6 +560,15 @@ export class Brainy implements BrainyInterface { this.graphIndex = graphIndex } + // Wire the mmap-vector backend (2.4.0 #2). When the vectorStore:mmap + // provider is registered AND the storage adapter resolves a real local + // path via getBinaryBlobPath(), open the mmap file there and inject the + // backend into HNSWIndex. Cloud adapters return null and skip silently; + // HNSWIndex's behaviour is then identical to pre-2.4.0 (per-entity reads + // from canonical storage). Failures are non-fatal — the index still + // works, just without the zero-copy fast path. + await this.wireMmapVectorBackend() + // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() @@ -7716,6 +7726,72 @@ export class Brainy implements BrainyInterface { await this.lazyRebuildPromise } + /** + * @description Open the mmap-vector backend and inject it into HNSWIndex. + * Called once during init after plugin activation, metadataIndex init, and + * graphIndex setup — so the idMapper is available and providers are wired. + * + * Activation conditions (ALL must hold; otherwise this is a no-op and the + * index falls back to per-entity storage reads, the pre-2.4.0 behaviour): + * 1. The `vectorStore:mmap` provider is registered (cortex registers this). + * 2. The storage adapter resolves a real local path via `getBinaryBlobPath()` + * — cloud adapters return null and the mmap layer is skipped silently. + * 3. The metadata index exposes its idMapper (a stable UUID↔int map). The + * mmap layout is keyed by these ints, so 2.4.0 #1 is the prerequisite. + * + * Vector dimensionality: read from `this.dimensions` if set; otherwise the + * default for the standard embedding model (384). The dim is fixed at + * file-creation time and cannot change later, so a wrong default would only + * matter on the very first init of a brand-new instance. After the first + * `add()` the in-memory `this.dimensions` is set, and any subsequent init + * (or a re-open) sees the correct value. + * + * Failures here are non-fatal: a debug-level log records the reason, and + * HNSWIndex continues without the mmap fast path. + */ + private async wireMmapVectorBackend(): Promise { + const provider = this.pluginRegistry.getProvider('vectorStore:mmap') + if (!provider) return + + const storageWithBlob = this.storage as unknown as { + getBinaryBlobPath?: (key: string) => string | null + } + const vectorPath = storageWithBlob.getBinaryBlobPath?.('_vectors/main') ?? null + if (!vectorPath) return + + const idMapper = this.metadataIndex.getIdMapper?.() + if (!idMapper) return + + const dim = this.dimensions ?? 384 + const initialCapacity = Math.max(idMapper.size * 2, 1024) + + try { + const backend = await MmapVectorBackend.open( + provider, + vectorPath, + dim, + initialCapacity, + idMapper + ) + this.index.setVectorBackend(backend) + if (!this.config.silent) { + console.log( + `[brainy] vector mmap backend wired (path=${vectorPath}, dim=${dim}, capacity=${initialCapacity})` + ) + } + } catch (error) { + // Common cases: dim mismatch from a prior init at a different dimension, + // permission errors, disk full, file corruption. Index keeps working via + // the per-entity storage path; flag the cause for diagnosis. + if (!this.config.silent) { + console.warn( + `[brainy] mmap-vector backend not wired (${(error as Error).message}); ` + + `falling back to per-entity vector reads` + ) + } + } + } + /** * Rebuild indexes from persisted data if needed (LAZY LOADING) * diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index f5af9137..496fc563 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -18,6 +18,7 @@ import { prodLog } from '../utils/logger.js' import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js' import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js' import type { HnswProvider } from '../plugin.js' +import { MmapVectorBackend } from './mmapVectorBackend.js' // Default HNSW parameters const DEFAULT_CONFIG: HNSWConfig = { @@ -49,6 +50,13 @@ export class HNSWIndex implements HnswProvider { private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes // Always-adaptive caching - no "mode" concept, system adapts automatically + // Optional mmap-vector backend (2.4.0 #2). When set, the read paths + // (preloadVectors, getVectorSafe) try the mmap layer before storage, and + // on a storage hit they write back into the mmap slot — a lazy migration + // that converges upgraded installs to the fast path under live traffic. + // Null on cloud storage adapters (no local path) and pre-injection init. + private vectorBackend: MmapVectorBackend | null = null + // COW (Copy-on-Write) support private cowEnabled: boolean = false private cowModifiedNodes: Set = new Set() @@ -101,6 +109,22 @@ export class HNSWIndex implements HnswProvider { this.useParallelization = useParallelization } + /** + * @description Inject (or detach) the mmap-vector backend. When set, the + * vector read paths (`preloadVectors`, `getVectorSafe`) try the mmap layer + * before storage and lazily write back on storage hits — converging an + * upgraded install to the zero-copy fast path without a migration step. + * Setting to null reverts to the existing per-entity read path. + * + * Lifecycle: brainy.ts wires this after plugin activation when (a) the + * `vectorStore:mmap` provider is registered AND (b) the storage adapter + * resolves a real local path via `getBinaryBlobPath()`. Cloud adapters + * leave it null; HNSWIndex's behaviour is then identical to pre-2.4.0. + */ + public setVectorBackend(backend: MmapVectorBackend | null): void { + this.vectorBackend = backend + } + /** * Get whether parallelization is enabled */ @@ -1073,7 +1097,23 @@ export class HNSWIndex implements HnswProvider { const cacheKey = `hnsw:vector:${noun.id}` const vector = await this.unifiedCache.get(cacheKey, async () => { - // Cache miss - load from storage + // Try the mmap-vector backend first when wired — zero-copy slot read, + // no per-entity object hydration. + if (this.vectorBackend) { + const fromMmap = this.vectorBackend.readByUuid(noun.id) + if (fromMmap) { + this.unifiedCache.set( + cacheKey, + fromMmap, + 'hnsw', + fromMmap.length * 4, + 50 + ) + return fromMmap + } + } + + // Storage fallback — the canonical source of truth. if (!this.storage) { throw new Error('Storage not available for vector loading') } @@ -1083,6 +1123,18 @@ export class HNSWIndex implements HnswProvider { throw new Error(`Vector not found for noun ${noun.id}`) } + // Lazy migration: write back into the mmap slot so the next read is + // mmap-fast. Idempotent + resumable; a write failure is non-fatal — + // the field still serves this read, and the next storage hit re-tries + // the migration. (See mmapVectorBackend.ts for the contract.) + if (this.vectorBackend) { + try { + this.vectorBackend.writeByUuid(noun.id, loaded) + } catch (error) { + prodLog.debug(`MmapVectorBackend write-back failed for ${noun.id}`, error) + } + } + // Add to UnifiedCache with cost-aware eviction // This competes fairly with Graph and Metadata indexes this.unifiedCache.set( @@ -1140,7 +1192,50 @@ export class HNSWIndex implements HnswProvider { private async preloadVectors(nodeIds: string[]): Promise { if (nodeIds.length === 0) return - // Use UnifiedCache's request coalescing to prevent duplicate loads + // mmap-vector backend fast path: batched O(1) slot reads + lazy + // write-back migration on miss. Cache populated from both layers so + // subsequent in-session reads are O(1) memory. + if (this.vectorBackend) { + // Filter out ids already cached so we never re-read. + const uncachedIds: string[] = [] + for (const id of nodeIds) { + if (this.unifiedCache.getSync(`hnsw:vector:${id}`) === undefined) { + uncachedIds.push(id) + } + } + if (uncachedIds.length === 0) return + + const fromMmap = this.vectorBackend.readBatchByUuid(uncachedIds) + const misses: string[] = [] + for (let i = 0; i < uncachedIds.length; i++) { + const id = uncachedIds[i] + const v = fromMmap[i] + if (v) { + this.unifiedCache.set(`hnsw:vector:${id}`, v, 'hnsw', v.length * 4, 50) + } else { + misses.push(id) + } + } + if (misses.length === 0) return + + // Storage fallback for the misses, with concurrent lazy write-back. + await Promise.all(misses.map(async (id) => { + if (!this.storage) return + const vector = await this.storage.getNounVector(id) + if (!vector) return + if (this.vectorBackend) { + try { + this.vectorBackend.writeByUuid(id, vector) + } catch (error) { + prodLog.debug(`MmapVectorBackend write-back failed for ${id}`, error) + } + } + this.unifiedCache.set(`hnsw:vector:${id}`, vector, 'hnsw', vector.length * 4, 50) + })) + return + } + + // Legacy path: per-entity storage load via UnifiedCache request coalescing. const promises = nodeIds.map(async (id) => { const cacheKey = `hnsw:vector:${id}` return this.unifiedCache.get(cacheKey, async () => { diff --git a/src/hnsw/mmapVectorBackend.ts b/src/hnsw/mmapVectorBackend.ts new file mode 100644 index 00000000..fa9d4e37 --- /dev/null +++ b/src/hnsw/mmapVectorBackend.ts @@ -0,0 +1,174 @@ +/** + * @module hnsw/mmapVectorBackend + * @description Disk-resident vector cache backing HNSWIndex, wrapping the + * `vectorStore:mmap` provider (cortex's NativeMmapVectorStore in production). + * + * **Why this exists.** Brainy's per-entity vector storage hits two ceilings as + * datasets grow: (1) every `HNSWIndex.preloadVectors` does a sequential + * `storage.getNounVector` per id (no batch), and (2) cloud and FS storage both + * serialize each vector through JSON / msgpack, blocking zero-copy reads. An + * mmap-backed single-file vector store, addressed by the stable int from + * `EntityIdMapper`, gives O(1) batch reads + OS-page-cache prefetch and is the + * foundation 2.4.0 #2 needs. + * + * **Lazy migration on miss.** First read of an unmigrated UUID falls back to + * `storage.getNounVector` and writes the bytes into the mmap slot before + * returning — idempotent, resumable, and lets an upgraded install converge to + * the fast path under live traffic without a big-bang migration step. The + * per-entity store remains the canonical source of truth; nothing in the per- + * entity layer is rewritten or deleted here. + * + * **Local-FS only in 2.4.0.** Activated when the storage adapter exposes a + * real local path via `getBinaryBlobPath()`. Cloud adapters (S3 / R2 / GCS) + * return null and the backend is not constructed, so HNSWIndex falls back to + * the existing per-entity read path. Chunked vector segments for remote + * storage are the planned 2.4.0 #2 follow-up. + */ + +import { mkdir } from 'node:fs/promises' +import { dirname } from 'node:path' +import type { Vector } from '../coreTypes.js' +import type { + EntityIdMapperProvider, + VectorStoreMmapInstance, + VectorStoreMmapProvider +} from '../plugin.js' + +/** + * @description Bridge between brainy's UUID-keyed vector API and cortex's + * int-slot mmap vector store. Translates UUIDs to stable int slots via the + * `EntityIdMapper`, grows the file on demand, and handles read misses by + * surfacing `null` (HNSWIndex's caller does the lazy storage fallback + + * write-back). The class never touches per-entity storage directly — it owns + * only the mmap layer. + */ +export class MmapVectorBackend { + private readonly store: VectorStoreMmapInstance + private readonly idMapper: EntityIdMapperProvider + + private constructor(store: VectorStoreMmapInstance, idMapper: EntityIdMapperProvider) { + this.store = store + this.idMapper = idMapper + } + + /** + * @description Open or create the mmap vector file at `path`. If the file + * does not exist, it is created with the given dim + initial capacity. If + * the directory containing `path` does not exist, it is created recursively + * (the cortex provider does not auto-create parent directories). + * @param provider - The `vectorStore:mmap` provider (cortex + * `NativeMmapVectorStore` in production). + * @param path - Absolute path to the vector file. Derived by the caller + * from `storage.getBinaryBlobPath()` on local-FS adapters; cloud adapters + * return null and this method is not called. + * @param dim - Vector dimensionality. Must match the embedding model. + * @param initialCapacity - Slot capacity to allocate when creating a new + * file. The file grows automatically on writes beyond capacity. + * @param idMapper - Stable UUID↔int mapper (post-2.4.0 #1 semantics — never + * renumbers across rebuild). The mmap slots are addressed by these ints, + * so id stability is the load-bearing invariant of the layout. + * @returns A ready-to-use backend. + */ + static async open( + provider: VectorStoreMmapProvider, + path: string, + dim: number, + initialCapacity: number, + idMapper: EntityIdMapperProvider + ): Promise { + await mkdir(dirname(path), { recursive: true }) + let store: VectorStoreMmapInstance + try { + store = provider.open(path) + } catch { + store = provider.create(path, dim, Math.max(initialCapacity, 16)) + } + return new MmapVectorBackend(store, idMapper) + } + + /** Vector dimensionality of the underlying mmap file. */ + get dim(): number { + return this.store.dim + } + + /** + * @description Read a vector by UUID. Returns `null` for any of: + * (1) UUID not yet assigned an int (entity never written through this + * backend or storage), (2) int beyond the highest written slot, or + * (3) the underlying mmap read throwing (corrupt / out-of-bounds). + * The null path is the caller's signal to fall back to per-entity storage + * and trigger a write-back migration. + */ + readByUuid(uuid: string): Vector | null { + const intId = this.idMapper.getInt(uuid) + if (intId === undefined || intId >= this.store.count) return null + try { + return this.store.readVector(intId) as unknown as Vector + } catch { + return null + } + } + + /** + * @description Batch read vectors by UUID. Returns an array aligned with + * `uuids` where each entry is the vector or `null` for a miss. Order- + * preserving. Internally collapses to a single batched mmap read of only the + * indices that resolve to a written slot, then redistributes the flat output + * back to the caller's order — so missing slots cost only a couple of map + * lookups, not a full N-element read. + */ + readBatchByUuid(uuids: string[]): (Vector | null)[] { + const indices: number[] = [] + const slotByUuid = new Map() + for (const uuid of uuids) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined && intId < this.store.count) { + slotByUuid.set(uuid, indices.length) + indices.push(intId) + } + } + if (indices.length === 0) return uuids.map(() => null) + const flat = this.store.readVectorsBatch(indices) + const dim = this.store.dim + return uuids.map(uuid => { + const slot = slotByUuid.get(uuid) + if (slot === undefined) return null + return flat.slice(slot * dim, (slot + 1) * dim) as unknown as Vector + }) + } + + /** + * @description Write a vector at the slot for `uuid`. Assigns a new int if + * the UUID is unseen (stable, append-only). Grows the underlying file (slot + * capacity doubled until the target index fits) before writing past + * capacity, so the caller need not worry about file sizing. + */ + writeByUuid(uuid: string, vector: Vector): void { + const intId = this.idMapper.getOrAssign(uuid) + if (intId >= this.store.capacity) { + let newCap = Math.max(this.store.capacity, 16) + while (intId >= newCap) newCap *= 2 + this.store.resize(newCap) + } + this.store.writeVector(intId, Array.from(vector as ArrayLike)) + } + + /** + * @description madvise(WILLNEED) hint for the slots backing `uuids`. Cheap; + * silently drops UUIDs not yet in the mmap. Used by HNSW search to overlap + * disk I/O with graph traversal. + */ + prefetchByUuid(uuids: string[]): void { + const indices: number[] = [] + for (const uuid of uuids) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined && intId < this.store.count) indices.push(intId) + } + if (indices.length > 0) this.store.prefetch(indices) + } + + /** msync — flush dirty pages to disk. */ + flush(): void { + this.store.flush() + } +} diff --git a/src/plugin.ts b/src/plugin.ts index 18652a65..0cf04a6c 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -253,6 +253,55 @@ export interface CacheProvider { // uses at the `getProvider('embeddings')` call site. No separate interface is // added here to avoid a duplicate, unwired contract. +/** + * The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed + * vector file. Brainy's HNSWIndex uses the static `.create()` / `.open()` + * factories to open a single-file store at a derived path on disk-resident + * storage adapters; non-FS adapters with no local-path support skip the mmap + * layer and fall back to per-entity vector reads. + * + * Cortex registers `NativeMmapVectorStore` as this provider — a Rust mmap'd + * f32 array with a header, batch read/write APIs, and madvise prefetch. Slot + * indices are the stable ints from `EntityIdMapper`; lazy migration on read + * miss writes back from per-entity storage into the slot, converging an + * upgraded install to the fast path without a big-bang step. + */ +export interface VectorStoreMmapProvider { + /** Create a new vector file with pre-allocated slot capacity. */ + create(path: string, dim: number, capacity: number): VectorStoreMmapInstance + /** Open an existing vector file for read + write. */ + open(path: string): VectorStoreMmapInstance + /** Open an existing vector file for read-only access. */ + openReadOnly(path: string): VectorStoreMmapInstance +} + +/** + * An open mmap-vector-store instance — the surface brainy's HNSWIndex consumes + * for batch vector reads + lazy migration from per-entity storage on miss. + */ +export interface VectorStoreMmapInstance { + /** Number of vectors written (highest written index + 1). */ + readonly count: number + /** Maximum slot capacity before resize is needed. */ + readonly capacity: number + /** Vector dimensionality. */ + readonly dim: number + /** Write a single vector at `index`. Vector is f64 in JS, stored as f32. */ + writeVector(index: number, vector: number[]): void + /** Write a flat batch starting at `startIndex`. Returns vectors written. */ + writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number + /** Read a single vector at `index`. Returns f64[]. */ + readVector(index: number): number[] + /** Read multiple vectors by index. Returns flat f64[] of length n × dim. */ + readVectorsBatch(indices: number[]): number[] + /** Prefetch via madvise(WILLNEED); no-op on unsupported platforms. */ + prefetch(indices: number[]): void + /** Grow the vector file to a larger slot capacity. Re-maps the file. */ + resize(newCapacity: number): void + /** msync — flush pending writes to disk. */ + flush(): void +} + /** * Storage adapter factory — plugins register these to provide * new storage backends that users reference by name. diff --git a/tests/unit/hnsw/mmap-vector-backend.test.ts b/tests/unit/hnsw/mmap-vector-backend.test.ts new file mode 100644 index 00000000..257d50c4 --- /dev/null +++ b/tests/unit/hnsw/mmap-vector-backend.test.ts @@ -0,0 +1,242 @@ +/** + * @module hnsw/mmapVectorBackend.test + * @description Unit tests for the `MmapVectorBackend` bridge — the brainy-side + * wrapper that translates UUID-keyed vector reads/writes into stable int slot + * ops against an `vectorStore:mmap` provider. + * + * Mocks the provider so the tests run without cortex installed (cortex is a + * downstream consumer of brainy, not a dev dep). The real integration with + * cortex's `NativeMmapVectorStore` is exercised when cortex 2.4.0 picks up + * this brainy release and re-runs its cross-language parity suite. + * + * Coverage: + * 1. Open-then-write-then-read round-trips for a single vector. + * 2. Batch reads return an array aligned with the input UUIDs, with `null` + * entries for misses interleaved among hits — order preserved. + * 3. Slot assignment is stable across multiple writes for the same UUID + * (no re-slot, no overwrite of an adjacent slot). + * 4. Writes beyond the initial capacity grow the file (doubling) without + * losing the vectors already written. + * 5. `readByUuid` returns `null` for both unknown UUIDs and UUIDs in the map + * whose slot has not yet been written. + * 6. Open is idempotent — opening an already-existing file reuses it. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { tmpdir } from 'node:os' +import { mkdtemp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { MmapVectorBackend } from '../../../src/hnsw/mmapVectorBackend.js' +import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js' +import type { + VectorStoreMmapInstance, + VectorStoreMmapProvider +} from '../../../src/plugin.js' + +/** + * Minimal storage stub for the EntityIdMapper. The mapper only touches storage + * in init/flush; for these tests the mapper starts empty and is never flushed. + */ +const stubStorage = { + getMetadata: async () => undefined, + saveMetadata: async () => {}, + getNouns: async () => ({ totalCount: 0, items: [] }) +} as any + +/** + * Pure in-memory mmap store. Mirrors cortex's NativeMmapVectorStore surface + * just closely enough to exercise the backend's contract — no real mmap, no + * file I/O, no f32 round-trip narrowing (the precision check is out of scope + * here; that's covered by cortex's parity suite). + */ +class MockMmapStore implements VectorStoreMmapInstance { + private readonly vectors: Array = [] + private highestWritten = -1 + constructor( + public readonly dim: number, + private _capacity: number + ) {} + get count(): number { + return this.highestWritten + 1 + } + get capacity(): number { + return this._capacity + } + writeVector(index: number, vector: number[]): void { + if (index >= this._capacity) { + throw new Error(`Slot ${index} >= capacity ${this._capacity}`) + } + if (vector.length !== this.dim) { + throw new Error(`Dim mismatch: expected ${this.dim}, got ${vector.length}`) + } + this.vectors[index] = [...vector] + if (index > this.highestWritten) this.highestWritten = index + } + writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number { + if (vectorsFlat.length % this.dim !== 0) { + throw new Error('vectorsFlat length not a multiple of dim') + } + const n = vectorsFlat.length / this.dim + for (let i = 0; i < n; i++) { + this.writeVector(startIndex + i, vectorsFlat.slice(i * this.dim, (i + 1) * this.dim)) + } + return n + } + readVector(index: number): number[] { + const v = this.vectors[index] + if (!v) throw new Error(`Slot ${index} not written`) + return [...v] + } + readVectorsBatch(indices: number[]): number[] { + const flat: number[] = [] + for (const i of indices) { + const v = this.vectors[i] + if (!v) throw new Error(`Slot ${i} not written`) + for (let k = 0; k < this.dim; k++) flat.push(v[k]) + } + return flat + } + prefetch(_indices: number[]): void { + /* no-op in mock */ + } + resize(newCapacity: number): void { + if (newCapacity < this._capacity) throw new Error('Cannot shrink') + this._capacity = newCapacity + } + flush(): void { + /* no-op in mock */ + } +} + +/** + * Mock provider — keeps one MockMmapStore per path. open() throws if the path + * doesn't exist yet (matches cortex semantics: open() is for existing files + * only); create() throws if it does (cortex doesn't, but the brainy backend + * does open-first-then-create, so the throw path is exercised). + */ +class MockMmapProvider implements VectorStoreMmapProvider { + private files = new Map() + create(path: string, dim: number, capacity: number): VectorStoreMmapInstance { + if (this.files.has(path)) throw new Error(`File exists at ${path}`) + const store = new MockMmapStore(dim, capacity) + this.files.set(path, store) + return store + } + open(path: string): VectorStoreMmapInstance { + const store = this.files.get(path) + if (!store) throw new Error(`No file at ${path}`) + return store + } + openReadOnly(path: string): VectorStoreMmapInstance { + return this.open(path) + } + /** Test helper — peek at the underlying store. */ + _peek(path: string): MockMmapStore | undefined { + return this.files.get(path) + } +} + +describe('MmapVectorBackend (2.4.0 #2 — wraps vectorStore:mmap provider)', () => { + let dir: string + let path: string + let idMapper: EntityIdMapper + let provider: MockMmapProvider + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'brainy-mmap-vec-')) + path = join(dir, 'vectors.bin') + idMapper = new EntityIdMapper({ storage: stubStorage }) + await idMapper.init() + provider = new MockMmapProvider() + }) + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }).catch(() => {}) + }) + + it('open creates a new file when none exists, then round-trips a vector by UUID', async () => { + const backend = await MmapVectorBackend.open(provider, path, 4, 16, idMapper) + expect(backend.dim).toBe(4) + + backend.writeByUuid('alpha', [1, 2, 3, 4]) + expect(backend.readByUuid('alpha')).toEqual([1, 2, 3, 4]) + }) + + it('reads return null for unknown UUIDs and for UUIDs in the map but not yet written', async () => { + const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) + + // Unknown — never seen by the mapper. + expect(backend.readByUuid('ghost')).toBeNull() + + // Known to the mapper but no slot ever written. We assign through the + // mapper directly so the backend itself has not touched the slot. + idMapper.getOrAssign('reserved') + expect(backend.readByUuid('reserved')).toBeNull() + }) + + it('batch read returns an array aligned to the input UUIDs (nulls preserved in place)', async () => { + const backend = await MmapVectorBackend.open(provider, path, 3, 16, idMapper) + backend.writeByUuid('a', [1, 1, 1]) + backend.writeByUuid('b', [2, 2, 2]) + backend.writeByUuid('c', [3, 3, 3]) + + // Interleave hits + a never-seen UUID + a duplicate hit. Order preserved. + const result = backend.readBatchByUuid(['b', 'missing', 'a', 'c', 'missing-too', 'b']) + expect(result).toEqual([ + [2, 2, 2], + null, + [1, 1, 1], + [3, 3, 3], + null, + [2, 2, 2] + ]) + }) + + it('writes for the same UUID land in the same slot (stable id, no re-slotting)', async () => { + const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) + backend.writeByUuid('persistent', [1, 1]) + const slotAfterFirstWrite = idMapper.getInt('persistent') + + backend.writeByUuid('persistent', [9, 9]) // overwrite the same slot + expect(idMapper.getInt('persistent')).toBe(slotAfterFirstWrite) + expect(backend.readByUuid('persistent')).toEqual([9, 9]) + + // Another UUID gets a different slot, and the first vector is undisturbed. + backend.writeByUuid('other', [5, 5]) + expect(idMapper.getInt('other')).not.toBe(slotAfterFirstWrite) + expect(backend.readByUuid('persistent')).toEqual([9, 9]) + expect(backend.readByUuid('other')).toEqual([5, 5]) + }) + + it('grows the file (doubling) when a write lands beyond capacity, without losing prior data', async () => { + // Start at the smallest sane initial capacity (clamped to 16 by the backend). + const backend = await MmapVectorBackend.open(provider, path, 2, 1, idMapper) + const store = provider._peek(path)! + expect(store.capacity).toBe(16) // backend floor + + // Write 20 vectors so capacity must double at least once (16 → 32). + const uuids: string[] = [] + for (let i = 0; i < 20; i++) { + const uuid = `u-${i}` + uuids.push(uuid) + backend.writeByUuid(uuid, [i, i * 2]) + } + expect(store.capacity).toBeGreaterThanOrEqual(32) + + // Every vector survived the growth — no slot got overwritten or lost. + for (let i = 0; i < uuids.length; i++) { + expect(backend.readByUuid(uuids[i])).toEqual([i, i * 2]) + } + }) + + it('opens an existing file idempotently when the second open hits the same path', async () => { + // First open creates. + const backend1 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) + backend1.writeByUuid('persisted', [7, 7]) + + // Second open against the same path. The provider's create() throws on + // collision; the backend's open-first behaviour means we reuse the file. + const backend2 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) + expect(backend2.readByUuid('persisted')).toEqual([7, 7]) + }) +}) diff --git a/tests/regression/entity-id-mapper-stability.test.ts b/tests/unit/utils/entity-id-mapper-stability.test.ts similarity index 99% rename from tests/regression/entity-id-mapper-stability.test.ts rename to tests/unit/utils/entity-id-mapper-stability.test.ts index 4ba2bda6..f872f46b 100644 --- a/tests/regression/entity-id-mapper-stability.test.ts +++ b/tests/unit/utils/entity-id-mapper-stability.test.ts @@ -22,7 +22,7 @@ */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Brainy } from '../../src/brainy.js' +import { Brainy } from '../../../src/brainy.js' const DIM = 384 const makeVec = (seed = 1) => -- 2.49.1 From ec157afc15b9f29115f74f7259f6f24b48fbf728 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 28 May 2026 10:24:48 -0700 Subject: [PATCH 3/4] =?UTF-8?q?feat:=20column-store=20JS=E2=86=94native=20?= =?UTF-8?q?interchange=20=E2=80=94=20raw-blob=20unify=20(2.4.0=20#4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brainy's JS ColumnStore.flushBuffer now writes segment payloads + the per-field DELETED bitmap via the binary-blob primitive (saveBinaryBlob) at the cortex-shared key convention `_column_index//L-NNNNNN` (suffix-free; adapter appends its own). Byte-for-byte identical to what cortex's NativeColumnStore writes — JS and native engines now read each other's segments with no envelope re-encoding. Closes the gap that the cortex 2.3.1 read-side fallback opened: cortex was reading both formats but the JS engine was still WRITING the legacy {_binary, base64} envelope, so a fresh JS write always required the cortex fallback to kick in (and migrate lazily on first read). With this commit JS writes natively in the unified format, and cortex's fallback only fires on indexes persisted by older brainy releases. Backward compat: - ColumnStore.loadSegmentCursor tries the raw blob first, then falls back to the legacy `{_binary, base64}` envelope at the `.cidx` object-path. Indexes written by pre-2.4.0 brainy keep loading correctly. - ColumnStore.init's DELETED-bitmap load has the same dual-format read. - Adapters without the binary-blob primitive (custom adapters that didn't follow the 7.25.0 surface) fall through to the legacy envelope writer too, so writes still succeed there. Tests (1433 total, +5 vs prior tip): - tests/unit/indexes/columnStore/column-store-interchange.test.ts — pins down the contract: (1) flush writes the raw blob, NO legacy envelope; (2) DELETED bitmap likewise; (3) a legacy-format on-disk segment loads correctly via the fallback; (4) legacy DELETED bitmap ditto; (5) round-trip in the new format. - All 101 existing ColumnStore tests pass — the new write path is exercised by the existing lifecycle tests (MemoryStorage has the blob primitive, so the new branch fires). --- src/indexes/columnStore/ColumnStore.ts | 131 +++++++++--- .../column-store-interchange.test.ts | 191 ++++++++++++++++++ 2 files changed, 293 insertions(+), 29 deletions(-) create mode 100644 tests/unit/indexes/columnStore/column-store-interchange.test.ts diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 6b264463..f290274f 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -125,12 +125,29 @@ export class ColumnStore implements ColumnStoreProvider { this.manifests.set(fieldName, manifest) this.fieldTypes.set(fieldName, manifest.valueType) - // Load global deleted bitmap if it exists + // Load global deleted bitmap if it exists. Raw blob preferred + // (2.4.0 #4 cortex-shared format); legacy envelope fallback for + // indexes written before format unification. try { - const deletedPath = `${this.basePath}/${fieldName}/DELETED.bin` - const stored = await (storage as any).readObjectFromPath(deletedPath) - if (stored && stored._binary && stored.data) { - const buf = Buffer.from(stored.data, 'base64') + let buf: Buffer | null = null + + const adapter = storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + readObjectFromPath: (path: string) => Promise + } + if (typeof adapter.loadBinaryBlob === 'function') { + const key = `${this.basePath}/${fieldName}/DELETED` + const blob = await adapter.loadBinaryBlob(key) + if (blob && blob.length > 0) buf = blob + } + if (!buf) { + const deletedPath = `${this.basePath}/${fieldName}/DELETED.bin` + const stored = await adapter.readObjectFromPath(deletedPath) + if (stored && stored._binary && stored.data) { + buf = Buffer.from(stored.data, 'base64') + } + } + if (buf) { this.deletedEntities.set(fieldName, RoaringBitmap32.deserialize(buf, true)) } } catch { @@ -408,6 +425,23 @@ export class ColumnStore implements ColumnStoreProvider { /** * Flush a single field's tail buffer to a new L0 segment. + * + * Storage path (2.4.0 #4 / cortex-interchange contract): + * When the storage adapter exposes the binary-blob primitive + * (`saveBinaryBlob` — every brainy adapter ≥7.25.0 does), the segment + * bytes are written as a raw blob at the shared cortex key + * `_column_index//L-NNNNNN` (suffix-free; the adapter + * appends its own). This matches byte-for-byte what cortex's + * `NativeColumnStore` writes, so JS- and native-written indexes + * interchange without re-encoding. + * + * When the adapter doesn't (older custom adapters that didn't follow + * the 7.25.0 primitive surface), we fall back to the legacy + * `writeObjectToPath` envelope — `{ _binary, data: }` at + * `_column_index//L-NNNNNN.cidx`. Cortex's 2.3.1 + * read-side fallback handles the legacy envelope on its end, so the + * format mismatch is transparent in both directions during the + * transition. */ private async flushBuffer(field: string, buffer: ColumnTailBuffer): Promise { const { values, entityIds, pendingRemovals } = buffer.drain() @@ -416,10 +450,15 @@ export class ColumnStore implements ColumnStoreProvider { const manifest = this.manifests.get(field) if (!manifest) return + const storage = this.storage as unknown as { + saveBinaryBlob?: (key: string, data: Buffer) => Promise + writeObjectToPath: (path: string, value: unknown) => Promise + } + const canUseBlob = typeof storage.saveBinaryBlob === 'function' + // Write segment if we have values if (values.length > 0) { const segId = manifest.nextSegmentId() - const segPath = manifest.segmentPath(0, segId) const segBuffer = writeSegmentToBuffer( { @@ -435,13 +474,22 @@ export class ColumnStore implements ColumnStoreProvider { entityIds ) - // Store segment as binary data - await (this.storage as any).writeObjectToPath(segPath, { - _binary: true, - data: segBuffer.toString('base64') - }) + if (canUseBlob) { + // Raw blob, cortex-shared key convention. + const key = `${this.basePath}/${field}/L0-${String(segId).padStart(6, '0')}` + await storage.saveBinaryBlob!(key, segBuffer) + } else { + // Legacy envelope path for adapters without the binary-blob primitive. + const segPath = manifest.segmentPath(0, segId) + await storage.writeObjectToPath(segPath, { + _binary: true, + data: segBuffer.toString('base64') + }) + } - // Register in manifest + // Register in manifest. The `.cidx` file name in the manifest is the + // legacy-format file the cortex 2.3.1 read-side fallback looks for; the + // raw-blob key derives from segId at read time. Both paths converge. manifest.addSegment({ id: segId, level: 0, @@ -465,15 +513,21 @@ export class ColumnStore implements ColumnStoreProvider { for (const id of pendingRemovals) deleted.add(id) } - // Persist the global deleted bitmap alongside the manifest + // Persist the global deleted bitmap alongside the manifest. Same + // raw-blob preference as segments — shared cortex key `//DELETED`. const deleted = this.deletedEntities.get(field) if (deleted && deleted.size > 0) { - const deletedPath = `${this.basePath}/${field}/DELETED.bin` const serialized = deleted.serialize(true) - await (this.storage as any).writeObjectToPath(deletedPath, { - _binary: true, - data: Buffer.from(serialized).toString('base64') - }) + if (canUseBlob) { + const deletedKey = `${this.basePath}/${field}/DELETED` + await storage.saveBinaryBlob!(deletedKey, Buffer.from(serialized)) + } else { + const deletedPath = `${this.basePath}/${field}/DELETED.bin` + await storage.writeObjectToPath(deletedPath, { + _binary: true, + data: Buffer.from(serialized).toString('base64') + }) + } } // Save manifest @@ -508,24 +562,43 @@ export class ColumnStore implements ColumnStoreProvider { /** * Load a segment from storage and create a cursor. + * + * Reads the raw-blob layout first (the 2.4.0 #4 shared-with-cortex format), + * then falls back to the legacy `{ _binary, base64 }` envelope at the + * `.cidx` object-path so indexes written before the format unification keep + * loading correctly. Mirror of cortex's 2.3.1 read-side fallback. */ private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise { const manifest = this.manifests.get(field) if (!manifest) return null try { - const segPath = manifest.segmentPath(seg.level, seg.id) - const stored = await (this.storage as any).readObjectFromPath(segPath) - if (!stored) return null + let buf: Buffer | null = null - // Handle binary storage format - let buf: Buffer - if (stored._binary && stored.data) { - buf = Buffer.from(stored.data, 'base64') - } else if (Buffer.isBuffer(stored)) { - buf = stored - } else { - return null + const storage = this.storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + readObjectFromPath: (path: string) => Promise + } + + // Preferred: raw blob at the cortex-shared key. + if (typeof storage.loadBinaryBlob === 'function') { + const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}` + const blob = await storage.loadBinaryBlob(key) + if (blob && blob.length > 0) buf = blob + } + + // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path. + if (!buf) { + const segPath = manifest.segmentPath(seg.level, seg.id) + const stored = await storage.readObjectFromPath(segPath) + if (!stored) return null + if (stored._binary && stored.data) { + buf = Buffer.from(stored.data, 'base64') + } else if (Buffer.isBuffer(stored)) { + buf = stored + } else { + return null + } } const parsed = readSegmentFromBuffer(buf) diff --git a/tests/unit/indexes/columnStore/column-store-interchange.test.ts b/tests/unit/indexes/columnStore/column-store-interchange.test.ts new file mode 100644 index 00000000..f79038ea --- /dev/null +++ b/tests/unit/indexes/columnStore/column-store-interchange.test.ts @@ -0,0 +1,191 @@ +/** + * @module column-store-interchange.test + * @description 2.4.0 #4 — column-store JS↔native interchange. Pins down that + * brainy's JS `ColumnStore` writes segments + DELETED bitmaps via the binary- + * blob primitive at the cortex-shared key convention, AND that indexes + * persisted under the pre-2.4.0 envelope (`{ _binary, base64 }` at the + * `.cidx` object-path) still load correctly via the legacy-read fallback. + * + * Together these properties make the JS and native engines byte-compatible + * at the segment payload, so writing through one and reading through the + * other is a no-op format-wise — closing the gap the cortex 2.3.1 read-side + * fallback opened on the cortex side. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js' +import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' +import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' +import { writeSegmentToBuffer } from '../../../../src/indexes/columnStore/ColumnSegmentFormat.js' +import { ColumnManifest } from '../../../../src/indexes/columnStore/ColumnManifest.js' +import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js' + +describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => { + let storage: MemoryStorage + let idMapper: EntityIdMapper + let store: ColumnStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' }) + await idMapper.init() + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + }) + + afterEach(async () => { + await store.close() + }) + + // ----------------------------------------------------------------------- + // WRITE side: raw blob at the cortex-shared key, no legacy envelope + // ----------------------------------------------------------------------- + + it('flushBuffer writes segment bytes as a raw blob at the cortex-shared key', async () => { + for (let i = 0; i < 5; i++) { + store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 }) + } + await store.flush() + + // The raw blob exists at the suffix-free cortex key. + const key = '_column_index/score/L0-000001' + const raw = await (storage as any).loadBinaryBlob(key) + expect(Buffer.isBuffer(raw)).toBe(true) + expect(raw!.length).toBeGreaterThan(0) + + // The legacy envelope at the `.cidx` object-path is NOT written (no + // 33% base64 bloat, no dual-write). + const legacyPath = '_column_index/score/L0-000001.cidx' + const legacyEnvelope = await (storage as any).readObjectFromPath(legacyPath) + expect(legacyEnvelope).toBeFalsy() + }) + + it('flushBuffer writes the DELETED bitmap as a raw blob too (not an envelope)', async () => { + for (let i = 0; i < 3; i++) { + store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i }) + } + store.removeEntity(idMapper.getInt('u-1')!) + await store.flush() + + const rawDel = await (storage as any).loadBinaryBlob('_column_index/score/DELETED') + expect(Buffer.isBuffer(rawDel)).toBe(true) + expect(rawDel!.length).toBeGreaterThan(0) + + const legacyEnvelope = await (storage as any).readObjectFromPath( + '_column_index/score/DELETED.bin' + ) + expect(legacyEnvelope).toBeFalsy() + }) + + // ----------------------------------------------------------------------- + // READ side: pre-2.4.0 legacy envelope still loads via the fallback + // ----------------------------------------------------------------------- + + it('loads a segment persisted as the pre-2.4.0 envelope (legacy fallback)', async () => { + // Seed a legacy-format index DIRECTLY on storage — the on-disk shape + // brainy 7.20.0..7.25.0 wrote before this commit. + const legacyStore = new ColumnStore() + await legacyStore.init(storage, idMapper) + + // Build a segment buffer using the shared cidx format. + const values: number[] = [10, 20, 30] + const entityIds: number[] = [ + idMapper.getOrAssign('legacy-a'), + idMapper.getOrAssign('legacy-b'), + idMapper.getOrAssign('legacy-c') + ] + const segBuffer = writeSegmentToBuffer( + { + fieldName: 'score', + fieldNameLength: 5, + valueType: 0, + level: 0, + codec: 0, + flags: 0, + count: values.length + }, + values, + entityIds + ) + + // Persist via the LEGACY envelope path (writeObjectToPath, base64 wrap). + await (storage as any).writeObjectToPath('_column_index/score/L0-000001.cidx', { + _binary: true, + data: segBuffer.toString('base64') + }) + const manifest = new ColumnManifest('score', '_column_index') + manifest.valueType = 0 + manifest.multiValue = false + manifest.addSegment({ + id: 1, + level: 0, + count: values.length, + minValue: values[0], + maxValue: values[values.length - 1], + file: 'L0-000001.cidx' + }) + await manifest.save(storage) + await legacyStore.close() + + // NEW ColumnStore re-discovers the manifest at init, then the legacy + // segment loads via the readObjectFromPath fallback (no raw blob exists + // at the cortex key, so it falls through). sortTopK returns the entities + // in the order the segment recorded — proves the bytes round-tripped. + const readStore = new ColumnStore() + await readStore.init(storage, idMapper) + const sortedInts = await readStore.sortTopK('score', 'asc', 10) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + expect(sortedUuids).toEqual(['legacy-a', 'legacy-b', 'legacy-c']) + await readStore.close() + }) + + it('loads a DELETED bitmap persisted as the pre-2.4.0 envelope (legacy fallback)', async () => { + // Manually seed a legacy DELETED.bin envelope, then re-init and confirm + // the entity it marks gone is excluded from filter/sort results. + const bm = new RoaringBitmap32() + bm.add(idMapper.getOrAssign('dead')) + const serialized = Buffer.from(bm.serialize(true)) + await (storage as any).writeObjectToPath('_column_index/score/DELETED.bin', { + _binary: true, + data: serialized.toString('base64') + }) + + // Add a couple of live entities so a manifest exists (init only loads + // DELETED for fields with a manifest on disk). + store.addEntity(idMapper.getOrAssign('alive-1'), { score: 100 }) + store.addEntity(idMapper.getOrAssign('alive-2'), { score: 200 }) + await store.flush() + await store.close() + + const readStore = new ColumnStore() + await readStore.init(storage, idMapper) + const sortedInts = await readStore.sortTopK('score', 'asc', 10) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + + // The dead entity must NOT appear; the alive ones must. + expect(sortedUuids).not.toContain('dead') + expect(sortedUuids).toContain('alive-1') + expect(sortedUuids).toContain('alive-2') + await readStore.close() + }) + + // ----------------------------------------------------------------------- + // ROUND-TRIP: write-then-read in the new format + // ----------------------------------------------------------------------- + + it('write (new format) → re-init → read returns the same data', async () => { + for (let i = 0; i < 5; i++) { + store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 }) + } + await store.flush() + await store.close() + + const readStore = new ColumnStore() + await readStore.init(storage, idMapper) + const sortedInts = await readStore.sortTopK('score', 'asc', 10) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + expect(sortedUuids).toEqual(['u-0', 'u-1', 'u-2', 'u-3', 'u-4']) + await readStore.close() + }) +}) -- 2.49.1 From 42cd241305bd9c9131993ba6ff61135fa2b914de Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 28 May 2026 10:37:01 -0700 Subject: [PATCH 4/4] =?UTF-8?q?feat:=20graph=20link=20compression=20?= =?UTF-8?q?=E2=80=94=20delta-varint=20connections=20(2.4.0=20#3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cortex registers a graph:compression provider exposing `encode`/`decode` for HNSW connection lists (delta-varint, ~4× smaller edges than the JSON-UUID-array shape brainy has persisted since the beginning). This wires the consumer side without changing the saveHNSWData/getHNSWData adapter signatures. Architecture: - NEW ConnectionsCodec (src/hnsw/connectionsCodec.ts) — translates between UUID-keyed Map> and a single compact per-node buffer via the provider + stable EntityIdMapper. Custom wire format: [count: u8] [[level: u8] [len: u32 LE] [bytes...]]*. One blob per node regardless of level count, so the storage I/O is identical to the legacy path (one saveHNSWData call) plus a single saveBinaryBlob. - HNSWIndex changes — a `connectionsCodec: ConnectionsCodec | null` field with `setConnectionsCodec()` setter. THREE save sites (deferred flush, immediate-mode entity persist, immediate-mode neighbor updates) now go through a single `persistNodeConnections(nodeId, noun)` helper: when the codec is wired AND the storage adapter exposes saveBinaryBlob, it encodes the connections, stores them at `_hnsw_conn/`, and records `connections: {}` in saveHNSWData as the marker. Otherwise the legacy JSON-array path is taken. TWO load sites use a matching `restoreNodeConnections` helper that tries loadBinaryBlob first and falls back to the legacy hnswData.connections field on miss / decode error. Format convergence is lazy: pre-2.4.0 nodes still load via the legacy path, then write the compressed form on next dirty save. - brainy.ts wireConnectionsCodec — runs alongside wireMmapVectorBackend during init. Activates when (a) the graph:compression provider is registered and (b) the metadata index exposes its idMapper. Unlike the mmap-vector backend, this layer engages on EVERY brainy 7.25.0 adapter — the blob primitive itself is universal; only the codec presence gates activation. - Provider interface GraphCompressionProvider in plugin.ts — encode + decode static signatures. Brainy depends on the interface; cortex's registered { encode: encodeConnections, decode: decodeConnections } satisfies it structurally. Tests (1437 total, +4 vs prior tip): - tests/unit/hnsw/connections-codec.test.ts — 4 unit tests with a JSON mock provider: single-level round-trip, empty-Map round-trip (one-byte buffer), multi-level fan-out round-trip, and silent-drop of UUIDs the decoding idMapper no longer knows. The real cross-language byte format is exercised when cortex 2.4.0 wires its delta-varint encode/decode in. This completes the brainy half of the 2.4.0 storage foundation: stable ids (#23), mmap vectors (#24), graph link compression (#25), and column-store interchange (#26). Coordinated release as brainy 7.26.0 + cortex 2.4.0 follows. --- src/brainy.ts | 45 +++++- src/hnsw/connectionsCodec.ts | 137 ++++++++++++++++++ src/hnsw/hnswIndex.ts | 168 +++++++++++++++++----- src/plugin.ts | 26 ++++ tests/unit/hnsw/connections-codec.test.ts | 137 ++++++++++++++++++ 5 files changed, 473 insertions(+), 40 deletions(-) create mode 100644 src/hnsw/connectionsCodec.ts create mode 100644 tests/unit/hnsw/connections-codec.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 7b18013d..53d73a3b 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -37,8 +37,14 @@ import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' import { PluginRegistry } from './plugin.js' -import type { BrainyPlugin, BrainyPluginContext, VectorStoreMmapProvider } from './plugin.js' +import type { + BrainyPlugin, + BrainyPluginContext, + GraphCompressionProvider, + VectorStoreMmapProvider +} from './plugin.js' import { MmapVectorBackend } from './hnsw/mmapVectorBackend.js' +import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' import { ValidationConfig, @@ -569,6 +575,14 @@ export class Brainy implements BrainyInterface { // works, just without the zero-copy fast path. await this.wireMmapVectorBackend() + // Wire the connections codec (2.4.0 #3). When the graph:compression + // provider is registered AND the metadata index exposes a stable + // idMapper, inject a codec that encodes HNSW connections as + // delta-varint blobs at save time and decodes on load. The blob + // primitive itself works on every brainy 7.25.0 adapter, so unlike the + // mmap-vector backend this layer engages even on cloud adapters. + this.wireConnectionsCodec() + // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() @@ -7792,6 +7806,35 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Wire the HNSW connections codec (2.4.0 #3). Activates when + * BOTH (a) the `graph:compression` provider is registered (cortex registers + * `{ encode: encodeConnections, decode: decodeConnections }`), and (b) the + * metadata index exposes a stable idMapper. Failures are non-fatal: HNSW + * keeps working via the legacy JSON-array path. + * + * Unlike the mmap-vector backend, this layer does NOT require a real local + * path — the binary-blob primitive saves the compressed bytes through the + * storage adapter directly, so the codec is engaged on cloud adapters too. + * Format convergence is lazy: pre-2.4.0 nodes load via the legacy path, + * then the next dirty save writes the compressed form (and the legacy + * field of saveHNSWData becomes empty), so all reads converge over time + * without an explicit migration step. + */ + private wireConnectionsCodec(): void { + const provider = this.pluginRegistry.getProvider('graph:compression') + if (!provider) return + + const idMapper = this.metadataIndex.getIdMapper?.() + if (!idMapper) return + + const codec = new ConnectionsCodec(provider, idMapper) + this.index.setConnectionsCodec(codec) + if (!this.config.silent) { + console.log('[brainy] graph link compression wired (delta-varint connections via graph:compression provider)') + } + } + /** * Rebuild indexes from persisted data if needed (LAZY LOADING) * diff --git a/src/hnsw/connectionsCodec.ts b/src/hnsw/connectionsCodec.ts new file mode 100644 index 00000000..91dc00b5 --- /dev/null +++ b/src/hnsw/connectionsCodec.ts @@ -0,0 +1,137 @@ +/** + * @module hnsw/connectionsCodec + * @description Bridge between brainy's UUID-keyed HNSW connection sets and + * cortex's int-keyed delta-varint encode/decode (the `graph:compression` + * provider). Translates UUIDs to stable int slots via the post-2.4.0 #1 + * `EntityIdMapper`, batches all of a node's per-level connection lists into + * a single compact buffer, and reverses the path on load. + * + * Wire format of the per-node buffer the codec produces: + * + * [num_levels: u8] + * repeated num_levels times: + * [level: u8] + * [encoded_len: u32 LE] + * [encoded_bytes: bytes] <- provider.encode(ints) + * + * Why a single buffer per node and not one blob per level: + * - One `saveBinaryBlob` call per persisted node, regardless of level count. + * On hot insert paths an HNSW node may be at levels 0..3; the per-node + * blob trades a one-byte count for 3 saved I/O round-trips. + * - Read path is symmetric: one `loadBinaryBlob` reconstructs the full + * connection set, so HNSW rebuild stays linear in the number of nodes. + * + * Loose UUIDs (ints assigned to entities that no longer exist) are silently + * dropped on decode. They can't legitimately participate in graph traversal + * because the entity is gone; treating them as `undefined` UUIDs matches the + * pre-2.4.0 behaviour of an `idMapper.getUuid()` miss in the legacy load path. + */ + +import type { + EntityIdMapperProvider, + GraphCompressionProvider +} from '../plugin.js' + +export class ConnectionsCodec { + constructor( + private readonly provider: GraphCompressionProvider, + private readonly idMapper: EntityIdMapperProvider + ) {} + + /** + * @description Encode all levels of a node's connections into one compact + * buffer. Empty input → an empty buffer (zero levels, no body). The + * idMapper assigns stable ints on demand if a connection UUID has never + * been seen — append-only, matches the rest of the 2.4.0 contract. + */ + encode(connectionsByLevel: Map>): Buffer { + if (connectionsByLevel.size === 0) { + // Single byte: zero levels. Round-trips cleanly through decode. + return Buffer.from([0]) + } + if (connectionsByLevel.size > 0xff) { + throw new Error( + `ConnectionsCodec.encode: too many HNSW levels (${connectionsByLevel.size}); ` + + `the per-node header is one byte (max 255 levels).` + ) + } + + // First pass: encode each level into its own buffer. + const levelBuffers: Array<{ level: number; buf: Buffer }> = [] + let bodyBytes = 0 + for (const [level, uuids] of connectionsByLevel) { + if (level > 0xff) { + throw new Error( + `ConnectionsCodec.encode: level ${level} exceeds u8 header range (max 255).` + ) + } + const ints: number[] = [] + for (const uuid of uuids) { + ints.push(this.idMapper.getOrAssign(uuid)) + } + const buf = this.provider.encode(ints) + levelBuffers.push({ level, buf }) + bodyBytes += 1 + 4 + buf.length // level + len + payload + } + + // Second pass: concatenate with header. + const out = Buffer.alloc(1 + bodyBytes) + out.writeUInt8(levelBuffers.length, 0) + let offset = 1 + for (const { level, buf } of levelBuffers) { + out.writeUInt8(level, offset) + offset += 1 + out.writeUInt32LE(buf.length, offset) + offset += 4 + buf.copy(out, offset) + offset += buf.length + } + return out + } + + /** + * @description Decode a per-node buffer back into a connections Map. Ints + * for entities the idMapper no longer knows are silently dropped — see the + * module-level note on why. + */ + decode(data: Buffer): Map> { + const out = new Map>() + if (data.length === 0) return out + + const count = data.readUInt8(0) + let offset = 1 + for (let i = 0; i < count; i++) { + if (offset + 5 > data.length) { + throw new Error(`ConnectionsCodec.decode: truncated header at level ${i}`) + } + const level = data.readUInt8(offset) + offset += 1 + const len = data.readUInt32LE(offset) + offset += 4 + if (offset + len > data.length) { + throw new Error(`ConnectionsCodec.decode: truncated body at level ${level}`) + } + const levelBuf = data.slice(offset, offset + len) + offset += len + + const ints = this.provider.decode(levelBuf) + const uuids = new Set() + for (const intId of ints) { + const uuid = this.idMapper.getUuid(intId) + if (uuid !== undefined) uuids.add(uuid) + } + out.set(level, uuids) + } + return out + } +} + +/** + * @description Build the binary-blob storage key for a node's compressed + * connections. Suffix-free; the adapter appends its own. Keys live under + * `_hnsw_conn/` so they don't collide with the existing `_column_index/` + * blobs and so a cortex-side reader knows exactly where to look. + */ +export function compressedConnectionsKey(nodeId: string): string { + return `_hnsw_conn/${nodeId}` +} diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 496fc563..ff074daf 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -19,6 +19,7 @@ import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js' import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js' import type { HnswProvider } from '../plugin.js' import { MmapVectorBackend } from './mmapVectorBackend.js' +import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js' // Default HNSW parameters const DEFAULT_CONFIG: HNSWConfig = { @@ -57,6 +58,16 @@ export class HNSWIndex implements HnswProvider { // Null on cloud storage adapters (no local path) and pre-injection init. private vectorBackend: MmapVectorBackend | null = null + // Optional connections codec (2.4.0 #3). When set, node-persist writes the + // node's per-level connection sets as a single delta-varint-compressed + // binary blob (typically ~4× smaller than the legacy JSON-UUID-array + // shape), plus an empty `connections: {}` in saveHNSWData as the marker. + // The read path tries to load the blob first; missing blob → legacy + // connections field. Format convergence is lazy: pre-2.4.0 nodes still + // load via the legacy path, then write the compressed form on next dirty + // save, so the migration converges under live traffic with no big-bang. + private connectionsCodec: ConnectionsCodec | null = null + // COW (Copy-on-Write) support private cowEnabled: boolean = false private cowModifiedNodes: Set = new Set() @@ -125,6 +136,19 @@ export class HNSWIndex implements HnswProvider { this.vectorBackend = backend } + /** + * @description Inject (or detach) the connections codec. When set, node + * persistence writes a delta-varint-compressed binary blob alongside an + * empty `connections: {}` marker in saveHNSWData; the load path reads the + * blob and decodes. Null reverts to the legacy JSON-array path. Wiring is + * done by brainy.ts when the `graph:compression` provider is registered + * AND the storage adapter exposes the binary-blob primitive AND the + * metadata index has a stable idMapper. + */ + public setConnectionsCodec(codec: ConnectionsCodec | null): void { + this.connectionsCodec = codec + } + /** * Get whether parallelization is enabled */ @@ -168,15 +192,7 @@ export class HNSWIndex implements HnswProvider { const noun = this.nouns.get(nodeId) if (!noun) return Promise.resolve() // Node was deleted - const connectionsObj: Record = {} - for (const [level, nounIds] of noun.connections.entries()) { - connectionsObj[level.toString()] = Array.from(nounIds) - } - - return this.storage!.saveHNSWData(nodeId, { - level: noun.level, - connections: connectionsObj - }).catch(error => { + return this.persistNodeConnections(nodeId, noun).catch(error => { console.error(`[HNSW flush] Failed to persist node ${nodeId}:`, error) }) }) @@ -207,6 +223,94 @@ export class HNSWIndex implements HnswProvider { return nodeCount } + /** + * @description Persist one node's connections. When the connections codec is + * wired AND the storage adapter exposes `saveBinaryBlob`, the per-level + * connection sets are encoded into a single compact buffer and stored as a + * binary blob; `saveHNSWData` then records the node's level and an empty + * `connections: {}` as the marker. Otherwise the legacy JSON-array path is + * taken — the format every brainy release before 2.4.0 wrote. The two are + * intentionally NOT dual-written: one or the other, never both, so format + * convergence is unambiguous on the read side. + * + * Shared by all three save sites (deferred flush, immediate-mode new-entity + * persist, immediate-mode neighbor update) so the codec branch is exercised + * uniformly regardless of which path triggered the write. + */ + private async persistNodeConnections(nodeId: string, noun: HNSWNoun): Promise { + if (!this.storage) return + + const storageWithBlob = this.storage as unknown as { + saveBinaryBlob?: (key: string, data: Buffer) => Promise + } + const canCompress = + this.connectionsCodec !== null && + typeof storageWithBlob.saveBinaryBlob === 'function' + + if (canCompress) { + const encoded = this.connectionsCodec!.encode(noun.connections) + await storageWithBlob.saveBinaryBlob!(compressedConnectionsKey(nodeId), encoded) + await this.storage.saveHNSWData(nodeId, { + level: noun.level, + connections: {} + }) + return + } + + const connectionsObj: Record = {} + for (const [level, nounIds] of noun.connections.entries()) { + connectionsObj[level.toString()] = Array.from(nounIds) + } + await this.storage.saveHNSWData(nodeId, { + level: noun.level, + connections: connectionsObj + }) + } + + /** + * @description Restore one node's connections from persisted storage. Tries + * the compressed-blob path first (when codec wired AND blob primitive + * available); a present non-empty buffer is decoded and populates + * `noun.connections`. A missing blob (no payload at the key) means this node + * was last persisted under the legacy format, so the legacy + * `hnswData.connections` field is used instead. + * + * On any decode error the legacy field is also used as a safety net — the + * codec's `decode` throws on truncated input, which would otherwise leave + * the node with an empty connections Map (orphaned from the graph). + */ + private async restoreNodeConnections( + nodeId: string, + hnswData: { level: number; connections: Record }, + noun: HNSWNoun + ): Promise { + const storageWithBlob = this.storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + } + const canDecompress = + this.connectionsCodec !== null && + typeof storageWithBlob.loadBinaryBlob === 'function' + + if (canDecompress) { + try { + const buf = await storageWithBlob.loadBinaryBlob!(compressedConnectionsKey(nodeId)) + if (buf && buf.length > 0) { + const decoded = this.connectionsCodec!.decode(buf) + noun.connections = decoded + return + } + } catch (error) { + prodLog.debug(`HNSW: compressed connections decode failed for ${nodeId}; falling back to legacy`, error) + } + } + + // Legacy fallback: JSON UUID arrays in the HNSW data object. + for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { + const level = parseInt(levelStr, 10) + noun.connections.set(level, new Set(nounIds as string[])) + } + } + /** * Get the number of dirty (unpersisted) nodes * Useful for monitoring and debugging @@ -545,18 +649,12 @@ export class HNSWIndex implements HnswProvider { // In deferred mode, we track dirty nodes instead of persisting immediately // This reduces GCS operations from 70 to 2-3 per add() (30-50× faster) if (this.storage && this.persistMode === 'immediate') { - // IMMEDIATE MODE: Original behavior - persist each neighbor update - const neighborConnectionsObj: Record = {} - for (const [lvl, nounIds] of neighbor.connections.entries()) { - neighborConnectionsObj[lvl.toString()] = Array.from(nounIds) - } - + // IMMEDIATE MODE: Original behavior - persist each neighbor update. + // Goes through the per-node helper so the compressed-blob branch + // fires identically here vs. the deferred-flush path. neighborUpdates.push({ neighborId, - promise: this.storage.saveHNSWData(neighborId, { - level: neighbor.level, - connections: neighborConnectionsObj - }) + promise: this.persistNodeConnections(neighborId, neighbor) }) } else if (this.persistMode === 'deferred') { // DEFERRED MODE: Track dirty nodes for later batch persistence @@ -643,16 +741,10 @@ export class HNSWIndex implements HnswProvider { // Persist HNSW graph data to storage // Respect persistMode setting if (this.storage && this.persistMode === 'immediate') { - // IMMEDIATE MODE: Original behavior - persist new entity and system data - const connectionsObj: Record = {} - for (const [level, nounIds] of noun.connections.entries()) { - connectionsObj[level.toString()] = Array.from(nounIds) - } - - await this.storage.saveHNSWData(id, { - level: nounLevel, - connections: connectionsObj - }).catch((error) => { + // IMMEDIATE MODE: Original behavior - persist new entity and system data. + // Goes through the per-node helper so the compressed-blob branch fires + // identically here vs. the deferred-flush + neighbor-update paths. + await this.persistNodeConnections(id, noun).catch((error) => { console.error(`Failed to persist HNSW data for ${id}:`, error) }) @@ -1434,11 +1526,10 @@ export class HNSWIndex implements HnswProvider { noun.codebookMax = sq8.max } - // Restore connections from persisted data - for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { - const level = parseInt(levelStr, 10) - noun.connections.set(level, new Set(nounIds as string[])) - } + // Restore connections from persisted data — compressed blob path + // first, legacy JSON-array fallback for indexes written before + // graph link compression landed. + await this.restoreNodeConnections(nounData.id, hnswData, noun) // Add to in-memory index this.nouns.set(nounData.id, noun) @@ -1519,11 +1610,10 @@ export class HNSWIndex implements HnswProvider { noun.codebookMax = sq8.max } - // Restore connections from persisted data - for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { - const level = parseInt(levelStr, 10) - noun.connections.set(level, new Set(nounIds as string[])) - } + // Restore connections from persisted data — compressed blob path + // first, legacy JSON-array fallback for indexes written before + // graph link compression landed. + await this.restoreNodeConnections(nounData.id, hnswData, noun) // Add to in-memory index this.nouns.set(nounData.id, noun) diff --git a/src/plugin.ts b/src/plugin.ts index 0cf04a6c..89e2bed4 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -253,6 +253,32 @@ export interface CacheProvider { // uses at the `getProvider('embeddings')` call site. No separate interface is // added here to avoid a duplicate, unwired contract. +/** + * The `'graph:compression'` provider — pure-function encode/decode for HNSW + * connection lists as compact delta-varint byte sequences (cortex's + * `encodeConnections` / `decodeConnections`). + * + * Brainy's `HNSWIndex` consumes this via a `ConnectionsCodec` that translates + * UUIDs to stable int slots via the `EntityIdMapper`, encodes, and persists + * the compressed bytes through the binary-blob primitive. On load, the blob + * is fetched + decoded back into UUID sets — `setConnectionsCodec()` on + * `HNSWIndex` is the injection point. Read path is dual-format: when no blob + * exists for a node, the connections fall back to the legacy JSON-array path + * embedded in `saveHNSWData`, so pre-2.4.0 indexes keep loading unchanged + * and convergence to the compressed form happens lazily on next save. + * + * Activated only when the storage adapter exposes the binary-blob primitive + * AND the metadata index resolves a stable idMapper. Cloud adapters that + * lack a real local-path resolution still benefit, since the blob primitive + * itself works across every adapter as of brainy 7.25.0. + */ +export interface GraphCompressionProvider { + /** Encode a list of u32 ints to compact delta-varint bytes. Sorts internally. */ + encode(ids: number[]): Buffer + /** Decode delta-varint bytes back to a u32 list. */ + decode(data: Buffer): number[] +} + /** * The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed * vector file. Brainy's HNSWIndex uses the static `.create()` / `.open()` diff --git a/tests/unit/hnsw/connections-codec.test.ts b/tests/unit/hnsw/connections-codec.test.ts new file mode 100644 index 00000000..c92c3888 --- /dev/null +++ b/tests/unit/hnsw/connections-codec.test.ts @@ -0,0 +1,137 @@ +/** + * @module hnsw/connectionsCodec.test + * @description Unit tests for `ConnectionsCodec` — the per-node buffer format + * brainy uses to persist HNSW connection sets as delta-varint-compressed + * binary blobs (2.4.0 #3). + * + * Mocks the `graph:compression` provider so the tests run without cortex + * installed. The real cross-language byte-format integration is exercised + * when cortex's `encodeConnections` / `decodeConnections` are wired in. + * + * Coverage: + * 1. Round-trip a Map of `level → Set` through encode + decode and + * verify the result is equal to the input (level-by-level Set equality). + * 2. Empty connections encode to a single-byte buffer (`0` = zero levels) and + * decode back to an empty Map — no `undefined` reads, no decode errors. + * 3. Multiple levels round-trip, including a level with a single UUID and a + * level with many UUIDs (the typical L0 fan-out). + * 4. UUIDs whose ints are no longer in the idMapper at decode time are + * silently dropped — matches the pre-2.4.0 behaviour of a missing + * `getUuid()` lookup in the legacy load path. + */ + +import { describe, it, expect } from 'vitest' +import { ConnectionsCodec } from '../../../src/hnsw/connectionsCodec.js' +import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js' +import type { GraphCompressionProvider } from '../../../src/plugin.js' + +const stubStorage = { + getMetadata: async () => undefined, + saveMetadata: async () => {}, + getNouns: async () => ({ totalCount: 0, items: [] }) +} as any + +/** + * Mock provider — encodes ints as a JSON byte array, decodes back. NOT the + * delta-varint format cortex produces, but format-agnostic round-trip + * verification is the unit-test concern here. + */ +const mockProvider: GraphCompressionProvider = { + encode(ids: number[]): Buffer { + const sorted = [...ids].sort((a, b) => a - b) + return Buffer.from(JSON.stringify(sorted), 'utf8') + }, + decode(data: Buffer): number[] { + if (data.length === 0) return [] + return JSON.parse(data.toString('utf8')) as number[] + } +} + +async function makeIdMapper(): Promise { + const mapper = new EntityIdMapper({ storage: stubStorage }) + await mapper.init() + return mapper +} + +function setsEqual(a: Set, b: Set): boolean { + if (a.size !== b.size) return false + for (const v of a) if (!b.has(v)) return false + return true +} + +describe('ConnectionsCodec (2.4.0 #3 — wraps graph:compression provider)', () => { + it('round-trips a single-level Set of UUIDs through encode + decode', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const input = new Map>([ + [0, new Set(['a', 'b', 'c', 'd'])] + ]) + const buf = codec.encode(input) + const out = codec.decode(buf) + + expect(out.size).toBe(1) + expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true) + }) + + it('empty connections encode to a one-byte buffer and decode to an empty Map', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const buf = codec.encode(new Map()) + expect(buf.length).toBe(1) + expect(buf.readUInt8(0)).toBe(0) + expect(codec.decode(buf).size).toBe(0) + }) + + it('round-trips multiple levels with very different fan-outs', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const manyUuids = Array.from({ length: 30 }, (_, i) => `u-${i}`) + const input = new Map>([ + [0, new Set(manyUuids)], + [1, new Set(['high-1', 'high-2'])], + [2, new Set(['solo'])] + ]) + + const buf = codec.encode(input) + const out = codec.decode(buf) + + expect(out.size).toBe(3) + expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true) + expect(setsEqual(out.get(1)!, input.get(1)!)).toBe(true) + expect(setsEqual(out.get(2)!, input.get(2)!)).toBe(true) + }) + + it('drops UUIDs whose ints the idMapper no longer recognises on decode', async () => { + const encodingMapper = await makeIdMapper() + const encodeCodec = new ConnectionsCodec(mockProvider, encodingMapper) + + // Encode through one mapper. + const input = new Map>([[0, new Set(['present', 'gone'])]]) + const buf = encodeCodec.encode(input) + + // Decode through a FRESH mapper that only knows about 'present'. + const decodingMapper = await makeIdMapper() + decodingMapper.getOrAssign('present') + + // Manually re-map 'present' to the same int it had on encode so the + // present UUID round-trips; 'gone' simply has no mapping → silent drop. + // We line up the int spaces by asserting equality below: only 'present' + // survives, 'gone' is missing — no errors, no `undefined` Set entries. + const decodeCodec = new ConnectionsCodec(mockProvider, decodingMapper) + // Inject the same int for 'present' so it round-trips one-for-one. (The + // encoding mapper assigns ints in insertion order: 'present'=1, 'gone'=2. + // The decoding mapper's first getOrAssign also returns 1, so 'present' + // matches; 'gone' (encoded as 2) has no mapping in the decoder.) + expect(encodingMapper.getInt('present')).toBe(1) + expect(decodingMapper.getInt('present')).toBe(1) + + const out = decodeCodec.decode(buf) + const l0 = out.get(0)! + expect(l0.has('present')).toBe(true) + expect(l0.has('gone')).toBe(false) + expect(l0.size).toBe(1) + }) +}) -- 2.49.1