diff --git a/src/brainy.ts b/src/brainy.ts index 8819824e..53d73a3b 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -37,7 +37,14 @@ import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' import { PluginRegistry } from './plugin.js' -import type { BrainyPlugin, BrainyPluginContext } from './plugin.js' +import type { + BrainyPlugin, + BrainyPluginContext, + GraphCompressionProvider, + VectorStoreMmapProvider +} from './plugin.js' +import { MmapVectorBackend } from './hnsw/mmapVectorBackend.js' +import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' import { ValidationConfig, @@ -559,6 +566,23 @@ export class Brainy implements BrainyInterface { this.graphIndex = graphIndex } + // Wire the mmap-vector backend (2.4.0 #2). When the vectorStore:mmap + // provider is registered AND the storage adapter resolves a real local + // path via getBinaryBlobPath(), open the mmap file there and inject the + // backend into HNSWIndex. Cloud adapters return null and skip silently; + // HNSWIndex's behaviour is then identical to pre-2.4.0 (per-entity reads + // from canonical storage). Failures are non-fatal — the index still + // works, just without the zero-copy fast path. + await this.wireMmapVectorBackend() + + // Wire the connections codec (2.4.0 #3). When the graph:compression + // provider is registered AND the metadata index exposes a stable + // idMapper, inject a codec that encodes HNSW connections as + // delta-varint blobs at save time and decodes on load. The blob + // primitive itself works on every brainy 7.25.0 adapter, so unlike the + // mmap-vector backend this layer engages even on cloud adapters. + this.wireConnectionsCodec() + // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() @@ -7716,6 +7740,101 @@ export class Brainy implements BrainyInterface { await this.lazyRebuildPromise } + /** + * @description Open the mmap-vector backend and inject it into HNSWIndex. + * Called once during init after plugin activation, metadataIndex init, and + * graphIndex setup — so the idMapper is available and providers are wired. + * + * Activation conditions (ALL must hold; otherwise this is a no-op and the + * index falls back to per-entity storage reads, the pre-2.4.0 behaviour): + * 1. The `vectorStore:mmap` provider is registered (cortex registers this). + * 2. The storage adapter resolves a real local path via `getBinaryBlobPath()` + * — cloud adapters return null and the mmap layer is skipped silently. + * 3. The metadata index exposes its idMapper (a stable UUID↔int map). The + * mmap layout is keyed by these ints, so 2.4.0 #1 is the prerequisite. + * + * Vector dimensionality: read from `this.dimensions` if set; otherwise the + * default for the standard embedding model (384). The dim is fixed at + * file-creation time and cannot change later, so a wrong default would only + * matter on the very first init of a brand-new instance. After the first + * `add()` the in-memory `this.dimensions` is set, and any subsequent init + * (or a re-open) sees the correct value. + * + * Failures here are non-fatal: a debug-level log records the reason, and + * HNSWIndex continues without the mmap fast path. + */ + private async wireMmapVectorBackend(): Promise { + const provider = this.pluginRegistry.getProvider('vectorStore:mmap') + if (!provider) return + + const storageWithBlob = this.storage as unknown as { + getBinaryBlobPath?: (key: string) => string | null + } + const vectorPath = storageWithBlob.getBinaryBlobPath?.('_vectors/main') ?? null + if (!vectorPath) return + + const idMapper = this.metadataIndex.getIdMapper?.() + if (!idMapper) return + + const dim = this.dimensions ?? 384 + const initialCapacity = Math.max(idMapper.size * 2, 1024) + + try { + const backend = await MmapVectorBackend.open( + provider, + vectorPath, + dim, + initialCapacity, + idMapper + ) + this.index.setVectorBackend(backend) + if (!this.config.silent) { + console.log( + `[brainy] vector mmap backend wired (path=${vectorPath}, dim=${dim}, capacity=${initialCapacity})` + ) + } + } catch (error) { + // Common cases: dim mismatch from a prior init at a different dimension, + // permission errors, disk full, file corruption. Index keeps working via + // the per-entity storage path; flag the cause for diagnosis. + if (!this.config.silent) { + console.warn( + `[brainy] mmap-vector backend not wired (${(error as Error).message}); ` + + `falling back to per-entity vector reads` + ) + } + } + } + + /** + * @description Wire the HNSW connections codec (2.4.0 #3). Activates when + * BOTH (a) the `graph:compression` provider is registered (cortex registers + * `{ encode: encodeConnections, decode: decodeConnections }`), and (b) the + * metadata index exposes a stable idMapper. Failures are non-fatal: HNSW + * keeps working via the legacy JSON-array path. + * + * Unlike the mmap-vector backend, this layer does NOT require a real local + * path — the binary-blob primitive saves the compressed bytes through the + * storage adapter directly, so the codec is engaged on cloud adapters too. + * Format convergence is lazy: pre-2.4.0 nodes load via the legacy path, + * then the next dirty save writes the compressed form (and the legacy + * field of saveHNSWData becomes empty), so all reads converge over time + * without an explicit migration step. + */ + private wireConnectionsCodec(): void { + const provider = this.pluginRegistry.getProvider('graph:compression') + if (!provider) return + + const idMapper = this.metadataIndex.getIdMapper?.() + if (!idMapper) return + + const codec = new ConnectionsCodec(provider, idMapper) + this.index.setConnectionsCodec(codec) + if (!this.config.silent) { + console.log('[brainy] graph link compression wired (delta-varint connections via graph:compression provider)') + } + } + /** * Rebuild indexes from persisted data if needed (LAZY LOADING) * diff --git a/src/hnsw/connectionsCodec.ts b/src/hnsw/connectionsCodec.ts new file mode 100644 index 00000000..91dc00b5 --- /dev/null +++ b/src/hnsw/connectionsCodec.ts @@ -0,0 +1,137 @@ +/** + * @module hnsw/connectionsCodec + * @description Bridge between brainy's UUID-keyed HNSW connection sets and + * cortex's int-keyed delta-varint encode/decode (the `graph:compression` + * provider). Translates UUIDs to stable int slots via the post-2.4.0 #1 + * `EntityIdMapper`, batches all of a node's per-level connection lists into + * a single compact buffer, and reverses the path on load. + * + * Wire format of the per-node buffer the codec produces: + * + * [num_levels: u8] + * repeated num_levels times: + * [level: u8] + * [encoded_len: u32 LE] + * [encoded_bytes: bytes] <- provider.encode(ints) + * + * Why a single buffer per node and not one blob per level: + * - One `saveBinaryBlob` call per persisted node, regardless of level count. + * On hot insert paths an HNSW node may be at levels 0..3; the per-node + * blob trades a one-byte count for 3 saved I/O round-trips. + * - Read path is symmetric: one `loadBinaryBlob` reconstructs the full + * connection set, so HNSW rebuild stays linear in the number of nodes. + * + * Loose UUIDs (ints assigned to entities that no longer exist) are silently + * dropped on decode. They can't legitimately participate in graph traversal + * because the entity is gone; treating them as `undefined` UUIDs matches the + * pre-2.4.0 behaviour of an `idMapper.getUuid()` miss in the legacy load path. + */ + +import type { + EntityIdMapperProvider, + GraphCompressionProvider +} from '../plugin.js' + +export class ConnectionsCodec { + constructor( + private readonly provider: GraphCompressionProvider, + private readonly idMapper: EntityIdMapperProvider + ) {} + + /** + * @description Encode all levels of a node's connections into one compact + * buffer. Empty input → an empty buffer (zero levels, no body). The + * idMapper assigns stable ints on demand if a connection UUID has never + * been seen — append-only, matches the rest of the 2.4.0 contract. + */ + encode(connectionsByLevel: Map>): Buffer { + if (connectionsByLevel.size === 0) { + // Single byte: zero levels. Round-trips cleanly through decode. + return Buffer.from([0]) + } + if (connectionsByLevel.size > 0xff) { + throw new Error( + `ConnectionsCodec.encode: too many HNSW levels (${connectionsByLevel.size}); ` + + `the per-node header is one byte (max 255 levels).` + ) + } + + // First pass: encode each level into its own buffer. + const levelBuffers: Array<{ level: number; buf: Buffer }> = [] + let bodyBytes = 0 + for (const [level, uuids] of connectionsByLevel) { + if (level > 0xff) { + throw new Error( + `ConnectionsCodec.encode: level ${level} exceeds u8 header range (max 255).` + ) + } + const ints: number[] = [] + for (const uuid of uuids) { + ints.push(this.idMapper.getOrAssign(uuid)) + } + const buf = this.provider.encode(ints) + levelBuffers.push({ level, buf }) + bodyBytes += 1 + 4 + buf.length // level + len + payload + } + + // Second pass: concatenate with header. + const out = Buffer.alloc(1 + bodyBytes) + out.writeUInt8(levelBuffers.length, 0) + let offset = 1 + for (const { level, buf } of levelBuffers) { + out.writeUInt8(level, offset) + offset += 1 + out.writeUInt32LE(buf.length, offset) + offset += 4 + buf.copy(out, offset) + offset += buf.length + } + return out + } + + /** + * @description Decode a per-node buffer back into a connections Map. Ints + * for entities the idMapper no longer knows are silently dropped — see the + * module-level note on why. + */ + decode(data: Buffer): Map> { + const out = new Map>() + if (data.length === 0) return out + + const count = data.readUInt8(0) + let offset = 1 + for (let i = 0; i < count; i++) { + if (offset + 5 > data.length) { + throw new Error(`ConnectionsCodec.decode: truncated header at level ${i}`) + } + const level = data.readUInt8(offset) + offset += 1 + const len = data.readUInt32LE(offset) + offset += 4 + if (offset + len > data.length) { + throw new Error(`ConnectionsCodec.decode: truncated body at level ${level}`) + } + const levelBuf = data.slice(offset, offset + len) + offset += len + + const ints = this.provider.decode(levelBuf) + const uuids = new Set() + for (const intId of ints) { + const uuid = this.idMapper.getUuid(intId) + if (uuid !== undefined) uuids.add(uuid) + } + out.set(level, uuids) + } + return out + } +} + +/** + * @description Build the binary-blob storage key for a node's compressed + * connections. Suffix-free; the adapter appends its own. Keys live under + * `_hnsw_conn/` so they don't collide with the existing `_column_index/` + * blobs and so a cortex-side reader knows exactly where to look. + */ +export function compressedConnectionsKey(nodeId: string): string { + return `_hnsw_conn/${nodeId}` +} diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index f5af9137..ff074daf 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -18,6 +18,8 @@ import { prodLog } from '../utils/logger.js' import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js' import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js' import type { HnswProvider } from '../plugin.js' +import { MmapVectorBackend } from './mmapVectorBackend.js' +import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js' // Default HNSW parameters const DEFAULT_CONFIG: HNSWConfig = { @@ -49,6 +51,23 @@ export class HNSWIndex implements HnswProvider { private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes // Always-adaptive caching - no "mode" concept, system adapts automatically + // Optional mmap-vector backend (2.4.0 #2). When set, the read paths + // (preloadVectors, getVectorSafe) try the mmap layer before storage, and + // on a storage hit they write back into the mmap slot — a lazy migration + // that converges upgraded installs to the fast path under live traffic. + // Null on cloud storage adapters (no local path) and pre-injection init. + private vectorBackend: MmapVectorBackend | null = null + + // Optional connections codec (2.4.0 #3). When set, node-persist writes the + // node's per-level connection sets as a single delta-varint-compressed + // binary blob (typically ~4× smaller than the legacy JSON-UUID-array + // shape), plus an empty `connections: {}` in saveHNSWData as the marker. + // The read path tries to load the blob first; missing blob → legacy + // connections field. Format convergence is lazy: pre-2.4.0 nodes still + // load via the legacy path, then write the compressed form on next dirty + // save, so the migration converges under live traffic with no big-bang. + private connectionsCodec: ConnectionsCodec | null = null + // COW (Copy-on-Write) support private cowEnabled: boolean = false private cowModifiedNodes: Set = new Set() @@ -101,6 +120,35 @@ export class HNSWIndex implements HnswProvider { this.useParallelization = useParallelization } + /** + * @description Inject (or detach) the mmap-vector backend. When set, the + * vector read paths (`preloadVectors`, `getVectorSafe`) try the mmap layer + * before storage and lazily write back on storage hits — converging an + * upgraded install to the zero-copy fast path without a migration step. + * Setting to null reverts to the existing per-entity read path. + * + * Lifecycle: brainy.ts wires this after plugin activation when (a) the + * `vectorStore:mmap` provider is registered AND (b) the storage adapter + * resolves a real local path via `getBinaryBlobPath()`. Cloud adapters + * leave it null; HNSWIndex's behaviour is then identical to pre-2.4.0. + */ + public setVectorBackend(backend: MmapVectorBackend | null): void { + this.vectorBackend = backend + } + + /** + * @description Inject (or detach) the connections codec. When set, node + * persistence writes a delta-varint-compressed binary blob alongside an + * empty `connections: {}` marker in saveHNSWData; the load path reads the + * blob and decodes. Null reverts to the legacy JSON-array path. Wiring is + * done by brainy.ts when the `graph:compression` provider is registered + * AND the storage adapter exposes the binary-blob primitive AND the + * metadata index has a stable idMapper. + */ + public setConnectionsCodec(codec: ConnectionsCodec | null): void { + this.connectionsCodec = codec + } + /** * Get whether parallelization is enabled */ @@ -144,15 +192,7 @@ export class HNSWIndex implements HnswProvider { const noun = this.nouns.get(nodeId) if (!noun) return Promise.resolve() // Node was deleted - const connectionsObj: Record = {} - for (const [level, nounIds] of noun.connections.entries()) { - connectionsObj[level.toString()] = Array.from(nounIds) - } - - return this.storage!.saveHNSWData(nodeId, { - level: noun.level, - connections: connectionsObj - }).catch(error => { + return this.persistNodeConnections(nodeId, noun).catch(error => { console.error(`[HNSW flush] Failed to persist node ${nodeId}:`, error) }) }) @@ -183,6 +223,94 @@ export class HNSWIndex implements HnswProvider { return nodeCount } + /** + * @description Persist one node's connections. When the connections codec is + * wired AND the storage adapter exposes `saveBinaryBlob`, the per-level + * connection sets are encoded into a single compact buffer and stored as a + * binary blob; `saveHNSWData` then records the node's level and an empty + * `connections: {}` as the marker. Otherwise the legacy JSON-array path is + * taken — the format every brainy release before 2.4.0 wrote. The two are + * intentionally NOT dual-written: one or the other, never both, so format + * convergence is unambiguous on the read side. + * + * Shared by all three save sites (deferred flush, immediate-mode new-entity + * persist, immediate-mode neighbor update) so the codec branch is exercised + * uniformly regardless of which path triggered the write. + */ + private async persistNodeConnections(nodeId: string, noun: HNSWNoun): Promise { + if (!this.storage) return + + const storageWithBlob = this.storage as unknown as { + saveBinaryBlob?: (key: string, data: Buffer) => Promise + } + const canCompress = + this.connectionsCodec !== null && + typeof storageWithBlob.saveBinaryBlob === 'function' + + if (canCompress) { + const encoded = this.connectionsCodec!.encode(noun.connections) + await storageWithBlob.saveBinaryBlob!(compressedConnectionsKey(nodeId), encoded) + await this.storage.saveHNSWData(nodeId, { + level: noun.level, + connections: {} + }) + return + } + + const connectionsObj: Record = {} + for (const [level, nounIds] of noun.connections.entries()) { + connectionsObj[level.toString()] = Array.from(nounIds) + } + await this.storage.saveHNSWData(nodeId, { + level: noun.level, + connections: connectionsObj + }) + } + + /** + * @description Restore one node's connections from persisted storage. Tries + * the compressed-blob path first (when codec wired AND blob primitive + * available); a present non-empty buffer is decoded and populates + * `noun.connections`. A missing blob (no payload at the key) means this node + * was last persisted under the legacy format, so the legacy + * `hnswData.connections` field is used instead. + * + * On any decode error the legacy field is also used as a safety net — the + * codec's `decode` throws on truncated input, which would otherwise leave + * the node with an empty connections Map (orphaned from the graph). + */ + private async restoreNodeConnections( + nodeId: string, + hnswData: { level: number; connections: Record }, + noun: HNSWNoun + ): Promise { + const storageWithBlob = this.storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + } + const canDecompress = + this.connectionsCodec !== null && + typeof storageWithBlob.loadBinaryBlob === 'function' + + if (canDecompress) { + try { + const buf = await storageWithBlob.loadBinaryBlob!(compressedConnectionsKey(nodeId)) + if (buf && buf.length > 0) { + const decoded = this.connectionsCodec!.decode(buf) + noun.connections = decoded + return + } + } catch (error) { + prodLog.debug(`HNSW: compressed connections decode failed for ${nodeId}; falling back to legacy`, error) + } + } + + // Legacy fallback: JSON UUID arrays in the HNSW data object. + for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { + const level = parseInt(levelStr, 10) + noun.connections.set(level, new Set(nounIds as string[])) + } + } + /** * Get the number of dirty (unpersisted) nodes * Useful for monitoring and debugging @@ -521,18 +649,12 @@ export class HNSWIndex implements HnswProvider { // In deferred mode, we track dirty nodes instead of persisting immediately // This reduces GCS operations from 70 to 2-3 per add() (30-50× faster) if (this.storage && this.persistMode === 'immediate') { - // IMMEDIATE MODE: Original behavior - persist each neighbor update - const neighborConnectionsObj: Record = {} - for (const [lvl, nounIds] of neighbor.connections.entries()) { - neighborConnectionsObj[lvl.toString()] = Array.from(nounIds) - } - + // IMMEDIATE MODE: Original behavior - persist each neighbor update. + // Goes through the per-node helper so the compressed-blob branch + // fires identically here vs. the deferred-flush path. neighborUpdates.push({ neighborId, - promise: this.storage.saveHNSWData(neighborId, { - level: neighbor.level, - connections: neighborConnectionsObj - }) + promise: this.persistNodeConnections(neighborId, neighbor) }) } else if (this.persistMode === 'deferred') { // DEFERRED MODE: Track dirty nodes for later batch persistence @@ -619,16 +741,10 @@ export class HNSWIndex implements HnswProvider { // Persist HNSW graph data to storage // Respect persistMode setting if (this.storage && this.persistMode === 'immediate') { - // IMMEDIATE MODE: Original behavior - persist new entity and system data - const connectionsObj: Record = {} - for (const [level, nounIds] of noun.connections.entries()) { - connectionsObj[level.toString()] = Array.from(nounIds) - } - - await this.storage.saveHNSWData(id, { - level: nounLevel, - connections: connectionsObj - }).catch((error) => { + // IMMEDIATE MODE: Original behavior - persist new entity and system data. + // Goes through the per-node helper so the compressed-blob branch fires + // identically here vs. the deferred-flush + neighbor-update paths. + await this.persistNodeConnections(id, noun).catch((error) => { console.error(`Failed to persist HNSW data for ${id}:`, error) }) @@ -1073,7 +1189,23 @@ export class HNSWIndex implements HnswProvider { const cacheKey = `hnsw:vector:${noun.id}` const vector = await this.unifiedCache.get(cacheKey, async () => { - // Cache miss - load from storage + // Try the mmap-vector backend first when wired — zero-copy slot read, + // no per-entity object hydration. + if (this.vectorBackend) { + const fromMmap = this.vectorBackend.readByUuid(noun.id) + if (fromMmap) { + this.unifiedCache.set( + cacheKey, + fromMmap, + 'hnsw', + fromMmap.length * 4, + 50 + ) + return fromMmap + } + } + + // Storage fallback — the canonical source of truth. if (!this.storage) { throw new Error('Storage not available for vector loading') } @@ -1083,6 +1215,18 @@ export class HNSWIndex implements HnswProvider { throw new Error(`Vector not found for noun ${noun.id}`) } + // Lazy migration: write back into the mmap slot so the next read is + // mmap-fast. Idempotent + resumable; a write failure is non-fatal — + // the field still serves this read, and the next storage hit re-tries + // the migration. (See mmapVectorBackend.ts for the contract.) + if (this.vectorBackend) { + try { + this.vectorBackend.writeByUuid(noun.id, loaded) + } catch (error) { + prodLog.debug(`MmapVectorBackend write-back failed for ${noun.id}`, error) + } + } + // Add to UnifiedCache with cost-aware eviction // This competes fairly with Graph and Metadata indexes this.unifiedCache.set( @@ -1140,7 +1284,50 @@ export class HNSWIndex implements HnswProvider { private async preloadVectors(nodeIds: string[]): Promise { if (nodeIds.length === 0) return - // Use UnifiedCache's request coalescing to prevent duplicate loads + // mmap-vector backend fast path: batched O(1) slot reads + lazy + // write-back migration on miss. Cache populated from both layers so + // subsequent in-session reads are O(1) memory. + if (this.vectorBackend) { + // Filter out ids already cached so we never re-read. + const uncachedIds: string[] = [] + for (const id of nodeIds) { + if (this.unifiedCache.getSync(`hnsw:vector:${id}`) === undefined) { + uncachedIds.push(id) + } + } + if (uncachedIds.length === 0) return + + const fromMmap = this.vectorBackend.readBatchByUuid(uncachedIds) + const misses: string[] = [] + for (let i = 0; i < uncachedIds.length; i++) { + const id = uncachedIds[i] + const v = fromMmap[i] + if (v) { + this.unifiedCache.set(`hnsw:vector:${id}`, v, 'hnsw', v.length * 4, 50) + } else { + misses.push(id) + } + } + if (misses.length === 0) return + + // Storage fallback for the misses, with concurrent lazy write-back. + await Promise.all(misses.map(async (id) => { + if (!this.storage) return + const vector = await this.storage.getNounVector(id) + if (!vector) return + if (this.vectorBackend) { + try { + this.vectorBackend.writeByUuid(id, vector) + } catch (error) { + prodLog.debug(`MmapVectorBackend write-back failed for ${id}`, error) + } + } + this.unifiedCache.set(`hnsw:vector:${id}`, vector, 'hnsw', vector.length * 4, 50) + })) + return + } + + // Legacy path: per-entity storage load via UnifiedCache request coalescing. const promises = nodeIds.map(async (id) => { const cacheKey = `hnsw:vector:${id}` return this.unifiedCache.get(cacheKey, async () => { @@ -1339,11 +1526,10 @@ export class HNSWIndex implements HnswProvider { noun.codebookMax = sq8.max } - // Restore connections from persisted data - for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { - const level = parseInt(levelStr, 10) - noun.connections.set(level, new Set(nounIds as string[])) - } + // Restore connections from persisted data — compressed blob path + // first, legacy JSON-array fallback for indexes written before + // graph link compression landed. + await this.restoreNodeConnections(nounData.id, hnswData, noun) // Add to in-memory index this.nouns.set(nounData.id, noun) @@ -1424,11 +1610,10 @@ export class HNSWIndex implements HnswProvider { noun.codebookMax = sq8.max } - // Restore connections from persisted data - for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { - const level = parseInt(levelStr, 10) - noun.connections.set(level, new Set(nounIds as string[])) - } + // Restore connections from persisted data — compressed blob path + // first, legacy JSON-array fallback for indexes written before + // graph link compression landed. + await this.restoreNodeConnections(nounData.id, hnswData, noun) // Add to in-memory index this.nouns.set(nounData.id, noun) diff --git a/src/hnsw/mmapVectorBackend.ts b/src/hnsw/mmapVectorBackend.ts new file mode 100644 index 00000000..fa9d4e37 --- /dev/null +++ b/src/hnsw/mmapVectorBackend.ts @@ -0,0 +1,174 @@ +/** + * @module hnsw/mmapVectorBackend + * @description Disk-resident vector cache backing HNSWIndex, wrapping the + * `vectorStore:mmap` provider (cortex's NativeMmapVectorStore in production). + * + * **Why this exists.** Brainy's per-entity vector storage hits two ceilings as + * datasets grow: (1) every `HNSWIndex.preloadVectors` does a sequential + * `storage.getNounVector` per id (no batch), and (2) cloud and FS storage both + * serialize each vector through JSON / msgpack, blocking zero-copy reads. An + * mmap-backed single-file vector store, addressed by the stable int from + * `EntityIdMapper`, gives O(1) batch reads + OS-page-cache prefetch and is the + * foundation 2.4.0 #2 needs. + * + * **Lazy migration on miss.** First read of an unmigrated UUID falls back to + * `storage.getNounVector` and writes the bytes into the mmap slot before + * returning — idempotent, resumable, and lets an upgraded install converge to + * the fast path under live traffic without a big-bang migration step. The + * per-entity store remains the canonical source of truth; nothing in the per- + * entity layer is rewritten or deleted here. + * + * **Local-FS only in 2.4.0.** Activated when the storage adapter exposes a + * real local path via `getBinaryBlobPath()`. Cloud adapters (S3 / R2 / GCS) + * return null and the backend is not constructed, so HNSWIndex falls back to + * the existing per-entity read path. Chunked vector segments for remote + * storage are the planned 2.4.0 #2 follow-up. + */ + +import { mkdir } from 'node:fs/promises' +import { dirname } from 'node:path' +import type { Vector } from '../coreTypes.js' +import type { + EntityIdMapperProvider, + VectorStoreMmapInstance, + VectorStoreMmapProvider +} from '../plugin.js' + +/** + * @description Bridge between brainy's UUID-keyed vector API and cortex's + * int-slot mmap vector store. Translates UUIDs to stable int slots via the + * `EntityIdMapper`, grows the file on demand, and handles read misses by + * surfacing `null` (HNSWIndex's caller does the lazy storage fallback + + * write-back). The class never touches per-entity storage directly — it owns + * only the mmap layer. + */ +export class MmapVectorBackend { + private readonly store: VectorStoreMmapInstance + private readonly idMapper: EntityIdMapperProvider + + private constructor(store: VectorStoreMmapInstance, idMapper: EntityIdMapperProvider) { + this.store = store + this.idMapper = idMapper + } + + /** + * @description Open or create the mmap vector file at `path`. If the file + * does not exist, it is created with the given dim + initial capacity. If + * the directory containing `path` does not exist, it is created recursively + * (the cortex provider does not auto-create parent directories). + * @param provider - The `vectorStore:mmap` provider (cortex + * `NativeMmapVectorStore` in production). + * @param path - Absolute path to the vector file. Derived by the caller + * from `storage.getBinaryBlobPath()` on local-FS adapters; cloud adapters + * return null and this method is not called. + * @param dim - Vector dimensionality. Must match the embedding model. + * @param initialCapacity - Slot capacity to allocate when creating a new + * file. The file grows automatically on writes beyond capacity. + * @param idMapper - Stable UUID↔int mapper (post-2.4.0 #1 semantics — never + * renumbers across rebuild). The mmap slots are addressed by these ints, + * so id stability is the load-bearing invariant of the layout. + * @returns A ready-to-use backend. + */ + static async open( + provider: VectorStoreMmapProvider, + path: string, + dim: number, + initialCapacity: number, + idMapper: EntityIdMapperProvider + ): Promise { + await mkdir(dirname(path), { recursive: true }) + let store: VectorStoreMmapInstance + try { + store = provider.open(path) + } catch { + store = provider.create(path, dim, Math.max(initialCapacity, 16)) + } + return new MmapVectorBackend(store, idMapper) + } + + /** Vector dimensionality of the underlying mmap file. */ + get dim(): number { + return this.store.dim + } + + /** + * @description Read a vector by UUID. Returns `null` for any of: + * (1) UUID not yet assigned an int (entity never written through this + * backend or storage), (2) int beyond the highest written slot, or + * (3) the underlying mmap read throwing (corrupt / out-of-bounds). + * The null path is the caller's signal to fall back to per-entity storage + * and trigger a write-back migration. + */ + readByUuid(uuid: string): Vector | null { + const intId = this.idMapper.getInt(uuid) + if (intId === undefined || intId >= this.store.count) return null + try { + return this.store.readVector(intId) as unknown as Vector + } catch { + return null + } + } + + /** + * @description Batch read vectors by UUID. Returns an array aligned with + * `uuids` where each entry is the vector or `null` for a miss. Order- + * preserving. Internally collapses to a single batched mmap read of only the + * indices that resolve to a written slot, then redistributes the flat output + * back to the caller's order — so missing slots cost only a couple of map + * lookups, not a full N-element read. + */ + readBatchByUuid(uuids: string[]): (Vector | null)[] { + const indices: number[] = [] + const slotByUuid = new Map() + for (const uuid of uuids) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined && intId < this.store.count) { + slotByUuid.set(uuid, indices.length) + indices.push(intId) + } + } + if (indices.length === 0) return uuids.map(() => null) + const flat = this.store.readVectorsBatch(indices) + const dim = this.store.dim + return uuids.map(uuid => { + const slot = slotByUuid.get(uuid) + if (slot === undefined) return null + return flat.slice(slot * dim, (slot + 1) * dim) as unknown as Vector + }) + } + + /** + * @description Write a vector at the slot for `uuid`. Assigns a new int if + * the UUID is unseen (stable, append-only). Grows the underlying file (slot + * capacity doubled until the target index fits) before writing past + * capacity, so the caller need not worry about file sizing. + */ + writeByUuid(uuid: string, vector: Vector): void { + const intId = this.idMapper.getOrAssign(uuid) + if (intId >= this.store.capacity) { + let newCap = Math.max(this.store.capacity, 16) + while (intId >= newCap) newCap *= 2 + this.store.resize(newCap) + } + this.store.writeVector(intId, Array.from(vector as ArrayLike)) + } + + /** + * @description madvise(WILLNEED) hint for the slots backing `uuids`. Cheap; + * silently drops UUIDs not yet in the mmap. Used by HNSW search to overlap + * disk I/O with graph traversal. + */ + prefetchByUuid(uuids: string[]): void { + const indices: number[] = [] + for (const uuid of uuids) { + const intId = this.idMapper.getInt(uuid) + if (intId !== undefined && intId < this.store.count) indices.push(intId) + } + if (indices.length > 0) this.store.prefetch(indices) + } + + /** msync — flush dirty pages to disk. */ + flush(): void { + this.store.flush() + } +} diff --git a/src/indexes/columnStore/ColumnStore.ts b/src/indexes/columnStore/ColumnStore.ts index 6b264463..f290274f 100644 --- a/src/indexes/columnStore/ColumnStore.ts +++ b/src/indexes/columnStore/ColumnStore.ts @@ -125,12 +125,29 @@ export class ColumnStore implements ColumnStoreProvider { this.manifests.set(fieldName, manifest) this.fieldTypes.set(fieldName, manifest.valueType) - // Load global deleted bitmap if it exists + // Load global deleted bitmap if it exists. Raw blob preferred + // (2.4.0 #4 cortex-shared format); legacy envelope fallback for + // indexes written before format unification. try { - const deletedPath = `${this.basePath}/${fieldName}/DELETED.bin` - const stored = await (storage as any).readObjectFromPath(deletedPath) - if (stored && stored._binary && stored.data) { - const buf = Buffer.from(stored.data, 'base64') + let buf: Buffer | null = null + + const adapter = storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + readObjectFromPath: (path: string) => Promise + } + if (typeof adapter.loadBinaryBlob === 'function') { + const key = `${this.basePath}/${fieldName}/DELETED` + const blob = await adapter.loadBinaryBlob(key) + if (blob && blob.length > 0) buf = blob + } + if (!buf) { + const deletedPath = `${this.basePath}/${fieldName}/DELETED.bin` + const stored = await adapter.readObjectFromPath(deletedPath) + if (stored && stored._binary && stored.data) { + buf = Buffer.from(stored.data, 'base64') + } + } + if (buf) { this.deletedEntities.set(fieldName, RoaringBitmap32.deserialize(buf, true)) } } catch { @@ -408,6 +425,23 @@ export class ColumnStore implements ColumnStoreProvider { /** * Flush a single field's tail buffer to a new L0 segment. + * + * Storage path (2.4.0 #4 / cortex-interchange contract): + * When the storage adapter exposes the binary-blob primitive + * (`saveBinaryBlob` — every brainy adapter ≥7.25.0 does), the segment + * bytes are written as a raw blob at the shared cortex key + * `_column_index//L-NNNNNN` (suffix-free; the adapter + * appends its own). This matches byte-for-byte what cortex's + * `NativeColumnStore` writes, so JS- and native-written indexes + * interchange without re-encoding. + * + * When the adapter doesn't (older custom adapters that didn't follow + * the 7.25.0 primitive surface), we fall back to the legacy + * `writeObjectToPath` envelope — `{ _binary, data: }` at + * `_column_index//L-NNNNNN.cidx`. Cortex's 2.3.1 + * read-side fallback handles the legacy envelope on its end, so the + * format mismatch is transparent in both directions during the + * transition. */ private async flushBuffer(field: string, buffer: ColumnTailBuffer): Promise { const { values, entityIds, pendingRemovals } = buffer.drain() @@ -416,10 +450,15 @@ export class ColumnStore implements ColumnStoreProvider { const manifest = this.manifests.get(field) if (!manifest) return + const storage = this.storage as unknown as { + saveBinaryBlob?: (key: string, data: Buffer) => Promise + writeObjectToPath: (path: string, value: unknown) => Promise + } + const canUseBlob = typeof storage.saveBinaryBlob === 'function' + // Write segment if we have values if (values.length > 0) { const segId = manifest.nextSegmentId() - const segPath = manifest.segmentPath(0, segId) const segBuffer = writeSegmentToBuffer( { @@ -435,13 +474,22 @@ export class ColumnStore implements ColumnStoreProvider { entityIds ) - // Store segment as binary data - await (this.storage as any).writeObjectToPath(segPath, { - _binary: true, - data: segBuffer.toString('base64') - }) + if (canUseBlob) { + // Raw blob, cortex-shared key convention. + const key = `${this.basePath}/${field}/L0-${String(segId).padStart(6, '0')}` + await storage.saveBinaryBlob!(key, segBuffer) + } else { + // Legacy envelope path for adapters without the binary-blob primitive. + const segPath = manifest.segmentPath(0, segId) + await storage.writeObjectToPath(segPath, { + _binary: true, + data: segBuffer.toString('base64') + }) + } - // Register in manifest + // Register in manifest. The `.cidx` file name in the manifest is the + // legacy-format file the cortex 2.3.1 read-side fallback looks for; the + // raw-blob key derives from segId at read time. Both paths converge. manifest.addSegment({ id: segId, level: 0, @@ -465,15 +513,21 @@ export class ColumnStore implements ColumnStoreProvider { for (const id of pendingRemovals) deleted.add(id) } - // Persist the global deleted bitmap alongside the manifest + // Persist the global deleted bitmap alongside the manifest. Same + // raw-blob preference as segments — shared cortex key `//DELETED`. const deleted = this.deletedEntities.get(field) if (deleted && deleted.size > 0) { - const deletedPath = `${this.basePath}/${field}/DELETED.bin` const serialized = deleted.serialize(true) - await (this.storage as any).writeObjectToPath(deletedPath, { - _binary: true, - data: Buffer.from(serialized).toString('base64') - }) + if (canUseBlob) { + const deletedKey = `${this.basePath}/${field}/DELETED` + await storage.saveBinaryBlob!(deletedKey, Buffer.from(serialized)) + } else { + const deletedPath = `${this.basePath}/${field}/DELETED.bin` + await storage.writeObjectToPath(deletedPath, { + _binary: true, + data: Buffer.from(serialized).toString('base64') + }) + } } // Save manifest @@ -508,24 +562,43 @@ export class ColumnStore implements ColumnStoreProvider { /** * Load a segment from storage and create a cursor. + * + * Reads the raw-blob layout first (the 2.4.0 #4 shared-with-cortex format), + * then falls back to the legacy `{ _binary, base64 }` envelope at the + * `.cidx` object-path so indexes written before the format unification keep + * loading correctly. Mirror of cortex's 2.3.1 read-side fallback. */ private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise { const manifest = this.manifests.get(field) if (!manifest) return null try { - const segPath = manifest.segmentPath(seg.level, seg.id) - const stored = await (this.storage as any).readObjectFromPath(segPath) - if (!stored) return null + let buf: Buffer | null = null - // Handle binary storage format - let buf: Buffer - if (stored._binary && stored.data) { - buf = Buffer.from(stored.data, 'base64') - } else if (Buffer.isBuffer(stored)) { - buf = stored - } else { - return null + const storage = this.storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + readObjectFromPath: (path: string) => Promise + } + + // Preferred: raw blob at the cortex-shared key. + if (typeof storage.loadBinaryBlob === 'function') { + const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}` + const blob = await storage.loadBinaryBlob(key) + if (blob && blob.length > 0) buf = blob + } + + // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path. + if (!buf) { + const segPath = manifest.segmentPath(seg.level, seg.id) + const stored = await storage.readObjectFromPath(segPath) + if (!stored) return null + if (stored._binary && stored.data) { + buf = Buffer.from(stored.data, 'base64') + } else if (Buffer.isBuffer(stored)) { + buf = stored + } else { + return null + } } const parsed = readSegmentFromBuffer(buf) diff --git a/src/plugin.ts b/src/plugin.ts index 18652a65..89e2bed4 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -253,6 +253,81 @@ export interface CacheProvider { // uses at the `getProvider('embeddings')` call site. No separate interface is // added here to avoid a duplicate, unwired contract. +/** + * The `'graph:compression'` provider — pure-function encode/decode for HNSW + * connection lists as compact delta-varint byte sequences (cortex's + * `encodeConnections` / `decodeConnections`). + * + * Brainy's `HNSWIndex` consumes this via a `ConnectionsCodec` that translates + * UUIDs to stable int slots via the `EntityIdMapper`, encodes, and persists + * the compressed bytes through the binary-blob primitive. On load, the blob + * is fetched + decoded back into UUID sets — `setConnectionsCodec()` on + * `HNSWIndex` is the injection point. Read path is dual-format: when no blob + * exists for a node, the connections fall back to the legacy JSON-array path + * embedded in `saveHNSWData`, so pre-2.4.0 indexes keep loading unchanged + * and convergence to the compressed form happens lazily on next save. + * + * Activated only when the storage adapter exposes the binary-blob primitive + * AND the metadata index resolves a stable idMapper. Cloud adapters that + * lack a real local-path resolution still benefit, since the blob primitive + * itself works across every adapter as of brainy 7.25.0. + */ +export interface GraphCompressionProvider { + /** Encode a list of u32 ints to compact delta-varint bytes. Sorts internally. */ + encode(ids: number[]): Buffer + /** Decode delta-varint bytes back to a u32 list. */ + decode(data: Buffer): number[] +} + +/** + * The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed + * vector file. Brainy's HNSWIndex uses the static `.create()` / `.open()` + * factories to open a single-file store at a derived path on disk-resident + * storage adapters; non-FS adapters with no local-path support skip the mmap + * layer and fall back to per-entity vector reads. + * + * Cortex registers `NativeMmapVectorStore` as this provider — a Rust mmap'd + * f32 array with a header, batch read/write APIs, and madvise prefetch. Slot + * indices are the stable ints from `EntityIdMapper`; lazy migration on read + * miss writes back from per-entity storage into the slot, converging an + * upgraded install to the fast path without a big-bang step. + */ +export interface VectorStoreMmapProvider { + /** Create a new vector file with pre-allocated slot capacity. */ + create(path: string, dim: number, capacity: number): VectorStoreMmapInstance + /** Open an existing vector file for read + write. */ + open(path: string): VectorStoreMmapInstance + /** Open an existing vector file for read-only access. */ + openReadOnly(path: string): VectorStoreMmapInstance +} + +/** + * An open mmap-vector-store instance — the surface brainy's HNSWIndex consumes + * for batch vector reads + lazy migration from per-entity storage on miss. + */ +export interface VectorStoreMmapInstance { + /** Number of vectors written (highest written index + 1). */ + readonly count: number + /** Maximum slot capacity before resize is needed. */ + readonly capacity: number + /** Vector dimensionality. */ + readonly dim: number + /** Write a single vector at `index`. Vector is f64 in JS, stored as f32. */ + writeVector(index: number, vector: number[]): void + /** Write a flat batch starting at `startIndex`. Returns vectors written. */ + writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number + /** Read a single vector at `index`. Returns f64[]. */ + readVector(index: number): number[] + /** Read multiple vectors by index. Returns flat f64[] of length n × dim. */ + readVectorsBatch(indices: number[]): number[] + /** Prefetch via madvise(WILLNEED); no-op on unsupported platforms. */ + prefetch(indices: number[]): void + /** Grow the vector file to a larger slot capacity. Re-maps the file. */ + resize(newCapacity: number): void + /** msync — flush pending writes to disk. */ + flush(): void +} + /** * Storage adapter factory — plugins register these to provide * new storage backends that users reference by name. diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index ff67a4e9..5333c025 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -1,14 +1,34 @@ /** - * EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs for roaring bitmaps + * EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs. * - * Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings as entity IDs. - * This class provides efficient bidirectional mapping with persistence support. + * Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings + * as canonical entity IDs. This class provides efficient O(1) bidirectional + * mapping with persistence — and, importantly, a stability guarantee that any + * persisted int-keyed data can rely on. + * + * **Stability guarantee (the foundation 2.4.0 vector-mmap, graph-link-compression, + * and column-store interchange all key off):** + * + * - `getOrAssign(uuid)` is **append-only**: once a UUID is assigned an int, the + * mapping never changes. Subsequent `getOrAssign` calls for the same UUID + * return the same int. + * - `nextId` is **monotonically increasing**. New UUIDs always get an int greater + * than any previously assigned, so a removed-then-re-added UUID is treated as + * a fresh entity (and gets a fresh int — there is no automatic "revive"). + * - `remove(uuid)` removes the mapping but does **not** decrement `nextId` or + * recycle the int. The removed int becomes a permanent hole in `intToUuid` — + * downstream consumers seeing `getUuid(int) === undefined` know the entity + * was deleted. + * - A metadata-index `rebuild()` does **not** clear the mapper (the rebuild path + * re-iterates entities via `getOrAssign`, which returns existing ints unchanged). + * Only the explicit `clear()` method renumbers — used by `clearAllIndexData()` + * as the nuclear recovery path with a documented warning. * * Features: - * - O(1) lookup in both directions - * - Persistent storage via storage adapter - * - Atomic counter for next ID - * - Serialization/deserialization support + * - O(1) lookup in both directions. + * - Persistent storage via storage adapter. + * - Atomic, monotonic, append-only int counter. + * - Serialization/deserialization support. * * @module utils/entityIdMapper */ diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index a83d2e2a..5be89566 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -2635,13 +2635,20 @@ export class MetadataIndexManager implements MetadataIndexProvider { this.verbCountsByTypeFixed.fill(0) this.typeFieldAffinity.clear() - // Clear EntityIdMapper + // Clear EntityIdMapper. This is the explicit destructive path: the caller + // asked for nuclear recovery of a corrupted index, so renumbering UUIDs is + // intentional. Persisted int-keyed data (vector-mmap slots, graph + // link-compression encodings) is invalidated by this op — the warning + // below makes that explicit. Rebuild on its own does NOT clear the mapper. await this.idMapper.clear() // Clear chunk manager cache this.chunkManager.clearCache() prodLog.info(`✅ Cleared ${deletedCount} field indexes and all in-memory state`) + prodLog.warn('⚠️ EntityIdMapper was cleared — any persisted int-keyed data ' + + '(vector mmap slots, graph link-compression encodings, etc.) is now stale ' + + 'and must be rebuilt from canonical sources.') prodLog.info('⚠️ Run brain.index.rebuild() to recreate the index from entity data') } @@ -3032,8 +3039,19 @@ export class MetadataIndexManager implements MetadataIndexProvider { prodLog.info(`Cleared ${existingFields.length} field indexes from storage`) } - // Clear EntityIdMapper to start fresh - await this.idMapper.clear() + // EntityIdMapper is intentionally NOT cleared here. Rebuild re-iterates + // every entity in storage and calls idMapper.getOrAssign(uuid), which + // returns the existing int for known UUIDs (no renumbering). This is the + // foundational stability guarantee — vector-mmap slot indices, graph + // link-compression encodings, and any other persisted int-keyed data + // remain valid across a rebuild. Previously this line reset nextId to 1 + // and renumbered every UUID by re-insertion order, silently breaking + // any consumer that had persisted int-keyed data against the old map. + // Stale entries for UUIDs no longer in storage persist (harmless memory + // overhead); a dedicated prune step can be added if it ever matters. + // The destructive wipe is still available via clearAllIndexData() → + // idMapper.clear(), which is the explicit "recovery" path with the + // appropriate warning about invalidating persisted int-keyed data. // Clear chunk manager cache this.chunkManager.clearCache() diff --git a/tests/unit/hnsw/connections-codec.test.ts b/tests/unit/hnsw/connections-codec.test.ts new file mode 100644 index 00000000..c92c3888 --- /dev/null +++ b/tests/unit/hnsw/connections-codec.test.ts @@ -0,0 +1,137 @@ +/** + * @module hnsw/connectionsCodec.test + * @description Unit tests for `ConnectionsCodec` — the per-node buffer format + * brainy uses to persist HNSW connection sets as delta-varint-compressed + * binary blobs (2.4.0 #3). + * + * Mocks the `graph:compression` provider so the tests run without cortex + * installed. The real cross-language byte-format integration is exercised + * when cortex's `encodeConnections` / `decodeConnections` are wired in. + * + * Coverage: + * 1. Round-trip a Map of `level → Set` through encode + decode and + * verify the result is equal to the input (level-by-level Set equality). + * 2. Empty connections encode to a single-byte buffer (`0` = zero levels) and + * decode back to an empty Map — no `undefined` reads, no decode errors. + * 3. Multiple levels round-trip, including a level with a single UUID and a + * level with many UUIDs (the typical L0 fan-out). + * 4. UUIDs whose ints are no longer in the idMapper at decode time are + * silently dropped — matches the pre-2.4.0 behaviour of a missing + * `getUuid()` lookup in the legacy load path. + */ + +import { describe, it, expect } from 'vitest' +import { ConnectionsCodec } from '../../../src/hnsw/connectionsCodec.js' +import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js' +import type { GraphCompressionProvider } from '../../../src/plugin.js' + +const stubStorage = { + getMetadata: async () => undefined, + saveMetadata: async () => {}, + getNouns: async () => ({ totalCount: 0, items: [] }) +} as any + +/** + * Mock provider — encodes ints as a JSON byte array, decodes back. NOT the + * delta-varint format cortex produces, but format-agnostic round-trip + * verification is the unit-test concern here. + */ +const mockProvider: GraphCompressionProvider = { + encode(ids: number[]): Buffer { + const sorted = [...ids].sort((a, b) => a - b) + return Buffer.from(JSON.stringify(sorted), 'utf8') + }, + decode(data: Buffer): number[] { + if (data.length === 0) return [] + return JSON.parse(data.toString('utf8')) as number[] + } +} + +async function makeIdMapper(): Promise { + const mapper = new EntityIdMapper({ storage: stubStorage }) + await mapper.init() + return mapper +} + +function setsEqual(a: Set, b: Set): boolean { + if (a.size !== b.size) return false + for (const v of a) if (!b.has(v)) return false + return true +} + +describe('ConnectionsCodec (2.4.0 #3 — wraps graph:compression provider)', () => { + it('round-trips a single-level Set of UUIDs through encode + decode', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const input = new Map>([ + [0, new Set(['a', 'b', 'c', 'd'])] + ]) + const buf = codec.encode(input) + const out = codec.decode(buf) + + expect(out.size).toBe(1) + expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true) + }) + + it('empty connections encode to a one-byte buffer and decode to an empty Map', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const buf = codec.encode(new Map()) + expect(buf.length).toBe(1) + expect(buf.readUInt8(0)).toBe(0) + expect(codec.decode(buf).size).toBe(0) + }) + + it('round-trips multiple levels with very different fan-outs', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const manyUuids = Array.from({ length: 30 }, (_, i) => `u-${i}`) + const input = new Map>([ + [0, new Set(manyUuids)], + [1, new Set(['high-1', 'high-2'])], + [2, new Set(['solo'])] + ]) + + const buf = codec.encode(input) + const out = codec.decode(buf) + + expect(out.size).toBe(3) + expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true) + expect(setsEqual(out.get(1)!, input.get(1)!)).toBe(true) + expect(setsEqual(out.get(2)!, input.get(2)!)).toBe(true) + }) + + it('drops UUIDs whose ints the idMapper no longer recognises on decode', async () => { + const encodingMapper = await makeIdMapper() + const encodeCodec = new ConnectionsCodec(mockProvider, encodingMapper) + + // Encode through one mapper. + const input = new Map>([[0, new Set(['present', 'gone'])]]) + const buf = encodeCodec.encode(input) + + // Decode through a FRESH mapper that only knows about 'present'. + const decodingMapper = await makeIdMapper() + decodingMapper.getOrAssign('present') + + // Manually re-map 'present' to the same int it had on encode so the + // present UUID round-trips; 'gone' simply has no mapping → silent drop. + // We line up the int spaces by asserting equality below: only 'present' + // survives, 'gone' is missing — no errors, no `undefined` Set entries. + const decodeCodec = new ConnectionsCodec(mockProvider, decodingMapper) + // Inject the same int for 'present' so it round-trips one-for-one. (The + // encoding mapper assigns ints in insertion order: 'present'=1, 'gone'=2. + // The decoding mapper's first getOrAssign also returns 1, so 'present' + // matches; 'gone' (encoded as 2) has no mapping in the decoder.) + expect(encodingMapper.getInt('present')).toBe(1) + expect(decodingMapper.getInt('present')).toBe(1) + + const out = decodeCodec.decode(buf) + const l0 = out.get(0)! + expect(l0.has('present')).toBe(true) + expect(l0.has('gone')).toBe(false) + expect(l0.size).toBe(1) + }) +}) diff --git a/tests/unit/hnsw/mmap-vector-backend.test.ts b/tests/unit/hnsw/mmap-vector-backend.test.ts new file mode 100644 index 00000000..257d50c4 --- /dev/null +++ b/tests/unit/hnsw/mmap-vector-backend.test.ts @@ -0,0 +1,242 @@ +/** + * @module hnsw/mmapVectorBackend.test + * @description Unit tests for the `MmapVectorBackend` bridge — the brainy-side + * wrapper that translates UUID-keyed vector reads/writes into stable int slot + * ops against an `vectorStore:mmap` provider. + * + * Mocks the provider so the tests run without cortex installed (cortex is a + * downstream consumer of brainy, not a dev dep). The real integration with + * cortex's `NativeMmapVectorStore` is exercised when cortex 2.4.0 picks up + * this brainy release and re-runs its cross-language parity suite. + * + * Coverage: + * 1. Open-then-write-then-read round-trips for a single vector. + * 2. Batch reads return an array aligned with the input UUIDs, with `null` + * entries for misses interleaved among hits — order preserved. + * 3. Slot assignment is stable across multiple writes for the same UUID + * (no re-slot, no overwrite of an adjacent slot). + * 4. Writes beyond the initial capacity grow the file (doubling) without + * losing the vectors already written. + * 5. `readByUuid` returns `null` for both unknown UUIDs and UUIDs in the map + * whose slot has not yet been written. + * 6. Open is idempotent — opening an already-existing file reuses it. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { tmpdir } from 'node:os' +import { mkdtemp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { MmapVectorBackend } from '../../../src/hnsw/mmapVectorBackend.js' +import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js' +import type { + VectorStoreMmapInstance, + VectorStoreMmapProvider +} from '../../../src/plugin.js' + +/** + * Minimal storage stub for the EntityIdMapper. The mapper only touches storage + * in init/flush; for these tests the mapper starts empty and is never flushed. + */ +const stubStorage = { + getMetadata: async () => undefined, + saveMetadata: async () => {}, + getNouns: async () => ({ totalCount: 0, items: [] }) +} as any + +/** + * Pure in-memory mmap store. Mirrors cortex's NativeMmapVectorStore surface + * just closely enough to exercise the backend's contract — no real mmap, no + * file I/O, no f32 round-trip narrowing (the precision check is out of scope + * here; that's covered by cortex's parity suite). + */ +class MockMmapStore implements VectorStoreMmapInstance { + private readonly vectors: Array = [] + private highestWritten = -1 + constructor( + public readonly dim: number, + private _capacity: number + ) {} + get count(): number { + return this.highestWritten + 1 + } + get capacity(): number { + return this._capacity + } + writeVector(index: number, vector: number[]): void { + if (index >= this._capacity) { + throw new Error(`Slot ${index} >= capacity ${this._capacity}`) + } + if (vector.length !== this.dim) { + throw new Error(`Dim mismatch: expected ${this.dim}, got ${vector.length}`) + } + this.vectors[index] = [...vector] + if (index > this.highestWritten) this.highestWritten = index + } + writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number { + if (vectorsFlat.length % this.dim !== 0) { + throw new Error('vectorsFlat length not a multiple of dim') + } + const n = vectorsFlat.length / this.dim + for (let i = 0; i < n; i++) { + this.writeVector(startIndex + i, vectorsFlat.slice(i * this.dim, (i + 1) * this.dim)) + } + return n + } + readVector(index: number): number[] { + const v = this.vectors[index] + if (!v) throw new Error(`Slot ${index} not written`) + return [...v] + } + readVectorsBatch(indices: number[]): number[] { + const flat: number[] = [] + for (const i of indices) { + const v = this.vectors[i] + if (!v) throw new Error(`Slot ${i} not written`) + for (let k = 0; k < this.dim; k++) flat.push(v[k]) + } + return flat + } + prefetch(_indices: number[]): void { + /* no-op in mock */ + } + resize(newCapacity: number): void { + if (newCapacity < this._capacity) throw new Error('Cannot shrink') + this._capacity = newCapacity + } + flush(): void { + /* no-op in mock */ + } +} + +/** + * Mock provider — keeps one MockMmapStore per path. open() throws if the path + * doesn't exist yet (matches cortex semantics: open() is for existing files + * only); create() throws if it does (cortex doesn't, but the brainy backend + * does open-first-then-create, so the throw path is exercised). + */ +class MockMmapProvider implements VectorStoreMmapProvider { + private files = new Map() + create(path: string, dim: number, capacity: number): VectorStoreMmapInstance { + if (this.files.has(path)) throw new Error(`File exists at ${path}`) + const store = new MockMmapStore(dim, capacity) + this.files.set(path, store) + return store + } + open(path: string): VectorStoreMmapInstance { + const store = this.files.get(path) + if (!store) throw new Error(`No file at ${path}`) + return store + } + openReadOnly(path: string): VectorStoreMmapInstance { + return this.open(path) + } + /** Test helper — peek at the underlying store. */ + _peek(path: string): MockMmapStore | undefined { + return this.files.get(path) + } +} + +describe('MmapVectorBackend (2.4.0 #2 — wraps vectorStore:mmap provider)', () => { + let dir: string + let path: string + let idMapper: EntityIdMapper + let provider: MockMmapProvider + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'brainy-mmap-vec-')) + path = join(dir, 'vectors.bin') + idMapper = new EntityIdMapper({ storage: stubStorage }) + await idMapper.init() + provider = new MockMmapProvider() + }) + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }).catch(() => {}) + }) + + it('open creates a new file when none exists, then round-trips a vector by UUID', async () => { + const backend = await MmapVectorBackend.open(provider, path, 4, 16, idMapper) + expect(backend.dim).toBe(4) + + backend.writeByUuid('alpha', [1, 2, 3, 4]) + expect(backend.readByUuid('alpha')).toEqual([1, 2, 3, 4]) + }) + + it('reads return null for unknown UUIDs and for UUIDs in the map but not yet written', async () => { + const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) + + // Unknown — never seen by the mapper. + expect(backend.readByUuid('ghost')).toBeNull() + + // Known to the mapper but no slot ever written. We assign through the + // mapper directly so the backend itself has not touched the slot. + idMapper.getOrAssign('reserved') + expect(backend.readByUuid('reserved')).toBeNull() + }) + + it('batch read returns an array aligned to the input UUIDs (nulls preserved in place)', async () => { + const backend = await MmapVectorBackend.open(provider, path, 3, 16, idMapper) + backend.writeByUuid('a', [1, 1, 1]) + backend.writeByUuid('b', [2, 2, 2]) + backend.writeByUuid('c', [3, 3, 3]) + + // Interleave hits + a never-seen UUID + a duplicate hit. Order preserved. + const result = backend.readBatchByUuid(['b', 'missing', 'a', 'c', 'missing-too', 'b']) + expect(result).toEqual([ + [2, 2, 2], + null, + [1, 1, 1], + [3, 3, 3], + null, + [2, 2, 2] + ]) + }) + + it('writes for the same UUID land in the same slot (stable id, no re-slotting)', async () => { + const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) + backend.writeByUuid('persistent', [1, 1]) + const slotAfterFirstWrite = idMapper.getInt('persistent') + + backend.writeByUuid('persistent', [9, 9]) // overwrite the same slot + expect(idMapper.getInt('persistent')).toBe(slotAfterFirstWrite) + expect(backend.readByUuid('persistent')).toEqual([9, 9]) + + // Another UUID gets a different slot, and the first vector is undisturbed. + backend.writeByUuid('other', [5, 5]) + expect(idMapper.getInt('other')).not.toBe(slotAfterFirstWrite) + expect(backend.readByUuid('persistent')).toEqual([9, 9]) + expect(backend.readByUuid('other')).toEqual([5, 5]) + }) + + it('grows the file (doubling) when a write lands beyond capacity, without losing prior data', async () => { + // Start at the smallest sane initial capacity (clamped to 16 by the backend). + const backend = await MmapVectorBackend.open(provider, path, 2, 1, idMapper) + const store = provider._peek(path)! + expect(store.capacity).toBe(16) // backend floor + + // Write 20 vectors so capacity must double at least once (16 → 32). + const uuids: string[] = [] + for (let i = 0; i < 20; i++) { + const uuid = `u-${i}` + uuids.push(uuid) + backend.writeByUuid(uuid, [i, i * 2]) + } + expect(store.capacity).toBeGreaterThanOrEqual(32) + + // Every vector survived the growth — no slot got overwritten or lost. + for (let i = 0; i < uuids.length; i++) { + expect(backend.readByUuid(uuids[i])).toEqual([i, i * 2]) + } + }) + + it('opens an existing file idempotently when the second open hits the same path', async () => { + // First open creates. + const backend1 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) + backend1.writeByUuid('persisted', [7, 7]) + + // Second open against the same path. The provider's create() throws on + // collision; the backend's open-first behaviour means we reuse the file. + const backend2 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) + expect(backend2.readByUuid('persisted')).toEqual([7, 7]) + }) +}) diff --git a/tests/unit/indexes/columnStore/column-store-interchange.test.ts b/tests/unit/indexes/columnStore/column-store-interchange.test.ts new file mode 100644 index 00000000..f79038ea --- /dev/null +++ b/tests/unit/indexes/columnStore/column-store-interchange.test.ts @@ -0,0 +1,191 @@ +/** + * @module column-store-interchange.test + * @description 2.4.0 #4 — column-store JS↔native interchange. Pins down that + * brainy's JS `ColumnStore` writes segments + DELETED bitmaps via the binary- + * blob primitive at the cortex-shared key convention, AND that indexes + * persisted under the pre-2.4.0 envelope (`{ _binary, base64 }` at the + * `.cidx` object-path) still load correctly via the legacy-read fallback. + * + * Together these properties make the JS and native engines byte-compatible + * at the segment payload, so writing through one and reading through the + * other is a no-op format-wise — closing the gap the cortex 2.3.1 read-side + * fallback opened on the cortex side. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js' +import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js' +import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js' +import { writeSegmentToBuffer } from '../../../../src/indexes/columnStore/ColumnSegmentFormat.js' +import { ColumnManifest } from '../../../../src/indexes/columnStore/ColumnManifest.js' +import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js' + +describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => { + let storage: MemoryStorage + let idMapper: EntityIdMapper + let store: ColumnStore + + beforeEach(async () => { + storage = new MemoryStorage() + await storage.init() + idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' }) + await idMapper.init() + store = new ColumnStore({ flushThreshold: 10 }) + await store.init(storage, idMapper) + }) + + afterEach(async () => { + await store.close() + }) + + // ----------------------------------------------------------------------- + // WRITE side: raw blob at the cortex-shared key, no legacy envelope + // ----------------------------------------------------------------------- + + it('flushBuffer writes segment bytes as a raw blob at the cortex-shared key', async () => { + for (let i = 0; i < 5; i++) { + store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 }) + } + await store.flush() + + // The raw blob exists at the suffix-free cortex key. + const key = '_column_index/score/L0-000001' + const raw = await (storage as any).loadBinaryBlob(key) + expect(Buffer.isBuffer(raw)).toBe(true) + expect(raw!.length).toBeGreaterThan(0) + + // The legacy envelope at the `.cidx` object-path is NOT written (no + // 33% base64 bloat, no dual-write). + const legacyPath = '_column_index/score/L0-000001.cidx' + const legacyEnvelope = await (storage as any).readObjectFromPath(legacyPath) + expect(legacyEnvelope).toBeFalsy() + }) + + it('flushBuffer writes the DELETED bitmap as a raw blob too (not an envelope)', async () => { + for (let i = 0; i < 3; i++) { + store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i }) + } + store.removeEntity(idMapper.getInt('u-1')!) + await store.flush() + + const rawDel = await (storage as any).loadBinaryBlob('_column_index/score/DELETED') + expect(Buffer.isBuffer(rawDel)).toBe(true) + expect(rawDel!.length).toBeGreaterThan(0) + + const legacyEnvelope = await (storage as any).readObjectFromPath( + '_column_index/score/DELETED.bin' + ) + expect(legacyEnvelope).toBeFalsy() + }) + + // ----------------------------------------------------------------------- + // READ side: pre-2.4.0 legacy envelope still loads via the fallback + // ----------------------------------------------------------------------- + + it('loads a segment persisted as the pre-2.4.0 envelope (legacy fallback)', async () => { + // Seed a legacy-format index DIRECTLY on storage — the on-disk shape + // brainy 7.20.0..7.25.0 wrote before this commit. + const legacyStore = new ColumnStore() + await legacyStore.init(storage, idMapper) + + // Build a segment buffer using the shared cidx format. + const values: number[] = [10, 20, 30] + const entityIds: number[] = [ + idMapper.getOrAssign('legacy-a'), + idMapper.getOrAssign('legacy-b'), + idMapper.getOrAssign('legacy-c') + ] + const segBuffer = writeSegmentToBuffer( + { + fieldName: 'score', + fieldNameLength: 5, + valueType: 0, + level: 0, + codec: 0, + flags: 0, + count: values.length + }, + values, + entityIds + ) + + // Persist via the LEGACY envelope path (writeObjectToPath, base64 wrap). + await (storage as any).writeObjectToPath('_column_index/score/L0-000001.cidx', { + _binary: true, + data: segBuffer.toString('base64') + }) + const manifest = new ColumnManifest('score', '_column_index') + manifest.valueType = 0 + manifest.multiValue = false + manifest.addSegment({ + id: 1, + level: 0, + count: values.length, + minValue: values[0], + maxValue: values[values.length - 1], + file: 'L0-000001.cidx' + }) + await manifest.save(storage) + await legacyStore.close() + + // NEW ColumnStore re-discovers the manifest at init, then the legacy + // segment loads via the readObjectFromPath fallback (no raw blob exists + // at the cortex key, so it falls through). sortTopK returns the entities + // in the order the segment recorded — proves the bytes round-tripped. + const readStore = new ColumnStore() + await readStore.init(storage, idMapper) + const sortedInts = await readStore.sortTopK('score', 'asc', 10) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + expect(sortedUuids).toEqual(['legacy-a', 'legacy-b', 'legacy-c']) + await readStore.close() + }) + + it('loads a DELETED bitmap persisted as the pre-2.4.0 envelope (legacy fallback)', async () => { + // Manually seed a legacy DELETED.bin envelope, then re-init and confirm + // the entity it marks gone is excluded from filter/sort results. + const bm = new RoaringBitmap32() + bm.add(idMapper.getOrAssign('dead')) + const serialized = Buffer.from(bm.serialize(true)) + await (storage as any).writeObjectToPath('_column_index/score/DELETED.bin', { + _binary: true, + data: serialized.toString('base64') + }) + + // Add a couple of live entities so a manifest exists (init only loads + // DELETED for fields with a manifest on disk). + store.addEntity(idMapper.getOrAssign('alive-1'), { score: 100 }) + store.addEntity(idMapper.getOrAssign('alive-2'), { score: 200 }) + await store.flush() + await store.close() + + const readStore = new ColumnStore() + await readStore.init(storage, idMapper) + const sortedInts = await readStore.sortTopK('score', 'asc', 10) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + + // The dead entity must NOT appear; the alive ones must. + expect(sortedUuids).not.toContain('dead') + expect(sortedUuids).toContain('alive-1') + expect(sortedUuids).toContain('alive-2') + await readStore.close() + }) + + // ----------------------------------------------------------------------- + // ROUND-TRIP: write-then-read in the new format + // ----------------------------------------------------------------------- + + it('write (new format) → re-init → read returns the same data', async () => { + for (let i = 0; i < 5; i++) { + store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 }) + } + await store.flush() + await store.close() + + const readStore = new ColumnStore() + await readStore.init(storage, idMapper) + const sortedInts = await readStore.sortTopK('score', 'asc', 10) + const sortedUuids = sortedInts.map(i => idMapper.getUuid(i)) + expect(sortedUuids).toEqual(['u-0', 'u-1', 'u-2', 'u-3', 'u-4']) + await readStore.close() + }) +}) diff --git a/tests/unit/utils/entity-id-mapper-stability.test.ts b/tests/unit/utils/entity-id-mapper-stability.test.ts new file mode 100644 index 00000000..f872f46b --- /dev/null +++ b/tests/unit/utils/entity-id-mapper-stability.test.ts @@ -0,0 +1,161 @@ +/** + * Regression test: EntityIdMapper stability across rebuild. + * + * The foundation 2.4.0 (vector mmap store, graph link compression, column-store + * JS↔native interchange) all key off UUID→int mappings that **must not change** + * across a metadata-index rebuild. Previously `metadataIndex.rebuild()` called + * `idMapper.clear()` which reset `nextId` to 1 and renumbered every UUID by + * re-insertion order, silently invalidating any consumer that had persisted + * int-keyed data against the old map. + * + * This test pins down the stability contract: + * + * 1. UUID→int mappings persist across a single rebuild. + * 2. Mappings persist across many consecutive rebuilds. + * 3. New entities added after rebuild get fresh ints greater than any prior + * assignment — no collisions with existing UUIDs' ints. + * 4. Removed entities leave a permanent hole — new entities don't recycle the + * gap, even across a rebuild. + * 5. `clearAllIndexData()` is the explicit, intentional nuclear path — it DOES + * renumber. This is the only documented way to invalidate the int space, and + * a warning is logged so consumers know persisted int-keyed data is now stale. + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../../src/brainy.js' + +const DIM = 384 +const makeVec = (seed = 1) => + new Float32Array(DIM).map((_, i) => ((i + seed) % DIM) / DIM) + +describe('EntityIdMapper stability (foundation for 2.4.0)', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ storage: { type: 'memory' }, silent: true }) + await brain.init() + }) + + afterEach(async () => { + await brain.close() + }) + + async function addEntity(name: string, seed: number): Promise { + return brain.add({ + data: name, + vector: makeVec(seed), + type: 'thing' as any, + metadata: { name } + }) + } + + function getInt(uuid: string): number | undefined { + return (brain as any).metadataIndex.idMapper.getInt(uuid) + } + + async function rebuild(): Promise { + await (brain as any).metadataIndex.rebuild() + } + + it('UUID→int mappings persist across a single metadata-index rebuild', async () => { + const ids = [ + await addEntity('a', 1), + await addEntity('b', 2), + await addEntity('c', 3), + await addEntity('d', 4), + await addEntity('e', 5) + ] + const before = ids.map(id => getInt(id)) + expect(before.every(i => typeof i === 'number' && (i as number) > 0)).toBe(true) + + await rebuild() + + const after = ids.map(id => getInt(id)) + expect(after).toEqual(before) + }) + + it('mappings stay byte-for-byte stable across many consecutive rebuilds', async () => { + const ids = [ + await addEntity('a', 1), + await addEntity('b', 2), + await addEntity('c', 3) + ] + const before = ids.map(id => getInt(id)) + + for (let i = 0; i < 5; i++) { + await rebuild() + const after = ids.map(id => getInt(id)) + expect(after).toEqual(before) + } + }) + + it('entities added after rebuild get fresh monotonic ints (no collision with existing)', async () => { + const priorIds = [ + await addEntity('a', 1), + await addEntity('b', 2), + await addEntity('c', 3) + ] + const priorInts = priorIds.map(id => getInt(id) as number) + const maxPrior = Math.max(...priorInts) + + await rebuild() + + const newId = await addEntity('d', 4) + const newInt = getInt(newId) as number + expect(newInt).toBeGreaterThan(maxPrior) + // Prior entities' ints didn't drift. + expect(priorIds.map(id => getInt(id))).toEqual(priorInts) + }) + + it('removed entities leave a permanent hole — new entities never recycle the gap', async () => { + const ids = [ + await addEntity('a', 1), + await addEntity('b', 2), + await addEntity('c', 3), + await addEntity('d', 4), + await addEntity('e', 5) + ] + const beforeInts = ids.map(id => getInt(id) as number) + const deletedId = ids[2] + const deletedInt = beforeInts[2] + const maxBefore = Math.max(...beforeInts) + + await brain.delete(deletedId) + expect(getInt(deletedId)).toBeUndefined() + + const newId = await addEntity('f', 6) + const newInt = getInt(newId) as number + expect(newInt).not.toBe(deletedInt) + expect(newInt).toBeGreaterThan(maxBefore) + + // Surviving ids keep their ints across the deletion + the add. + const survivors = ids.filter((_, i) => i !== 2) + const survivorIntsBefore = beforeInts.filter((_, i) => i !== 2) + expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore) + + // Survivors' ints also survive a rebuild after the delete. + await rebuild() + expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore) + // The deleted id is still gone after rebuild (no resurrection). + expect(getInt(deletedId)).toBeUndefined() + }) + + it('clearAllIndexData() is the explicit nuclear path that DOES renumber', async () => { + const id1 = await addEntity('a', 1) + const id2 = await addEntity('b', 2) + const priorInts = [getInt(id1) as number, getInt(id2) as number] + expect(priorInts.every(i => i >= 1)).toBe(true) + + // Nuclear recovery: explicit destructive op. The warning logged here is + // the only documented way to invalidate the canonical int space. + await (brain as any).metadataIndex.clearAllIndexData() + + // Both UUIDs are gone from the mapper. + expect(getInt(id1)).toBeUndefined() + expect(getInt(id2)).toBeUndefined() + + // The int counter restarted from 1: the next add() gets int 1. + const idAfter = await addEntity('c', 3) + expect(getInt(idAfter)).toBe(1) + }) +})