src/brainy.ts

@@ -37,7 +37,14 @@ import { setGlobalCache } from './utils/unifiedCache.js'
 import type { UnifiedCache } from './utils/unifiedCache.js'
 import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js'
 import { PluginRegistry } from './plugin.js'
-import type { BrainyPlugin, BrainyPluginContext } from './plugin.js'
+import type {
+  BrainyPlugin,
+  BrainyPluginContext,
+  GraphCompressionProvider,
+  VectorStoreMmapProvider
+} from './plugin.js'
+import { MmapVectorBackend } from './hnsw/mmapVectorBackend.js'
+import { ConnectionsCodec } from './hnsw/connectionsCodec.js'
 import { TransactionManager } from './transaction/TransactionManager.js'
 import {
   ValidationConfig,
@@ -559,6 +566,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         this.graphIndex = graphIndex
       }
 
+      // Wire the mmap-vector backend (2.4.0 #2). When the vectorStore:mmap
+      // provider is registered AND the storage adapter resolves a real local
+      // path via getBinaryBlobPath(), open the mmap file there and inject the
+      // backend into HNSWIndex. Cloud adapters return null and skip silently;
+      // HNSWIndex's behaviour is then identical to pre-2.4.0 (per-entity reads
+      // from canonical storage). Failures are non-fatal — the index still
+      // works, just without the zero-copy fast path.
+      await this.wireMmapVectorBackend()
+
+      // Wire the connections codec (2.4.0 #3). When the graph:compression
+      // provider is registered AND the metadata index exposes a stable
+      // idMapper, inject a codec that encodes HNSW connections as
+      // delta-varint blobs at save time and decodes on load. The blob
+      // primitive itself works on every brainy 7.25.0 adapter, so unlike the
+      // mmap-vector backend this layer engages even on cloud adapters.
+      this.wireConnectionsCodec()
+
       // Rebuild indexes if needed for existing data
       await this.rebuildIndexesIfNeeded()
 
@@ -7716,6 +7740,101 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     await this.lazyRebuildPromise
   }
 
+  /**
+   * @description Open the mmap-vector backend and inject it into HNSWIndex.
+   * Called once during init after plugin activation, metadataIndex init, and
+   * graphIndex setup — so the idMapper is available and providers are wired.
+   *
+   * Activation conditions (ALL must hold; otherwise this is a no-op and the
+   * index falls back to per-entity storage reads, the pre-2.4.0 behaviour):
+   * 1. The `vectorStore:mmap` provider is registered (cortex registers this).
+   * 2. The storage adapter resolves a real local path via `getBinaryBlobPath()`
+   *    — cloud adapters return null and the mmap layer is skipped silently.
+   * 3. The metadata index exposes its idMapper (a stable UUID↔int map). The
+   *    mmap layout is keyed by these ints, so 2.4.0 #1 is the prerequisite.
+   *
+   * Vector dimensionality: read from `this.dimensions` if set; otherwise the
+   * default for the standard embedding model (384). The dim is fixed at
+   * file-creation time and cannot change later, so a wrong default would only
+   * matter on the very first init of a brand-new instance. After the first
+   * `add()` the in-memory `this.dimensions` is set, and any subsequent init
+   * (or a re-open) sees the correct value.
+   *
+   * Failures here are non-fatal: a debug-level log records the reason, and
+   * HNSWIndex continues without the mmap fast path.
+   */
+  private async wireMmapVectorBackend(): Promise<void> {
+    const provider = this.pluginRegistry.getProvider<VectorStoreMmapProvider>('vectorStore:mmap')
+    if (!provider) return
+
+    const storageWithBlob = this.storage as unknown as {
+      getBinaryBlobPath?: (key: string) => string | null
+    }
+    const vectorPath = storageWithBlob.getBinaryBlobPath?.('_vectors/main') ?? null
+    if (!vectorPath) return
+
+    const idMapper = this.metadataIndex.getIdMapper?.()
+    if (!idMapper) return
+
+    const dim = this.dimensions ?? 384
+    const initialCapacity = Math.max(idMapper.size * 2, 1024)
+
+    try {
+      const backend = await MmapVectorBackend.open(
+        provider,
+        vectorPath,
+        dim,
+        initialCapacity,
+        idMapper
+      )
+      this.index.setVectorBackend(backend)
+      if (!this.config.silent) {
+        console.log(
+          `[brainy] vector mmap backend wired (path=${vectorPath}, dim=${dim}, capacity=${initialCapacity})`
+        )
+      }
+    } catch (error) {
+      // Common cases: dim mismatch from a prior init at a different dimension,
+      // permission errors, disk full, file corruption. Index keeps working via
+      // the per-entity storage path; flag the cause for diagnosis.
+      if (!this.config.silent) {
+        console.warn(
+          `[brainy] mmap-vector backend not wired (${(error as Error).message}); ` +
+            `falling back to per-entity vector reads`
+        )
+      }
+    }
+  }
+
+  /**
+   * @description Wire the HNSW connections codec (2.4.0 #3). Activates when
+   * BOTH (a) the `graph:compression` provider is registered (cortex registers
+   * `{ encode: encodeConnections, decode: decodeConnections }`), and (b) the
+   * metadata index exposes a stable idMapper. Failures are non-fatal: HNSW
+   * keeps working via the legacy JSON-array path.
+   *
+   * Unlike the mmap-vector backend, this layer does NOT require a real local
+   * path — the binary-blob primitive saves the compressed bytes through the
+   * storage adapter directly, so the codec is engaged on cloud adapters too.
+   * Format convergence is lazy: pre-2.4.0 nodes load via the legacy path,
+   * then the next dirty save writes the compressed form (and the legacy
+   * field of saveHNSWData becomes empty), so all reads converge over time
+   * without an explicit migration step.
+   */
+  private wireConnectionsCodec(): void {
+    const provider = this.pluginRegistry.getProvider<GraphCompressionProvider>('graph:compression')
+    if (!provider) return
+
+    const idMapper = this.metadataIndex.getIdMapper?.()
+    if (!idMapper) return
+
+    const codec = new ConnectionsCodec(provider, idMapper)
+    this.index.setConnectionsCodec(codec)
+    if (!this.config.silent) {
+      console.log('[brainy] graph link compression wired (delta-varint connections via graph:compression provider)')
+    }
+  }
+
   /**
    * Rebuild indexes from persisted data if needed (LAZY LOADING)
    *

src/hnsw/connectionsCodec.ts

@@ -0,0 +1,137 @@
+/**
+ * @module hnsw/connectionsCodec
+ * @description Bridge between brainy's UUID-keyed HNSW connection sets and
+ * cortex's int-keyed delta-varint encode/decode (the `graph:compression`
+ * provider). Translates UUIDs to stable int slots via the post-2.4.0 #1
+ * `EntityIdMapper`, batches all of a node's per-level connection lists into
+ * a single compact buffer, and reverses the path on load.
+ *
+ * Wire format of the per-node buffer the codec produces:
+ *
+ *     [num_levels: u8]
+ *     repeated num_levels times:
+ *       [level: u8]
+ *       [encoded_len: u32 LE]
+ *       [encoded_bytes: <encoded_len> bytes]   <- provider.encode(ints)
+ *
+ * Why a single buffer per node and not one blob per level:
+ * - One `saveBinaryBlob` call per persisted node, regardless of level count.
+ *   On hot insert paths an HNSW node may be at levels 0..3; the per-node
+ *   blob trades a one-byte count for 3 saved I/O round-trips.
+ * - Read path is symmetric: one `loadBinaryBlob` reconstructs the full
+ *   connection set, so HNSW rebuild stays linear in the number of nodes.
+ *
+ * Loose UUIDs (ints assigned to entities that no longer exist) are silently
+ * dropped on decode. They can't legitimately participate in graph traversal
+ * because the entity is gone; treating them as `undefined` UUIDs matches the
+ * pre-2.4.0 behaviour of an `idMapper.getUuid()` miss in the legacy load path.
+ */
+
+import type {
+  EntityIdMapperProvider,
+  GraphCompressionProvider
+} from '../plugin.js'
+
+export class ConnectionsCodec {
+  constructor(
+    private readonly provider: GraphCompressionProvider,
+    private readonly idMapper: EntityIdMapperProvider
+  ) {}
+
+  /**
+   * @description Encode all levels of a node's connections into one compact
+   * buffer. Empty input → an empty buffer (zero levels, no body). The
+   * idMapper assigns stable ints on demand if a connection UUID has never
+   * been seen — append-only, matches the rest of the 2.4.0 contract.
+   */
+  encode(connectionsByLevel: Map<number, Set<string>>): Buffer {
+    if (connectionsByLevel.size === 0) {
+      // Single byte: zero levels. Round-trips cleanly through decode.
+      return Buffer.from([0])
+    }
+    if (connectionsByLevel.size > 0xff) {
+      throw new Error(
+        `ConnectionsCodec.encode: too many HNSW levels (${connectionsByLevel.size}); ` +
+          `the per-node header is one byte (max 255 levels).`
+      )
+    }
+
+    // First pass: encode each level into its own buffer.
+    const levelBuffers: Array<{ level: number; buf: Buffer }> = []
+    let bodyBytes = 0
+    for (const [level, uuids] of connectionsByLevel) {
+      if (level > 0xff) {
+        throw new Error(
+          `ConnectionsCodec.encode: level ${level} exceeds u8 header range (max 255).`
+        )
+      }
+      const ints: number[] = []
+      for (const uuid of uuids) {
+        ints.push(this.idMapper.getOrAssign(uuid))
+      }
+      const buf = this.provider.encode(ints)
+      levelBuffers.push({ level, buf })
+      bodyBytes += 1 + 4 + buf.length // level + len + payload
+    }
+
+    // Second pass: concatenate with header.
+    const out = Buffer.alloc(1 + bodyBytes)
+    out.writeUInt8(levelBuffers.length, 0)
+    let offset = 1
+    for (const { level, buf } of levelBuffers) {
+      out.writeUInt8(level, offset)
+      offset += 1
+      out.writeUInt32LE(buf.length, offset)
+      offset += 4
+      buf.copy(out, offset)
+      offset += buf.length
+    }
+    return out
+  }
+
+  /**
+   * @description Decode a per-node buffer back into a connections Map. Ints
+   * for entities the idMapper no longer knows are silently dropped — see the
+   * module-level note on why.
+   */
+  decode(data: Buffer): Map<number, Set<string>> {
+    const out = new Map<number, Set<string>>()
+    if (data.length === 0) return out
+
+    const count = data.readUInt8(0)
+    let offset = 1
+    for (let i = 0; i < count; i++) {
+      if (offset + 5 > data.length) {
+        throw new Error(`ConnectionsCodec.decode: truncated header at level ${i}`)
+      }
+      const level = data.readUInt8(offset)
+      offset += 1
+      const len = data.readUInt32LE(offset)
+      offset += 4
+      if (offset + len > data.length) {
+        throw new Error(`ConnectionsCodec.decode: truncated body at level ${level}`)
+      }
+      const levelBuf = data.slice(offset, offset + len)
+      offset += len
+
+      const ints = this.provider.decode(levelBuf)
+      const uuids = new Set<string>()
+      for (const intId of ints) {
+        const uuid = this.idMapper.getUuid(intId)
+        if (uuid !== undefined) uuids.add(uuid)
+      }
+      out.set(level, uuids)
+    }
+    return out
+  }
+}
+
+/**
+ * @description Build the binary-blob storage key for a node's compressed
+ * connections. Suffix-free; the adapter appends its own. Keys live under
+ * `_hnsw_conn/` so they don't collide with the existing `_column_index/`
+ * blobs and so a cortex-side reader knows exactly where to look.
+ */
+export function compressedConnectionsKey(nodeId: string): string {
+  return `_hnsw_conn/${nodeId}`
+}

src/hnsw/hnswIndex.ts

@@ -18,6 +18,8 @@ import { prodLog } from '../utils/logger.js'
 import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js'
 import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js'
 import type { HnswProvider } from '../plugin.js'
+import { MmapVectorBackend } from './mmapVectorBackend.js'
+import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js'
 
 // Default HNSW parameters
 const DEFAULT_CONFIG: HNSWConfig = {
@@ -49,6 +51,23 @@ export class HNSWIndex implements HnswProvider {
   private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes
   // Always-adaptive caching - no "mode" concept, system adapts automatically
 
+  // Optional mmap-vector backend (2.4.0 #2). When set, the read paths
+  // (preloadVectors, getVectorSafe) try the mmap layer before storage, and
+  // on a storage hit they write back into the mmap slot — a lazy migration
+  // that converges upgraded installs to the fast path under live traffic.
+  // Null on cloud storage adapters (no local path) and pre-injection init.
+  private vectorBackend: MmapVectorBackend | null = null
+
+  // Optional connections codec (2.4.0 #3). When set, node-persist writes the
+  // node's per-level connection sets as a single delta-varint-compressed
+  // binary blob (typically ~4× smaller than the legacy JSON-UUID-array
+  // shape), plus an empty `connections: {}` in saveHNSWData as the marker.
+  // The read path tries to load the blob first; missing blob → legacy
+  // connections field. Format convergence is lazy: pre-2.4.0 nodes still
+  // load via the legacy path, then write the compressed form on next dirty
+  // save, so the migration converges under live traffic with no big-bang.
+  private connectionsCodec: ConnectionsCodec | null = null
+
   // COW (Copy-on-Write) support
   private cowEnabled: boolean = false
   private cowModifiedNodes: Set<string> = new Set()
@@ -101,6 +120,35 @@ export class HNSWIndex implements HnswProvider {
     this.useParallelization = useParallelization
   }
 
+  /**
+   * @description Inject (or detach) the mmap-vector backend. When set, the
+   * vector read paths (`preloadVectors`, `getVectorSafe`) try the mmap layer
+   * before storage and lazily write back on storage hits — converging an
+   * upgraded install to the zero-copy fast path without a migration step.
+   * Setting to null reverts to the existing per-entity read path.
+   *
+   * Lifecycle: brainy.ts wires this after plugin activation when (a) the
+   * `vectorStore:mmap` provider is registered AND (b) the storage adapter
+   * resolves a real local path via `getBinaryBlobPath()`. Cloud adapters
+   * leave it null; HNSWIndex's behaviour is then identical to pre-2.4.0.
+   */
+  public setVectorBackend(backend: MmapVectorBackend | null): void {
+    this.vectorBackend = backend
+  }
+
+  /**
+   * @description Inject (or detach) the connections codec. When set, node
+   * persistence writes a delta-varint-compressed binary blob alongside an
+   * empty `connections: {}` marker in saveHNSWData; the load path reads the
+   * blob and decodes. Null reverts to the legacy JSON-array path. Wiring is
+   * done by brainy.ts when the `graph:compression` provider is registered
+   * AND the storage adapter exposes the binary-blob primitive AND the
+   * metadata index has a stable idMapper.
+   */
+  public setConnectionsCodec(codec: ConnectionsCodec | null): void {
+    this.connectionsCodec = codec
+  }
+
   /**
    * Get whether parallelization is enabled
    */
@@ -144,15 +192,7 @@ export class HNSWIndex implements HnswProvider {
           const noun = this.nouns.get(nodeId)
           if (!noun) return Promise.resolve() // Node was deleted
 
-          const connectionsObj: Record<string, string[]> = {}
+          return this.persistNodeConnections(nodeId, noun).catch(error => {
-          for (const [level, nounIds] of noun.connections.entries()) {
-            connectionsObj[level.toString()] = Array.from(nounIds)
-          }
-
-          return this.storage!.saveHNSWData(nodeId, {
-            level: noun.level,
-            connections: connectionsObj
-          }).catch(error => {
             console.error(`[HNSW flush] Failed to persist node ${nodeId}:`, error)
           })
         })
@@ -183,6 +223,94 @@ export class HNSWIndex implements HnswProvider {
     return nodeCount
   }
 
+  /**
+   * @description Persist one node's connections. When the connections codec is
+   * wired AND the storage adapter exposes `saveBinaryBlob`, the per-level
+   * connection sets are encoded into a single compact buffer and stored as a
+   * binary blob; `saveHNSWData` then records the node's level and an empty
+   * `connections: {}` as the marker. Otherwise the legacy JSON-array path is
+   * taken — the format every brainy release before 2.4.0 wrote. The two are
+   * intentionally NOT dual-written: one or the other, never both, so format
+   * convergence is unambiguous on the read side.
+   *
+   * Shared by all three save sites (deferred flush, immediate-mode new-entity
+   * persist, immediate-mode neighbor update) so the codec branch is exercised
+   * uniformly regardless of which path triggered the write.
+   */
+  private async persistNodeConnections(nodeId: string, noun: HNSWNoun): Promise<void> {
+    if (!this.storage) return
+
+    const storageWithBlob = this.storage as unknown as {
+      saveBinaryBlob?: (key: string, data: Buffer) => Promise<void>
+    }
+    const canCompress =
+      this.connectionsCodec !== null &&
+      typeof storageWithBlob.saveBinaryBlob === 'function'
+
+    if (canCompress) {
+      const encoded = this.connectionsCodec!.encode(noun.connections)
+      await storageWithBlob.saveBinaryBlob!(compressedConnectionsKey(nodeId), encoded)
+      await this.storage.saveHNSWData(nodeId, {
+        level: noun.level,
+        connections: {}
+      })
+      return
+    }
+
+    const connectionsObj: Record<string, string[]> = {}
+    for (const [level, nounIds] of noun.connections.entries()) {
+      connectionsObj[level.toString()] = Array.from(nounIds)
+    }
+    await this.storage.saveHNSWData(nodeId, {
+      level: noun.level,
+      connections: connectionsObj
+    })
+  }
+
+  /**
+   * @description Restore one node's connections from persisted storage. Tries
+   * the compressed-blob path first (when codec wired AND blob primitive
+   * available); a present non-empty buffer is decoded and populates
+   * `noun.connections`. A missing blob (no payload at the key) means this node
+   * was last persisted under the legacy format, so the legacy
+   * `hnswData.connections` field is used instead.
+   *
+   * On any decode error the legacy field is also used as a safety net — the
+   * codec's `decode` throws on truncated input, which would otherwise leave
+   * the node with an empty connections Map (orphaned from the graph).
+   */
+  private async restoreNodeConnections(
+    nodeId: string,
+    hnswData: { level: number; connections: Record<string, string[]> },
+    noun: HNSWNoun
+  ): Promise<void> {
+    const storageWithBlob = this.storage as unknown as {
+      loadBinaryBlob?: (key: string) => Promise<Buffer | null>
+    }
+    const canDecompress =
+      this.connectionsCodec !== null &&
+      typeof storageWithBlob.loadBinaryBlob === 'function'
+
+    if (canDecompress) {
+      try {
+        const buf = await storageWithBlob.loadBinaryBlob!(compressedConnectionsKey(nodeId))
+        if (buf && buf.length > 0) {
+          const decoded = this.connectionsCodec!.decode(buf)
+          noun.connections = decoded
+          return
+        }
+      } catch (error) {
+        prodLog.debug(`HNSW: compressed connections decode failed for ${nodeId}; falling back to legacy`, error)
+      }
+    }
+
+    // Legacy fallback: JSON UUID arrays in the HNSW data object.
+    for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
+      const level = parseInt(levelStr, 10)
+      noun.connections.set(level, new Set<string>(nounIds as string[]))
+    }
+  }
+
   /**
    * Get the number of dirty (unpersisted) nodes
    * Useful for monitoring and debugging
@@ -521,18 +649,12 @@ export class HNSWIndex implements HnswProvider {
         // In deferred mode, we track dirty nodes instead of persisting immediately
         // This reduces GCS operations from 70 to 2-3 per add() (30-50× faster)
         if (this.storage && this.persistMode === 'immediate') {
-          // IMMEDIATE MODE: Original behavior - persist each neighbor update
+          // IMMEDIATE MODE: Original behavior - persist each neighbor update.
-          const neighborConnectionsObj: Record<string, string[]> = {}
+          // Goes through the per-node helper so the compressed-blob branch
-          for (const [lvl, nounIds] of neighbor.connections.entries()) {
+          // fires identically here vs. the deferred-flush path.
-            neighborConnectionsObj[lvl.toString()] = Array.from(nounIds)
-          }
-
           neighborUpdates.push({
             neighborId,
-            promise: this.storage.saveHNSWData(neighborId, {
+            promise: this.persistNodeConnections(neighborId, neighbor)
-              level: neighbor.level,
-              connections: neighborConnectionsObj
-            })
           })
         } else if (this.persistMode === 'deferred') {
           // DEFERRED MODE: Track dirty nodes for later batch persistence
@@ -619,16 +741,10 @@ export class HNSWIndex implements HnswProvider {
     // Persist HNSW graph data to storage
     // Respect persistMode setting
     if (this.storage && this.persistMode === 'immediate') {
-      // IMMEDIATE MODE: Original behavior - persist new entity and system data
+      // IMMEDIATE MODE: Original behavior - persist new entity and system data.
-      const connectionsObj: Record<string, string[]> = {}
+      // Goes through the per-node helper so the compressed-blob branch fires
-      for (const [level, nounIds] of noun.connections.entries()) {
+      // identically here vs. the deferred-flush + neighbor-update paths.
-        connectionsObj[level.toString()] = Array.from(nounIds)
+      await this.persistNodeConnections(id, noun).catch((error) => {
-      }
-
-      await this.storage.saveHNSWData(id, {
-        level: nounLevel,
-        connections: connectionsObj
-      }).catch((error) => {
         console.error(`Failed to persist HNSW data for ${id}:`, error)
       })
 
@@ -1073,7 +1189,23 @@ export class HNSWIndex implements HnswProvider {
     const cacheKey = `hnsw:vector:${noun.id}`
 
     const vector = await this.unifiedCache.get(cacheKey, async () => {
-      // Cache miss - load from storage
+      // Try the mmap-vector backend first when wired — zero-copy slot read,
+      // no per-entity object hydration.
+      if (this.vectorBackend) {
+        const fromMmap = this.vectorBackend.readByUuid(noun.id)
+        if (fromMmap) {
+          this.unifiedCache.set(
+            cacheKey,
+            fromMmap,
+            'hnsw',
+            fromMmap.length * 4,
+            50
+          )
+          return fromMmap
+        }
+      }
+
+      // Storage fallback — the canonical source of truth.
       if (!this.storage) {
         throw new Error('Storage not available for vector loading')
       }
@@ -1083,6 +1215,18 @@ export class HNSWIndex implements HnswProvider {
         throw new Error(`Vector not found for noun ${noun.id}`)
       }
 
+      // Lazy migration: write back into the mmap slot so the next read is
+      // mmap-fast. Idempotent + resumable; a write failure is non-fatal —
+      // the field still serves this read, and the next storage hit re-tries
+      // the migration. (See mmapVectorBackend.ts for the contract.)
+      if (this.vectorBackend) {
+        try {
+          this.vectorBackend.writeByUuid(noun.id, loaded)
+        } catch (error) {
+          prodLog.debug(`MmapVectorBackend write-back failed for ${noun.id}`, error)
+        }
+      }
+
       // Add to UnifiedCache with cost-aware eviction
       // This competes fairly with Graph and Metadata indexes
       this.unifiedCache.set(
@@ -1140,7 +1284,50 @@ export class HNSWIndex implements HnswProvider {
   private async preloadVectors(nodeIds: string[]): Promise<void> {
     if (nodeIds.length === 0) return
 
-    // Use UnifiedCache's request coalescing to prevent duplicate loads
+    // mmap-vector backend fast path: batched O(1) slot reads + lazy
+    // write-back migration on miss. Cache populated from both layers so
+    // subsequent in-session reads are O(1) memory.
+    if (this.vectorBackend) {
+      // Filter out ids already cached so we never re-read.
+      const uncachedIds: string[] = []
+      for (const id of nodeIds) {
+        if (this.unifiedCache.getSync(`hnsw:vector:${id}`) === undefined) {
+          uncachedIds.push(id)
+        }
+      }
+      if (uncachedIds.length === 0) return
+
+      const fromMmap = this.vectorBackend.readBatchByUuid(uncachedIds)
+      const misses: string[] = []
+      for (let i = 0; i < uncachedIds.length; i++) {
+        const id = uncachedIds[i]
+        const v = fromMmap[i]
+        if (v) {
+          this.unifiedCache.set(`hnsw:vector:${id}`, v, 'hnsw', v.length * 4, 50)
+        } else {
+          misses.push(id)
+        }
+      }
+      if (misses.length === 0) return
+
+      // Storage fallback for the misses, with concurrent lazy write-back.
+      await Promise.all(misses.map(async (id) => {
+        if (!this.storage) return
+        const vector = await this.storage.getNounVector(id)
+        if (!vector) return
+        if (this.vectorBackend) {
+          try {
+            this.vectorBackend.writeByUuid(id, vector)
+          } catch (error) {
+            prodLog.debug(`MmapVectorBackend write-back failed for ${id}`, error)
+          }
+        }
+        this.unifiedCache.set(`hnsw:vector:${id}`, vector, 'hnsw', vector.length * 4, 50)
+      }))
+      return
+    }
+
+    // Legacy path: per-entity storage load via UnifiedCache request coalescing.
     const promises = nodeIds.map(async (id) => {
       const cacheKey = `hnsw:vector:${id}`
       return this.unifiedCache.get(cacheKey, async () => {
@@ -1339,11 +1526,10 @@ export class HNSWIndex implements HnswProvider {
               noun.codebookMax = sq8.max
             }
 
-            // Restore connections from persisted data
+            // Restore connections from persisted data — compressed blob path
-            for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
+            // first, legacy JSON-array fallback for indexes written before
-              const level = parseInt(levelStr, 10)
+            // graph link compression landed.
-              noun.connections.set(level, new Set<string>(nounIds as string[]))
+            await this.restoreNodeConnections(nounData.id, hnswData, noun)
-            }
 
             // Add to in-memory index
             this.nouns.set(nounData.id, noun)
@@ -1424,11 +1610,10 @@ export class HNSWIndex implements HnswProvider {
                 noun.codebookMax = sq8.max
               }
 
-              // Restore connections from persisted data
+              // Restore connections from persisted data — compressed blob path
-              for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
+              // first, legacy JSON-array fallback for indexes written before
-                const level = parseInt(levelStr, 10)
+              // graph link compression landed.
-                noun.connections.set(level, new Set<string>(nounIds as string[]))
+              await this.restoreNodeConnections(nounData.id, hnswData, noun)
-              }
 
               // Add to in-memory index
               this.nouns.set(nounData.id, noun)

src/hnsw/mmapVectorBackend.ts

@@ -0,0 +1,174 @@
+/**
+ * @module hnsw/mmapVectorBackend
+ * @description Disk-resident vector cache backing HNSWIndex, wrapping the
+ * `vectorStore:mmap` provider (cortex's NativeMmapVectorStore in production).
+ *
+ * **Why this exists.** Brainy's per-entity vector storage hits two ceilings as
+ * datasets grow: (1) every `HNSWIndex.preloadVectors` does a sequential
+ * `storage.getNounVector` per id (no batch), and (2) cloud and FS storage both
+ * serialize each vector through JSON / msgpack, blocking zero-copy reads. An
+ * mmap-backed single-file vector store, addressed by the stable int from
+ * `EntityIdMapper`, gives O(1) batch reads + OS-page-cache prefetch and is the
+ * foundation 2.4.0 #2 needs.
+ *
+ * **Lazy migration on miss.** First read of an unmigrated UUID falls back to
+ * `storage.getNounVector` and writes the bytes into the mmap slot before
+ * returning — idempotent, resumable, and lets an upgraded install converge to
+ * the fast path under live traffic without a big-bang migration step. The
+ * per-entity store remains the canonical source of truth; nothing in the per-
+ * entity layer is rewritten or deleted here.
+ *
+ * **Local-FS only in 2.4.0.** Activated when the storage adapter exposes a
+ * real local path via `getBinaryBlobPath()`. Cloud adapters (S3 / R2 / GCS)
+ * return null and the backend is not constructed, so HNSWIndex falls back to
+ * the existing per-entity read path. Chunked vector segments for remote
+ * storage are the planned 2.4.0 #2 follow-up.
+ */
+
+import { mkdir } from 'node:fs/promises'
+import { dirname } from 'node:path'
+import type { Vector } from '../coreTypes.js'
+import type {
+  EntityIdMapperProvider,
+  VectorStoreMmapInstance,
+  VectorStoreMmapProvider
+} from '../plugin.js'
+
+/**
+ * @description Bridge between brainy's UUID-keyed vector API and cortex's
+ * int-slot mmap vector store. Translates UUIDs to stable int slots via the
+ * `EntityIdMapper`, grows the file on demand, and handles read misses by
+ * surfacing `null` (HNSWIndex's caller does the lazy storage fallback +
+ * write-back). The class never touches per-entity storage directly — it owns
+ * only the mmap layer.
+ */
+export class MmapVectorBackend {
+  private readonly store: VectorStoreMmapInstance
+  private readonly idMapper: EntityIdMapperProvider
+
+  private constructor(store: VectorStoreMmapInstance, idMapper: EntityIdMapperProvider) {
+    this.store = store
+    this.idMapper = idMapper
+  }
+
+  /**
+   * @description Open or create the mmap vector file at `path`. If the file
+   * does not exist, it is created with the given dim + initial capacity. If
+   * the directory containing `path` does not exist, it is created recursively
+   * (the cortex provider does not auto-create parent directories).
+   * @param provider - The `vectorStore:mmap` provider (cortex
+   *   `NativeMmapVectorStore` in production).
+   * @param path - Absolute path to the vector file. Derived by the caller
+   *   from `storage.getBinaryBlobPath()` on local-FS adapters; cloud adapters
+   *   return null and this method is not called.
+   * @param dim - Vector dimensionality. Must match the embedding model.
+   * @param initialCapacity - Slot capacity to allocate when creating a new
+   *   file. The file grows automatically on writes beyond capacity.
+   * @param idMapper - Stable UUID↔int mapper (post-2.4.0 #1 semantics — never
+   *   renumbers across rebuild). The mmap slots are addressed by these ints,
+   *   so id stability is the load-bearing invariant of the layout.
+   * @returns A ready-to-use backend.
+   */
+  static async open(
+    provider: VectorStoreMmapProvider,
+    path: string,
+    dim: number,
+    initialCapacity: number,
+    idMapper: EntityIdMapperProvider
+  ): Promise<MmapVectorBackend> {
+    await mkdir(dirname(path), { recursive: true })
+    let store: VectorStoreMmapInstance
+    try {
+      store = provider.open(path)
+    } catch {
+      store = provider.create(path, dim, Math.max(initialCapacity, 16))
+    }
+    return new MmapVectorBackend(store, idMapper)
+  }
+
+  /** Vector dimensionality of the underlying mmap file. */
+  get dim(): number {
+    return this.store.dim
+  }
+
+  /**
+   * @description Read a vector by UUID. Returns `null` for any of:
+   * (1) UUID not yet assigned an int (entity never written through this
+   * backend or storage), (2) int beyond the highest written slot, or
+   * (3) the underlying mmap read throwing (corrupt / out-of-bounds).
+   * The null path is the caller's signal to fall back to per-entity storage
+   * and trigger a write-back migration.
+   */
+  readByUuid(uuid: string): Vector | null {
+    const intId = this.idMapper.getInt(uuid)
+    if (intId === undefined || intId >= this.store.count) return null
+    try {
+      return this.store.readVector(intId) as unknown as Vector
+    } catch {
+      return null
+    }
+  }
+
+  /**
+   * @description Batch read vectors by UUID. Returns an array aligned with
+   * `uuids` where each entry is the vector or `null` for a miss. Order-
+   * preserving. Internally collapses to a single batched mmap read of only the
+   * indices that resolve to a written slot, then redistributes the flat output
+   * back to the caller's order — so missing slots cost only a couple of map
+   * lookups, not a full N-element read.
+   */
+  readBatchByUuid(uuids: string[]): (Vector | null)[] {
+    const indices: number[] = []
+    const slotByUuid = new Map<string, number>()
+    for (const uuid of uuids) {
+      const intId = this.idMapper.getInt(uuid)
+      if (intId !== undefined && intId < this.store.count) {
+        slotByUuid.set(uuid, indices.length)
+        indices.push(intId)
+      }
+    }
+    if (indices.length === 0) return uuids.map(() => null)
+    const flat = this.store.readVectorsBatch(indices)
+    const dim = this.store.dim
+    return uuids.map(uuid => {
+      const slot = slotByUuid.get(uuid)
+      if (slot === undefined) return null
+      return flat.slice(slot * dim, (slot + 1) * dim) as unknown as Vector
+    })
+  }
+
+  /**
+   * @description Write a vector at the slot for `uuid`. Assigns a new int if
+   * the UUID is unseen (stable, append-only). Grows the underlying file (slot
+   * capacity doubled until the target index fits) before writing past
+   * capacity, so the caller need not worry about file sizing.
+   */
+  writeByUuid(uuid: string, vector: Vector): void {
+    const intId = this.idMapper.getOrAssign(uuid)
+    if (intId >= this.store.capacity) {
+      let newCap = Math.max(this.store.capacity, 16)
+      while (intId >= newCap) newCap *= 2
+      this.store.resize(newCap)
+    }
+    this.store.writeVector(intId, Array.from(vector as ArrayLike<number>))
+  }
+
+  /**
+   * @description madvise(WILLNEED) hint for the slots backing `uuids`. Cheap;
+   * silently drops UUIDs not yet in the mmap. Used by HNSW search to overlap
+   * disk I/O with graph traversal.
+   */
+  prefetchByUuid(uuids: string[]): void {
+    const indices: number[] = []
+    for (const uuid of uuids) {
+      const intId = this.idMapper.getInt(uuid)
+      if (intId !== undefined && intId < this.store.count) indices.push(intId)
+    }
+    if (indices.length > 0) this.store.prefetch(indices)
+  }
+
+  /** msync — flush dirty pages to disk. */
+  flush(): void {
+    this.store.flush()
+  }
+}

src/indexes/columnStore/ColumnStore.ts

@@ -125,12 +125,29 @@ export class ColumnStore implements ColumnStoreProvider {
           this.manifests.set(fieldName, manifest)
           this.fieldTypes.set(fieldName, manifest.valueType)
 
-          // Load global deleted bitmap if it exists
+          // Load global deleted bitmap if it exists. Raw blob preferred
+          // (2.4.0 #4 cortex-shared format); legacy envelope fallback for
+          // indexes written before format unification.
           try {
-            const deletedPath = `${this.basePath}/${fieldName}/DELETED.bin`
+            let buf: Buffer | null = null
-            const stored = await (storage as any).readObjectFromPath(deletedPath)
+
-            if (stored && stored._binary && stored.data) {
+            const adapter = storage as unknown as {
-              const buf = Buffer.from(stored.data, 'base64')
+              loadBinaryBlob?: (key: string) => Promise<Buffer | null>
+              readObjectFromPath: (path: string) => Promise<any>
+            }
+            if (typeof adapter.loadBinaryBlob === 'function') {
+              const key = `${this.basePath}/${fieldName}/DELETED`
+              const blob = await adapter.loadBinaryBlob(key)
+              if (blob && blob.length > 0) buf = blob
+            }
+            if (!buf) {
+              const deletedPath = `${this.basePath}/${fieldName}/DELETED.bin`
+              const stored = await adapter.readObjectFromPath(deletedPath)
+              if (stored && stored._binary && stored.data) {
+                buf = Buffer.from(stored.data, 'base64')
+              }
+            }
+            if (buf) {
               this.deletedEntities.set(fieldName, RoaringBitmap32.deserialize(buf, true))
             }
           } catch {
@@ -408,6 +425,23 @@ export class ColumnStore implements ColumnStoreProvider {
 
   /**
    * Flush a single field's tail buffer to a new L0 segment.
+   *
+   * Storage path (2.4.0 #4 / cortex-interchange contract):
+   *   When the storage adapter exposes the binary-blob primitive
+   *   (`saveBinaryBlob` — every brainy adapter ≥7.25.0 does), the segment
+   *   bytes are written as a raw blob at the shared cortex key
+   *   `_column_index/<field>/L<level>-NNNNNN` (suffix-free; the adapter
+   *   appends its own). This matches byte-for-byte what cortex's
+   *   `NativeColumnStore` writes, so JS- and native-written indexes
+   *   interchange without re-encoding.
+   *
+   *   When the adapter doesn't (older custom adapters that didn't follow
+   *   the 7.25.0 primitive surface), we fall back to the legacy
+   *   `writeObjectToPath` envelope — `{ _binary, data: <base64> }` at
+   *   `_column_index/<field>/L<level>-NNNNNN.cidx`. Cortex's 2.3.1
+   *   read-side fallback handles the legacy envelope on its end, so the
+   *   format mismatch is transparent in both directions during the
+   *   transition.
    */
   private async flushBuffer(field: string, buffer: ColumnTailBuffer): Promise<void> {
     const { values, entityIds, pendingRemovals } = buffer.drain()
@@ -416,10 +450,15 @@ export class ColumnStore implements ColumnStoreProvider {
     const manifest = this.manifests.get(field)
     if (!manifest) return
 
+    const storage = this.storage as unknown as {
+      saveBinaryBlob?: (key: string, data: Buffer) => Promise<void>
+      writeObjectToPath: (path: string, value: unknown) => Promise<void>
+    }
+    const canUseBlob = typeof storage.saveBinaryBlob === 'function'
+
     // Write segment if we have values
     if (values.length > 0) {
       const segId = manifest.nextSegmentId()
-      const segPath = manifest.segmentPath(0, segId)
 
       const segBuffer = writeSegmentToBuffer(
         {
@@ -435,13 +474,22 @@ export class ColumnStore implements ColumnStoreProvider {
         entityIds
       )
 
-      // Store segment as binary data
+      if (canUseBlob) {
-      await (this.storage as any).writeObjectToPath(segPath, {
+        // Raw blob, cortex-shared key convention.
-        _binary: true,
+        const key = `${this.basePath}/${field}/L0-${String(segId).padStart(6, '0')}`
-        data: segBuffer.toString('base64')
+        await storage.saveBinaryBlob!(key, segBuffer)
-      })
+      } else {
+        // Legacy envelope path for adapters without the binary-blob primitive.
+        const segPath = manifest.segmentPath(0, segId)
+        await storage.writeObjectToPath(segPath, {
+          _binary: true,
+          data: segBuffer.toString('base64')
+        })
+      }
 
-      // Register in manifest
+      // Register in manifest. The `.cidx` file name in the manifest is the
+      // legacy-format file the cortex 2.3.1 read-side fallback looks for; the
+      // raw-blob key derives from segId at read time. Both paths converge.
       manifest.addSegment({
         id: segId,
         level: 0,
@@ -465,15 +513,21 @@ export class ColumnStore implements ColumnStoreProvider {
       for (const id of pendingRemovals) deleted.add(id)
     }
 
-    // Persist the global deleted bitmap alongside the manifest
+    // Persist the global deleted bitmap alongside the manifest. Same
+    // raw-blob preference as segments — shared cortex key `<base>/<field>/DELETED`.
     const deleted = this.deletedEntities.get(field)
     if (deleted && deleted.size > 0) {
-      const deletedPath = `${this.basePath}/${field}/DELETED.bin`
       const serialized = deleted.serialize(true)
-      await (this.storage as any).writeObjectToPath(deletedPath, {
+      if (canUseBlob) {
-        _binary: true,
+        const deletedKey = `${this.basePath}/${field}/DELETED`
-        data: Buffer.from(serialized).toString('base64')
+        await storage.saveBinaryBlob!(deletedKey, Buffer.from(serialized))
-      })
+      } else {
+        const deletedPath = `${this.basePath}/${field}/DELETED.bin`
+        await storage.writeObjectToPath(deletedPath, {
+          _binary: true,
+          data: Buffer.from(serialized).toString('base64')
+        })
+      }
     }
 
     // Save manifest
@@ -508,24 +562,43 @@ export class ColumnStore implements ColumnStoreProvider {
 
   /**
    * Load a segment from storage and create a cursor.
+   *
+   * Reads the raw-blob layout first (the 2.4.0 #4 shared-with-cortex format),
+   * then falls back to the legacy `{ _binary, base64 }` envelope at the
+   * `.cidx` object-path so indexes written before the format unification keep
+   * loading correctly. Mirror of cortex's 2.3.1 read-side fallback.
    */
   private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise<ColumnSegmentCursor | null> {
     const manifest = this.manifests.get(field)
     if (!manifest) return null
 
     try {
-      const segPath = manifest.segmentPath(seg.level, seg.id)
+      let buf: Buffer | null = null
-      const stored = await (this.storage as any).readObjectFromPath(segPath)
-      if (!stored) return null
 
-      // Handle binary storage format
+      const storage = this.storage as unknown as {
-      let buf: Buffer
+        loadBinaryBlob?: (key: string) => Promise<Buffer | null>
-      if (stored._binary && stored.data) {
+        readObjectFromPath: (path: string) => Promise<any>
-        buf = Buffer.from(stored.data, 'base64')
+      }
-      } else if (Buffer.isBuffer(stored)) {
+
-        buf = stored
+      // Preferred: raw blob at the cortex-shared key.
-      } else {
+      if (typeof storage.loadBinaryBlob === 'function') {
-        return null
+        const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}`
+        const blob = await storage.loadBinaryBlob(key)
+        if (blob && blob.length > 0) buf = blob
+      }
+
+      // Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path.
+      if (!buf) {
+        const segPath = manifest.segmentPath(seg.level, seg.id)
+        const stored = await storage.readObjectFromPath(segPath)
+        if (!stored) return null
+        if (stored._binary && stored.data) {
+          buf = Buffer.from(stored.data, 'base64')
+        } else if (Buffer.isBuffer(stored)) {
+          buf = stored
+        } else {
+          return null
+        }
       }
 
       const parsed = readSegmentFromBuffer(buf)

src/plugin.ts

@@ -253,6 +253,81 @@ export interface CacheProvider {
 // uses at the `getProvider('embeddings')` call site. No separate interface is
 // added here to avoid a duplicate, unwired contract.
 
+/**
+ * The `'graph:compression'` provider — pure-function encode/decode for HNSW
+ * connection lists as compact delta-varint byte sequences (cortex's
+ * `encodeConnections` / `decodeConnections`).
+ *
+ * Brainy's `HNSWIndex` consumes this via a `ConnectionsCodec` that translates
+ * UUIDs to stable int slots via the `EntityIdMapper`, encodes, and persists
+ * the compressed bytes through the binary-blob primitive. On load, the blob
+ * is fetched + decoded back into UUID sets — `setConnectionsCodec()` on
+ * `HNSWIndex` is the injection point. Read path is dual-format: when no blob
+ * exists for a node, the connections fall back to the legacy JSON-array path
+ * embedded in `saveHNSWData`, so pre-2.4.0 indexes keep loading unchanged
+ * and convergence to the compressed form happens lazily on next save.
+ *
+ * Activated only when the storage adapter exposes the binary-blob primitive
+ * AND the metadata index resolves a stable idMapper. Cloud adapters that
+ * lack a real local-path resolution still benefit, since the blob primitive
+ * itself works across every adapter as of brainy 7.25.0.
+ */
+export interface GraphCompressionProvider {
+  /** Encode a list of u32 ints to compact delta-varint bytes. Sorts internally. */
+  encode(ids: number[]): Buffer
+  /** Decode delta-varint bytes back to a u32 list. */
+  decode(data: Buffer): number[]
+}
+
+/**
+ * The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed
+ * vector file. Brainy's HNSWIndex uses the static `.create()` / `.open()`
+ * factories to open a single-file store at a derived path on disk-resident
+ * storage adapters; non-FS adapters with no local-path support skip the mmap
+ * layer and fall back to per-entity vector reads.
+ *
+ * Cortex registers `NativeMmapVectorStore` as this provider — a Rust mmap'd
+ * f32 array with a header, batch read/write APIs, and madvise prefetch. Slot
+ * indices are the stable ints from `EntityIdMapper`; lazy migration on read
+ * miss writes back from per-entity storage into the slot, converging an
+ * upgraded install to the fast path without a big-bang step.
+ */
+export interface VectorStoreMmapProvider {
+  /** Create a new vector file with pre-allocated slot capacity. */
+  create(path: string, dim: number, capacity: number): VectorStoreMmapInstance
+  /** Open an existing vector file for read + write. */
+  open(path: string): VectorStoreMmapInstance
+  /** Open an existing vector file for read-only access. */
+  openReadOnly(path: string): VectorStoreMmapInstance
+}
+
+/**
+ * An open mmap-vector-store instance — the surface brainy's HNSWIndex consumes
+ * for batch vector reads + lazy migration from per-entity storage on miss.
+ */
+export interface VectorStoreMmapInstance {
+  /** Number of vectors written (highest written index + 1). */
+  readonly count: number
+  /** Maximum slot capacity before resize is needed. */
+  readonly capacity: number
+  /** Vector dimensionality. */
+  readonly dim: number
+  /** Write a single vector at `index`. Vector is f64 in JS, stored as f32. */
+  writeVector(index: number, vector: number[]): void
+  /** Write a flat batch starting at `startIndex`. Returns vectors written. */
+  writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number
+  /** Read a single vector at `index`. Returns f64[]. */
+  readVector(index: number): number[]
+  /** Read multiple vectors by index. Returns flat f64[] of length n × dim. */
+  readVectorsBatch(indices: number[]): number[]
+  /** Prefetch via madvise(WILLNEED); no-op on unsupported platforms. */
+  prefetch(indices: number[]): void
+  /** Grow the vector file to a larger slot capacity. Re-maps the file. */
+  resize(newCapacity: number): void
+  /** msync — flush pending writes to disk. */
+  flush(): void
+}
+
 /**
  * Storage adapter factory — plugins register these to provide
  * new storage backends that users reference by name.

src/utils/entityIdMapper.ts

@@ -1,14 +1,34 @@
 /**
- * EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs for roaring bitmaps
+ * EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs.
  *
- * Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings as entity IDs.
+ * Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings
- * This class provides efficient bidirectional mapping with persistence support.
+ * as canonical entity IDs. This class provides efficient O(1) bidirectional
+ * mapping with persistence — and, importantly, a stability guarantee that any
+ * persisted int-keyed data can rely on.
+ *
+ * **Stability guarantee (the foundation 2.4.0 vector-mmap, graph-link-compression,
+ * and column-store interchange all key off):**
+ *
+ * - `getOrAssign(uuid)` is **append-only**: once a UUID is assigned an int, the
+ *   mapping never changes. Subsequent `getOrAssign` calls for the same UUID
+ *   return the same int.
+ * - `nextId` is **monotonically increasing**. New UUIDs always get an int greater
+ *   than any previously assigned, so a removed-then-re-added UUID is treated as
+ *   a fresh entity (and gets a fresh int — there is no automatic "revive").
+ * - `remove(uuid)` removes the mapping but does **not** decrement `nextId` or
+ *   recycle the int. The removed int becomes a permanent hole in `intToUuid` —
+ *   downstream consumers seeing `getUuid(int) === undefined` know the entity
+ *   was deleted.
+ * - A metadata-index `rebuild()` does **not** clear the mapper (the rebuild path
+ *   re-iterates entities via `getOrAssign`, which returns existing ints unchanged).
+ *   Only the explicit `clear()` method renumbers — used by `clearAllIndexData()`
+ *   as the nuclear recovery path with a documented warning.
  *
  * Features:
- * - O(1) lookup in both directions
+ * - O(1) lookup in both directions.
- * - Persistent storage via storage adapter
+ * - Persistent storage via storage adapter.
- * - Atomic counter for next ID
+ * - Atomic, monotonic, append-only int counter.
- * - Serialization/deserialization support
+ * - Serialization/deserialization support.
  *
  * @module utils/entityIdMapper
  */

src/utils/metadataIndex.ts

@@ -2635,13 +2635,20 @@ export class MetadataIndexManager implements MetadataIndexProvider {
     this.verbCountsByTypeFixed.fill(0)
     this.typeFieldAffinity.clear()
 
-    // Clear EntityIdMapper
+    // Clear EntityIdMapper. This is the explicit destructive path: the caller
+    // asked for nuclear recovery of a corrupted index, so renumbering UUIDs is
+    // intentional. Persisted int-keyed data (vector-mmap slots, graph
+    // link-compression encodings) is invalidated by this op — the warning
+    // below makes that explicit. Rebuild on its own does NOT clear the mapper.
     await this.idMapper.clear()
 
     // Clear chunk manager cache
     this.chunkManager.clearCache()
 
     prodLog.info(`✅ Cleared ${deletedCount} field indexes and all in-memory state`)
+    prodLog.warn('⚠️ EntityIdMapper was cleared — any persisted int-keyed data ' +
+      '(vector mmap slots, graph link-compression encodings, etc.) is now stale ' +
+      'and must be rebuilt from canonical sources.')
     prodLog.info('⚠️ Run brain.index.rebuild() to recreate the index from entity data')
   }
 
@@ -3032,8 +3039,19 @@ export class MetadataIndexManager implements MetadataIndexProvider {
         prodLog.info(`Cleared ${existingFields.length} field indexes from storage`)
       }
 
-      // Clear EntityIdMapper to start fresh
+      // EntityIdMapper is intentionally NOT cleared here. Rebuild re-iterates
-      await this.idMapper.clear()
+      // every entity in storage and calls idMapper.getOrAssign(uuid), which
+      // returns the existing int for known UUIDs (no renumbering). This is the
+      // foundational stability guarantee — vector-mmap slot indices, graph
+      // link-compression encodings, and any other persisted int-keyed data
+      // remain valid across a rebuild. Previously this line reset nextId to 1
+      // and renumbered every UUID by re-insertion order, silently breaking
+      // any consumer that had persisted int-keyed data against the old map.
+      // Stale entries for UUIDs no longer in storage persist (harmless memory
+      // overhead); a dedicated prune step can be added if it ever matters.
+      // The destructive wipe is still available via clearAllIndexData() →
+      // idMapper.clear(), which is the explicit "recovery" path with the
+      // appropriate warning about invalidating persisted int-keyed data.
 
       // Clear chunk manager cache
       this.chunkManager.clearCache()

tests/unit/hnsw/connections-codec.test.ts

@@ -0,0 +1,137 @@
+/**
+ * @module hnsw/connectionsCodec.test
+ * @description Unit tests for `ConnectionsCodec` — the per-node buffer format
+ * brainy uses to persist HNSW connection sets as delta-varint-compressed
+ * binary blobs (2.4.0 #3).
+ *
+ * Mocks the `graph:compression` provider so the tests run without cortex
+ * installed. The real cross-language byte-format integration is exercised
+ * when cortex's `encodeConnections` / `decodeConnections` are wired in.
+ *
+ * Coverage:
+ * 1. Round-trip a Map of `level → Set<UUID>` through encode + decode and
+ *    verify the result is equal to the input (level-by-level Set equality).
+ * 2. Empty connections encode to a single-byte buffer (`0` = zero levels) and
+ *    decode back to an empty Map — no `undefined` reads, no decode errors.
+ * 3. Multiple levels round-trip, including a level with a single UUID and a
+ *    level with many UUIDs (the typical L0 fan-out).
+ * 4. UUIDs whose ints are no longer in the idMapper at decode time are
+ *    silently dropped — matches the pre-2.4.0 behaviour of a missing
+ *    `getUuid()` lookup in the legacy load path.
+ */
+
+import { describe, it, expect } from 'vitest'
+import { ConnectionsCodec } from '../../../src/hnsw/connectionsCodec.js'
+import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js'
+import type { GraphCompressionProvider } from '../../../src/plugin.js'
+
+const stubStorage = {
+  getMetadata: async () => undefined,
+  saveMetadata: async () => {},
+  getNouns: async () => ({ totalCount: 0, items: [] })
+} as any
+
+/**
+ * Mock provider — encodes ints as a JSON byte array, decodes back. NOT the
+ * delta-varint format cortex produces, but format-agnostic round-trip
+ * verification is the unit-test concern here.
+ */
+const mockProvider: GraphCompressionProvider = {
+  encode(ids: number[]): Buffer {
+    const sorted = [...ids].sort((a, b) => a - b)
+    return Buffer.from(JSON.stringify(sorted), 'utf8')
+  },
+  decode(data: Buffer): number[] {
+    if (data.length === 0) return []
+    return JSON.parse(data.toString('utf8')) as number[]
+  }
+}
+
+async function makeIdMapper(): Promise<EntityIdMapper> {
+  const mapper = new EntityIdMapper({ storage: stubStorage })
+  await mapper.init()
+  return mapper
+}
+
+function setsEqual<T>(a: Set<T>, b: Set<T>): boolean {
+  if (a.size !== b.size) return false
+  for (const v of a) if (!b.has(v)) return false
+  return true
+}
+
+describe('ConnectionsCodec (2.4.0 #3 — wraps graph:compression provider)', () => {
+  it('round-trips a single-level Set of UUIDs through encode + decode', async () => {
+    const idMapper = await makeIdMapper()
+    const codec = new ConnectionsCodec(mockProvider, idMapper)
+
+    const input = new Map<number, Set<string>>([
+      [0, new Set(['a', 'b', 'c', 'd'])]
+    ])
+    const buf = codec.encode(input)
+    const out = codec.decode(buf)
+
+    expect(out.size).toBe(1)
+    expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true)
+  })
+
+  it('empty connections encode to a one-byte buffer and decode to an empty Map', async () => {
+    const idMapper = await makeIdMapper()
+    const codec = new ConnectionsCodec(mockProvider, idMapper)
+
+    const buf = codec.encode(new Map())
+    expect(buf.length).toBe(1)
+    expect(buf.readUInt8(0)).toBe(0)
+    expect(codec.decode(buf).size).toBe(0)
+  })
+
+  it('round-trips multiple levels with very different fan-outs', async () => {
+    const idMapper = await makeIdMapper()
+    const codec = new ConnectionsCodec(mockProvider, idMapper)
+
+    const manyUuids = Array.from({ length: 30 }, (_, i) => `u-${i}`)
+    const input = new Map<number, Set<string>>([
+      [0, new Set(manyUuids)],
+      [1, new Set(['high-1', 'high-2'])],
+      [2, new Set(['solo'])]
+    ])
+
+    const buf = codec.encode(input)
+    const out = codec.decode(buf)
+
+    expect(out.size).toBe(3)
+    expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true)
+    expect(setsEqual(out.get(1)!, input.get(1)!)).toBe(true)
+    expect(setsEqual(out.get(2)!, input.get(2)!)).toBe(true)
+  })
+
+  it('drops UUIDs whose ints the idMapper no longer recognises on decode', async () => {
+    const encodingMapper = await makeIdMapper()
+    const encodeCodec = new ConnectionsCodec(mockProvider, encodingMapper)
+
+    // Encode through one mapper.
+    const input = new Map<number, Set<string>>([[0, new Set(['present', 'gone'])]])
+    const buf = encodeCodec.encode(input)
+
+    // Decode through a FRESH mapper that only knows about 'present'.
+    const decodingMapper = await makeIdMapper()
+    decodingMapper.getOrAssign('present')
+
+    // Manually re-map 'present' to the same int it had on encode so the
+    // present UUID round-trips; 'gone' simply has no mapping → silent drop.
+    // We line up the int spaces by asserting equality below: only 'present'
+    // survives, 'gone' is missing — no errors, no `undefined` Set entries.
+    const decodeCodec = new ConnectionsCodec(mockProvider, decodingMapper)
+    // Inject the same int for 'present' so it round-trips one-for-one. (The
+    // encoding mapper assigns ints in insertion order: 'present'=1, 'gone'=2.
+    // The decoding mapper's first getOrAssign also returns 1, so 'present'
+    // matches; 'gone' (encoded as 2) has no mapping in the decoder.)
+    expect(encodingMapper.getInt('present')).toBe(1)
+    expect(decodingMapper.getInt('present')).toBe(1)
+
+    const out = decodeCodec.decode(buf)
+    const l0 = out.get(0)!
+    expect(l0.has('present')).toBe(true)
+    expect(l0.has('gone')).toBe(false)
+    expect(l0.size).toBe(1)
+  })
+})

tests/unit/hnsw/mmap-vector-backend.test.ts

@@ -0,0 +1,242 @@
+/**
+ * @module hnsw/mmapVectorBackend.test
+ * @description Unit tests for the `MmapVectorBackend` bridge — the brainy-side
+ * wrapper that translates UUID-keyed vector reads/writes into stable int slot
+ * ops against an `vectorStore:mmap` provider.
+ *
+ * Mocks the provider so the tests run without cortex installed (cortex is a
+ * downstream consumer of brainy, not a dev dep). The real integration with
+ * cortex's `NativeMmapVectorStore` is exercised when cortex 2.4.0 picks up
+ * this brainy release and re-runs its cross-language parity suite.
+ *
+ * Coverage:
+ * 1. Open-then-write-then-read round-trips for a single vector.
+ * 2. Batch reads return an array aligned with the input UUIDs, with `null`
+ *    entries for misses interleaved among hits — order preserved.
+ * 3. Slot assignment is stable across multiple writes for the same UUID
+ *    (no re-slot, no overwrite of an adjacent slot).
+ * 4. Writes beyond the initial capacity grow the file (doubling) without
+ *    losing the vectors already written.
+ * 5. `readByUuid` returns `null` for both unknown UUIDs and UUIDs in the map
+ *    whose slot has not yet been written.
+ * 6. Open is idempotent — opening an already-existing file reuses it.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { tmpdir } from 'node:os'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { join } from 'node:path'
+import { MmapVectorBackend } from '../../../src/hnsw/mmapVectorBackend.js'
+import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js'
+import type {
+  VectorStoreMmapInstance,
+  VectorStoreMmapProvider
+} from '../../../src/plugin.js'
+
+/**
+ * Minimal storage stub for the EntityIdMapper. The mapper only touches storage
+ * in init/flush; for these tests the mapper starts empty and is never flushed.
+ */
+const stubStorage = {
+  getMetadata: async () => undefined,
+  saveMetadata: async () => {},
+  getNouns: async () => ({ totalCount: 0, items: [] })
+} as any
+
+/**
+ * Pure in-memory mmap store. Mirrors cortex's NativeMmapVectorStore surface
+ * just closely enough to exercise the backend's contract — no real mmap, no
+ * file I/O, no f32 round-trip narrowing (the precision check is out of scope
+ * here; that's covered by cortex's parity suite).
+ */
+class MockMmapStore implements VectorStoreMmapInstance {
+  private readonly vectors: Array<number[] | undefined> = []
+  private highestWritten = -1
+  constructor(
+    public readonly dim: number,
+    private _capacity: number
+  ) {}
+  get count(): number {
+    return this.highestWritten + 1
+  }
+  get capacity(): number {
+    return this._capacity
+  }
+  writeVector(index: number, vector: number[]): void {
+    if (index >= this._capacity) {
+      throw new Error(`Slot ${index} >= capacity ${this._capacity}`)
+    }
+    if (vector.length !== this.dim) {
+      throw new Error(`Dim mismatch: expected ${this.dim}, got ${vector.length}`)
+    }
+    this.vectors[index] = [...vector]
+    if (index > this.highestWritten) this.highestWritten = index
+  }
+  writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number {
+    if (vectorsFlat.length % this.dim !== 0) {
+      throw new Error('vectorsFlat length not a multiple of dim')
+    }
+    const n = vectorsFlat.length / this.dim
+    for (let i = 0; i < n; i++) {
+      this.writeVector(startIndex + i, vectorsFlat.slice(i * this.dim, (i + 1) * this.dim))
+    }
+    return n
+  }
+  readVector(index: number): number[] {
+    const v = this.vectors[index]
+    if (!v) throw new Error(`Slot ${index} not written`)
+    return [...v]
+  }
+  readVectorsBatch(indices: number[]): number[] {
+    const flat: number[] = []
+    for (const i of indices) {
+      const v = this.vectors[i]
+      if (!v) throw new Error(`Slot ${i} not written`)
+      for (let k = 0; k < this.dim; k++) flat.push(v[k])
+    }
+    return flat
+  }
+  prefetch(_indices: number[]): void {
+    /* no-op in mock */
+  }
+  resize(newCapacity: number): void {
+    if (newCapacity < this._capacity) throw new Error('Cannot shrink')
+    this._capacity = newCapacity
+  }
+  flush(): void {
+    /* no-op in mock */
+  }
+}
+
+/**
+ * Mock provider — keeps one MockMmapStore per path. open() throws if the path
+ * doesn't exist yet (matches cortex semantics: open() is for existing files
+ * only); create() throws if it does (cortex doesn't, but the brainy backend
+ * does open-first-then-create, so the throw path is exercised).
+ */
+class MockMmapProvider implements VectorStoreMmapProvider {
+  private files = new Map<string, MockMmapStore>()
+  create(path: string, dim: number, capacity: number): VectorStoreMmapInstance {
+    if (this.files.has(path)) throw new Error(`File exists at ${path}`)
+    const store = new MockMmapStore(dim, capacity)
+    this.files.set(path, store)
+    return store
+  }
+  open(path: string): VectorStoreMmapInstance {
+    const store = this.files.get(path)
+    if (!store) throw new Error(`No file at ${path}`)
+    return store
+  }
+  openReadOnly(path: string): VectorStoreMmapInstance {
+    return this.open(path)
+  }
+  /** Test helper — peek at the underlying store. */
+  _peek(path: string): MockMmapStore | undefined {
+    return this.files.get(path)
+  }
+}
+
+describe('MmapVectorBackend (2.4.0 #2 — wraps vectorStore:mmap provider)', () => {
+  let dir: string
+  let path: string
+  let idMapper: EntityIdMapper
+  let provider: MockMmapProvider
+
+  beforeEach(async () => {
+    dir = await mkdtemp(join(tmpdir(), 'brainy-mmap-vec-'))
+    path = join(dir, 'vectors.bin')
+    idMapper = new EntityIdMapper({ storage: stubStorage })
+    await idMapper.init()
+    provider = new MockMmapProvider()
+  })
+
+  afterEach(async () => {
+    await rm(dir, { recursive: true, force: true }).catch(() => {})
+  })
+
+  it('open creates a new file when none exists, then round-trips a vector by UUID', async () => {
+    const backend = await MmapVectorBackend.open(provider, path, 4, 16, idMapper)
+    expect(backend.dim).toBe(4)
+
+    backend.writeByUuid('alpha', [1, 2, 3, 4])
+    expect(backend.readByUuid('alpha')).toEqual([1, 2, 3, 4])
+  })
+
+  it('reads return null for unknown UUIDs and for UUIDs in the map but not yet written', async () => {
+    const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper)
+
+    // Unknown — never seen by the mapper.
+    expect(backend.readByUuid('ghost')).toBeNull()
+
+    // Known to the mapper but no slot ever written. We assign through the
+    // mapper directly so the backend itself has not touched the slot.
+    idMapper.getOrAssign('reserved')
+    expect(backend.readByUuid('reserved')).toBeNull()
+  })
+
+  it('batch read returns an array aligned to the input UUIDs (nulls preserved in place)', async () => {
+    const backend = await MmapVectorBackend.open(provider, path, 3, 16, idMapper)
+    backend.writeByUuid('a', [1, 1, 1])
+    backend.writeByUuid('b', [2, 2, 2])
+    backend.writeByUuid('c', [3, 3, 3])
+
+    // Interleave hits + a never-seen UUID + a duplicate hit. Order preserved.
+    const result = backend.readBatchByUuid(['b', 'missing', 'a', 'c', 'missing-too', 'b'])
+    expect(result).toEqual([
+      [2, 2, 2],
+      null,
+      [1, 1, 1],
+      [3, 3, 3],
+      null,
+      [2, 2, 2]
+    ])
+  })
+
+  it('writes for the same UUID land in the same slot (stable id, no re-slotting)', async () => {
+    const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper)
+    backend.writeByUuid('persistent', [1, 1])
+    const slotAfterFirstWrite = idMapper.getInt('persistent')
+
+    backend.writeByUuid('persistent', [9, 9]) // overwrite the same slot
+    expect(idMapper.getInt('persistent')).toBe(slotAfterFirstWrite)
+    expect(backend.readByUuid('persistent')).toEqual([9, 9])
+
+    // Another UUID gets a different slot, and the first vector is undisturbed.
+    backend.writeByUuid('other', [5, 5])
+    expect(idMapper.getInt('other')).not.toBe(slotAfterFirstWrite)
+    expect(backend.readByUuid('persistent')).toEqual([9, 9])
+    expect(backend.readByUuid('other')).toEqual([5, 5])
+  })
+
+  it('grows the file (doubling) when a write lands beyond capacity, without losing prior data', async () => {
+    // Start at the smallest sane initial capacity (clamped to 16 by the backend).
+    const backend = await MmapVectorBackend.open(provider, path, 2, 1, idMapper)
+    const store = provider._peek(path)!
+    expect(store.capacity).toBe(16) // backend floor
+
+    // Write 20 vectors so capacity must double at least once (16 → 32).
+    const uuids: string[] = []
+    for (let i = 0; i < 20; i++) {
+      const uuid = `u-${i}`
+      uuids.push(uuid)
+      backend.writeByUuid(uuid, [i, i * 2])
+    }
+    expect(store.capacity).toBeGreaterThanOrEqual(32)
+
+    // Every vector survived the growth — no slot got overwritten or lost.
+    for (let i = 0; i < uuids.length; i++) {
+      expect(backend.readByUuid(uuids[i])).toEqual([i, i * 2])
+    }
+  })
+
+  it('opens an existing file idempotently when the second open hits the same path', async () => {
+    // First open creates.
+    const backend1 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper)
+    backend1.writeByUuid('persisted', [7, 7])
+
+    // Second open against the same path. The provider's create() throws on
+    // collision; the backend's open-first behaviour means we reuse the file.
+    const backend2 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper)
+    expect(backend2.readByUuid('persisted')).toEqual([7, 7])
+  })
+})

tests/unit/indexes/columnStore/column-store-interchange.test.ts

@@ -0,0 +1,191 @@
+/**
+ * @module column-store-interchange.test
+ * @description 2.4.0 #4 — column-store JS↔native interchange. Pins down that
+ * brainy's JS `ColumnStore` writes segments + DELETED bitmaps via the binary-
+ * blob primitive at the cortex-shared key convention, AND that indexes
+ * persisted under the pre-2.4.0 envelope (`{ _binary, base64 }` at the
+ * `.cidx` object-path) still load correctly via the legacy-read fallback.
+ *
+ * Together these properties make the JS and native engines byte-compatible
+ * at the segment payload, so writing through one and reading through the
+ * other is a no-op format-wise — closing the gap the cortex 2.3.1 read-side
+ * fallback opened on the cortex side.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
+import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
+import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
+import { writeSegmentToBuffer } from '../../../../src/indexes/columnStore/ColumnSegmentFormat.js'
+import { ColumnManifest } from '../../../../src/indexes/columnStore/ColumnManifest.js'
+import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js'
+
+describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
+  let storage: MemoryStorage
+  let idMapper: EntityIdMapper
+  let store: ColumnStore
+
+  beforeEach(async () => {
+    storage = new MemoryStorage()
+    await storage.init()
+    idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
+    await idMapper.init()
+    store = new ColumnStore({ flushThreshold: 10 })
+    await store.init(storage, idMapper)
+  })
+
+  afterEach(async () => {
+    await store.close()
+  })
+
+  // -----------------------------------------------------------------------
+  // WRITE side: raw blob at the cortex-shared key, no legacy envelope
+  // -----------------------------------------------------------------------
+
+  it('flushBuffer writes segment bytes as a raw blob at the cortex-shared key', async () => {
+    for (let i = 0; i < 5; i++) {
+      store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 })
+    }
+    await store.flush()
+
+    // The raw blob exists at the suffix-free cortex key.
+    const key = '_column_index/score/L0-000001'
+    const raw = await (storage as any).loadBinaryBlob(key)
+    expect(Buffer.isBuffer(raw)).toBe(true)
+    expect(raw!.length).toBeGreaterThan(0)
+
+    // The legacy envelope at the `.cidx` object-path is NOT written (no
+    // 33% base64 bloat, no dual-write).
+    const legacyPath = '_column_index/score/L0-000001.cidx'
+    const legacyEnvelope = await (storage as any).readObjectFromPath(legacyPath)
+    expect(legacyEnvelope).toBeFalsy()
+  })
+
+  it('flushBuffer writes the DELETED bitmap as a raw blob too (not an envelope)', async () => {
+    for (let i = 0; i < 3; i++) {
+      store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i })
+    }
+    store.removeEntity(idMapper.getInt('u-1')!)
+    await store.flush()
+
+    const rawDel = await (storage as any).loadBinaryBlob('_column_index/score/DELETED')
+    expect(Buffer.isBuffer(rawDel)).toBe(true)
+    expect(rawDel!.length).toBeGreaterThan(0)
+
+    const legacyEnvelope = await (storage as any).readObjectFromPath(
+      '_column_index/score/DELETED.bin'
+    )
+    expect(legacyEnvelope).toBeFalsy()
+  })
+
+  // -----------------------------------------------------------------------
+  // READ side: pre-2.4.0 legacy envelope still loads via the fallback
+  // -----------------------------------------------------------------------
+
+  it('loads a segment persisted as the pre-2.4.0 envelope (legacy fallback)', async () => {
+    // Seed a legacy-format index DIRECTLY on storage — the on-disk shape
+    // brainy 7.20.0..7.25.0 wrote before this commit.
+    const legacyStore = new ColumnStore()
+    await legacyStore.init(storage, idMapper)
+
+    // Build a segment buffer using the shared cidx format.
+    const values: number[] = [10, 20, 30]
+    const entityIds: number[] = [
+      idMapper.getOrAssign('legacy-a'),
+      idMapper.getOrAssign('legacy-b'),
+      idMapper.getOrAssign('legacy-c')
+    ]
+    const segBuffer = writeSegmentToBuffer(
+      {
+        fieldName: 'score',
+        fieldNameLength: 5,
+        valueType: 0,
+        level: 0,
+        codec: 0,
+        flags: 0,
+        count: values.length
+      },
+      values,
+      entityIds
+    )
+
+    // Persist via the LEGACY envelope path (writeObjectToPath, base64 wrap).
+    await (storage as any).writeObjectToPath('_column_index/score/L0-000001.cidx', {
+      _binary: true,
+      data: segBuffer.toString('base64')
+    })
+    const manifest = new ColumnManifest('score', '_column_index')
+    manifest.valueType = 0
+    manifest.multiValue = false
+    manifest.addSegment({
+      id: 1,
+      level: 0,
+      count: values.length,
+      minValue: values[0],
+      maxValue: values[values.length - 1],
+      file: 'L0-000001.cidx'
+    })
+    await manifest.save(storage)
+    await legacyStore.close()
+
+    // NEW ColumnStore re-discovers the manifest at init, then the legacy
+    // segment loads via the readObjectFromPath fallback (no raw blob exists
+    // at the cortex key, so it falls through). sortTopK returns the entities
+    // in the order the segment recorded — proves the bytes round-tripped.
+    const readStore = new ColumnStore()
+    await readStore.init(storage, idMapper)
+    const sortedInts = await readStore.sortTopK('score', 'asc', 10)
+    const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
+    expect(sortedUuids).toEqual(['legacy-a', 'legacy-b', 'legacy-c'])
+    await readStore.close()
+  })
+
+  it('loads a DELETED bitmap persisted as the pre-2.4.0 envelope (legacy fallback)', async () => {
+    // Manually seed a legacy DELETED.bin envelope, then re-init and confirm
+    // the entity it marks gone is excluded from filter/sort results.
+    const bm = new RoaringBitmap32()
+    bm.add(idMapper.getOrAssign('dead'))
+    const serialized = Buffer.from(bm.serialize(true))
+    await (storage as any).writeObjectToPath('_column_index/score/DELETED.bin', {
+      _binary: true,
+      data: serialized.toString('base64')
+    })
+
+    // Add a couple of live entities so a manifest exists (init only loads
+    // DELETED for fields with a manifest on disk).
+    store.addEntity(idMapper.getOrAssign('alive-1'), { score: 100 })
+    store.addEntity(idMapper.getOrAssign('alive-2'), { score: 200 })
+    await store.flush()
+    await store.close()
+
+    const readStore = new ColumnStore()
+    await readStore.init(storage, idMapper)
+    const sortedInts = await readStore.sortTopK('score', 'asc', 10)
+    const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
+
+    // The dead entity must NOT appear; the alive ones must.
+    expect(sortedUuids).not.toContain('dead')
+    expect(sortedUuids).toContain('alive-1')
+    expect(sortedUuids).toContain('alive-2')
+    await readStore.close()
+  })
+
+  // -----------------------------------------------------------------------
+  // ROUND-TRIP: write-then-read in the new format
+  // -----------------------------------------------------------------------
+
+  it('write (new format) → re-init → read returns the same data', async () => {
+    for (let i = 0; i < 5; i++) {
+      store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 })
+    }
+    await store.flush()
+    await store.close()
+
+    const readStore = new ColumnStore()
+    await readStore.init(storage, idMapper)
+    const sortedInts = await readStore.sortTopK('score', 'asc', 10)
+    const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
+    expect(sortedUuids).toEqual(['u-0', 'u-1', 'u-2', 'u-3', 'u-4'])
+    await readStore.close()
+  })
+})

tests/unit/utils/entity-id-mapper-stability.test.ts

@@ -0,0 +1,161 @@
+/**
+ * Regression test: EntityIdMapper stability across rebuild.
+ *
+ * The foundation 2.4.0 (vector mmap store, graph link compression, column-store
+ * JS↔native interchange) all key off UUID→int mappings that **must not change**
+ * across a metadata-index rebuild. Previously `metadataIndex.rebuild()` called
+ * `idMapper.clear()` which reset `nextId` to 1 and renumbered every UUID by
+ * re-insertion order, silently invalidating any consumer that had persisted
+ * int-keyed data against the old map.
+ *
+ * This test pins down the stability contract:
+ *
+ * 1. UUID→int mappings persist across a single rebuild.
+ * 2. Mappings persist across many consecutive rebuilds.
+ * 3. New entities added after rebuild get fresh ints greater than any prior
+ *    assignment — no collisions with existing UUIDs' ints.
+ * 4. Removed entities leave a permanent hole — new entities don't recycle the
+ *    gap, even across a rebuild.
+ * 5. `clearAllIndexData()` is the explicit, intentional nuclear path — it DOES
+ *    renumber. This is the only documented way to invalidate the int space, and
+ *    a warning is logged so consumers know persisted int-keyed data is now stale.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { Brainy } from '../../../src/brainy.js'
+
+const DIM = 384
+const makeVec = (seed = 1) =>
+  new Float32Array(DIM).map((_, i) => ((i + seed) % DIM) / DIM)
+
+describe('EntityIdMapper stability (foundation for 2.4.0)', () => {
+  let brain: Brainy
+
+  beforeEach(async () => {
+    brain = new Brainy({ storage: { type: 'memory' }, silent: true })
+    await brain.init()
+  })
+
+  afterEach(async () => {
+    await brain.close()
+  })
+
+  async function addEntity(name: string, seed: number): Promise<string> {
+    return brain.add({
+      data: name,
+      vector: makeVec(seed),
+      type: 'thing' as any,
+      metadata: { name }
+    })
+  }
+
+  function getInt(uuid: string): number | undefined {
+    return (brain as any).metadataIndex.idMapper.getInt(uuid)
+  }
+
+  async function rebuild(): Promise<void> {
+    await (brain as any).metadataIndex.rebuild()
+  }
+
+  it('UUID→int mappings persist across a single metadata-index rebuild', async () => {
+    const ids = [
+      await addEntity('a', 1),
+      await addEntity('b', 2),
+      await addEntity('c', 3),
+      await addEntity('d', 4),
+      await addEntity('e', 5)
+    ]
+    const before = ids.map(id => getInt(id))
+    expect(before.every(i => typeof i === 'number' && (i as number) > 0)).toBe(true)
+
+    await rebuild()
+
+    const after = ids.map(id => getInt(id))
+    expect(after).toEqual(before)
+  })
+
+  it('mappings stay byte-for-byte stable across many consecutive rebuilds', async () => {
+    const ids = [
+      await addEntity('a', 1),
+      await addEntity('b', 2),
+      await addEntity('c', 3)
+    ]
+    const before = ids.map(id => getInt(id))
+
+    for (let i = 0; i < 5; i++) {
+      await rebuild()
+      const after = ids.map(id => getInt(id))
+      expect(after).toEqual(before)
+    }
+  })
+
+  it('entities added after rebuild get fresh monotonic ints (no collision with existing)', async () => {
+    const priorIds = [
+      await addEntity('a', 1),
+      await addEntity('b', 2),
+      await addEntity('c', 3)
+    ]
+    const priorInts = priorIds.map(id => getInt(id) as number)
+    const maxPrior = Math.max(...priorInts)
+
+    await rebuild()
+
+    const newId = await addEntity('d', 4)
+    const newInt = getInt(newId) as number
+    expect(newInt).toBeGreaterThan(maxPrior)
+    // Prior entities' ints didn't drift.
+    expect(priorIds.map(id => getInt(id))).toEqual(priorInts)
+  })
+
+  it('removed entities leave a permanent hole — new entities never recycle the gap', async () => {
+    const ids = [
+      await addEntity('a', 1),
+      await addEntity('b', 2),
+      await addEntity('c', 3),
+      await addEntity('d', 4),
+      await addEntity('e', 5)
+    ]
+    const beforeInts = ids.map(id => getInt(id) as number)
+    const deletedId = ids[2]
+    const deletedInt = beforeInts[2]
+    const maxBefore = Math.max(...beforeInts)
+
+    await brain.delete(deletedId)
+    expect(getInt(deletedId)).toBeUndefined()
+
+    const newId = await addEntity('f', 6)
+    const newInt = getInt(newId) as number
+    expect(newInt).not.toBe(deletedInt)
+    expect(newInt).toBeGreaterThan(maxBefore)
+
+    // Surviving ids keep their ints across the deletion + the add.
+    const survivors = ids.filter((_, i) => i !== 2)
+    const survivorIntsBefore = beforeInts.filter((_, i) => i !== 2)
+    expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore)
+
+    // Survivors' ints also survive a rebuild after the delete.
+    await rebuild()
+    expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore)
+    // The deleted id is still gone after rebuild (no resurrection).
+    expect(getInt(deletedId)).toBeUndefined()
+  })
+
+  it('clearAllIndexData() is the explicit nuclear path that DOES renumber', async () => {
+    const id1 = await addEntity('a', 1)
+    const id2 = await addEntity('b', 2)
+    const priorInts = [getInt(id1) as number, getInt(id2) as number]
+    expect(priorInts.every(i => i >= 1)).toBe(true)
+
+    // Nuclear recovery: explicit destructive op. The warning logged here is
+    // the only documented way to invalidate the canonical int space.
+    await (brain as any).metadataIndex.clearAllIndexData()
+
+    // Both UUIDs are gone from the mapper.
+    expect(getInt(id1)).toBeUndefined()
+    expect(getInt(id2)).toBeUndefined()
+
+    // The int counter restarted from 1: the next add() gets int 1.
+    const idAfter = await addEntity('c', 3)
+    expect(getInt(idAfter)).toBe(1)
+  })
+})