soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

feat: stable EntityIdMapper — rebuild() no longer renumbers (2.4.0 #1, the foundation) #1

Merged
dpsifr merged 4 commits from feat/2.4.0-storage into main 2026-05-28 19:37:48 +02:00
Conversation 0 Commits 4 Files changed 12 +1613 -81
12 changed files with 1613 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

121
src/brainy.ts
View file

@ -37,7 +37,14 @@ import { setGlobalCache } from './utils/unifiedCache.js'
import type { UnifiedCache } from './utils/unifiedCache.js'
import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js'
import { PluginRegistry } from './plugin.js'
import type { BrainyPlugin, BrainyPluginContext } from './plugin.js'
import type {
BrainyPlugin,
BrainyPluginContext,
GraphCompressionProvider,
VectorStoreMmapProvider
} from './plugin.js'
import { MmapVectorBackend } from './hnsw/mmapVectorBackend.js'
import { ConnectionsCodec } from './hnsw/connectionsCodec.js'
import { TransactionManager } from './transaction/TransactionManager.js'
import {
ValidationConfig,
@ -559,6 +566,23 @@ export class Brainy<T = any> implements BrainyInterface<T> {
this.graphIndex = graphIndex
}
// Wire the mmap-vector backend (2.4.0 #2). When the vectorStore:mmap
// provider is registered AND the storage adapter resolves a real local
// path via getBinaryBlobPath(), open the mmap file there and inject the
// backend into HNSWIndex. Cloud adapters return null and skip silently;
// HNSWIndex's behaviour is then identical to pre-2.4.0 (per-entity reads
// from canonical storage). Failures are non-fatal — the index still
// works, just without the zero-copy fast path.
await this.wireMmapVectorBackend()
// Wire the connections codec (2.4.0 #3). When the graph:compression
// provider is registered AND the metadata index exposes a stable
// idMapper, inject a codec that encodes HNSW connections as
// delta-varint blobs at save time and decodes on load. The blob
// primitive itself works on every brainy 7.25.0 adapter, so unlike the
// mmap-vector backend this layer engages even on cloud adapters.
this.wireConnectionsCodec()
// Rebuild indexes if needed for existing data
await this.rebuildIndexesIfNeeded()
@ -7716,6 +7740,101 @@ export class Brainy<T = any> implements BrainyInterface<T> {
await this.lazyRebuildPromise
}
/**
* @description Open the mmap-vector backend and inject it into HNSWIndex.
* Called once during init after plugin activation, metadataIndex init, and
* graphIndex setup — so the idMapper is available and providers are wired.
*
* Activation conditions (ALL must hold; otherwise this is a no-op and the
* index falls back to per-entity storage reads, the pre-2.4.0 behaviour):
* 1. The `vectorStore:mmap` provider is registered (cortex registers this).
* 2. The storage adapter resolves a real local path via `getBinaryBlobPath()`
* — cloud adapters return null and the mmap layer is skipped silently.
* 3. The metadata index exposes its idMapper (a stable UUID↔int map). The
* mmap layout is keyed by these ints, so 2.4.0 #1 is the prerequisite.
*
* Vector dimensionality: read from `this.dimensions` if set; otherwise the
* default for the standard embedding model (384). The dim is fixed at
* file-creation time and cannot change later, so a wrong default would only
* matter on the very first init of a brand-new instance. After the first
* `add()` the in-memory `this.dimensions` is set, and any subsequent init
* (or a re-open) sees the correct value.
*
* Failures here are non-fatal: a debug-level log records the reason, and
* HNSWIndex continues without the mmap fast path.
*/
private async wireMmapVectorBackend(): Promise<void> {
const provider = this.pluginRegistry.getProvider<VectorStoreMmapProvider>('vectorStore:mmap')
if (!provider) return
const storageWithBlob = this.storage as unknown as {
getBinaryBlobPath?: (key: string) => string | null
}
const vectorPath = storageWithBlob.getBinaryBlobPath?.('_vectors/main') ?? null
if (!vectorPath) return
const idMapper = this.metadataIndex.getIdMapper?.()
if (!idMapper) return
const dim = this.dimensions ?? 384
const initialCapacity = Math.max(idMapper.size * 2, 1024)
try {
const backend = await MmapVectorBackend.open(
provider,
vectorPath,
dim,
initialCapacity,
idMapper
)
this.index.setVectorBackend(backend)
if (!this.config.silent) {
console.log(
`[brainy] vector mmap backend wired (path=${vectorPath}, dim=${dim}, capacity=${initialCapacity})`
)
}
} catch (error) {
// Common cases: dim mismatch from a prior init at a different dimension,
// permission errors, disk full, file corruption. Index keeps working via
// the per-entity storage path; flag the cause for diagnosis.
if (!this.config.silent) {
console.warn(
`[brainy] mmap-vector backend not wired (${(error as Error).message}); ` +
`falling back to per-entity vector reads`
)
}
}
}
/**
* @description Wire the HNSW connections codec (2.4.0 #3). Activates when
* BOTH (a) the `graph:compression` provider is registered (cortex registers
* `{ encode: encodeConnections, decode: decodeConnections }`), and (b) the
* metadata index exposes a stable idMapper. Failures are non-fatal: HNSW
* keeps working via the legacy JSON-array path.
*
* Unlike the mmap-vector backend, this layer does NOT require a real local
* path — the binary-blob primitive saves the compressed bytes through the
* storage adapter directly, so the codec is engaged on cloud adapters too.
* Format convergence is lazy: pre-2.4.0 nodes load via the legacy path,
* then the next dirty save writes the compressed form (and the legacy
* field of saveHNSWData becomes empty), so all reads converge over time
* without an explicit migration step.
*/
private wireConnectionsCodec(): void {
const provider = this.pluginRegistry.getProvider<GraphCompressionProvider>('graph:compression')
if (!provider) return
const idMapper = this.metadataIndex.getIdMapper?.()
if (!idMapper) return
const codec = new ConnectionsCodec(provider, idMapper)
this.index.setConnectionsCodec(codec)
if (!this.config.silent) {
console.log('[brainy] graph link compression wired (delta-varint connections via graph:compression provider)')
}
}
/**
* Rebuild indexes from persisted data if needed (LAZY LOADING)
*

137
src/hnsw/connectionsCodec.ts Normal file
View file

@ -0,0 +1,137 @@
/**
* @module hnsw/connectionsCodec
* @description Bridge between brainy's UUID-keyed HNSW connection sets and
* cortex's int-keyed delta-varint encode/decode (the `graph:compression`
* provider). Translates UUIDs to stable int slots via the post-2.4.0 #1
* `EntityIdMapper`, batches all of a node's per-level connection lists into
* a single compact buffer, and reverses the path on load.
*
* Wire format of the per-node buffer the codec produces:
*
* [num_levels: u8]
* repeated num_levels times:
* [level: u8]
* [encoded_len: u32 LE]
* [encoded_bytes: <encoded_len> bytes] <- provider.encode(ints)
*
* Why a single buffer per node and not one blob per level:
* - One `saveBinaryBlob` call per persisted node, regardless of level count.
* On hot insert paths an HNSW node may be at levels 0..3; the per-node
* blob trades a one-byte count for 3 saved I/O round-trips.
* - Read path is symmetric: one `loadBinaryBlob` reconstructs the full
* connection set, so HNSW rebuild stays linear in the number of nodes.
*
* Loose UUIDs (ints assigned to entities that no longer exist) are silently
* dropped on decode. They can't legitimately participate in graph traversal
* because the entity is gone; treating them as `undefined` UUIDs matches the
* pre-2.4.0 behaviour of an `idMapper.getUuid()` miss in the legacy load path.
*/
import type {
EntityIdMapperProvider,
GraphCompressionProvider
} from '../plugin.js'
export class ConnectionsCodec {
constructor(
private readonly provider: GraphCompressionProvider,
private readonly idMapper: EntityIdMapperProvider
) {}
/**
* @description Encode all levels of a node's connections into one compact
* buffer. Empty input → an empty buffer (zero levels, no body). The
* idMapper assigns stable ints on demand if a connection UUID has never
* been seen — append-only, matches the rest of the 2.4.0 contract.
*/
encode(connectionsByLevel: Map<number, Set<string>>): Buffer {
if (connectionsByLevel.size === 0) {
// Single byte: zero levels. Round-trips cleanly through decode.
return Buffer.from([0])
}
if (connectionsByLevel.size > 0xff) {
throw new Error(
`ConnectionsCodec.encode: too many HNSW levels (${connectionsByLevel.size}); ` +
`the per-node header is one byte (max 255 levels).`
)
}
// First pass: encode each level into its own buffer.
const levelBuffers: Array<{ level: number; buf: Buffer }> = []
let bodyBytes = 0
for (const [level, uuids] of connectionsByLevel) {
if (level > 0xff) {
throw new Error(
`ConnectionsCodec.encode: level ${level} exceeds u8 header range (max 255).`
)
}
const ints: number[] = []
for (const uuid of uuids) {
ints.push(this.idMapper.getOrAssign(uuid))
}
const buf = this.provider.encode(ints)
levelBuffers.push({ level, buf })
bodyBytes += 1 + 4 + buf.length // level + len + payload
}
// Second pass: concatenate with header.
const out = Buffer.alloc(1 + bodyBytes)
out.writeUInt8(levelBuffers.length, 0)
let offset = 1
for (const { level, buf } of levelBuffers) {
out.writeUInt8(level, offset)
offset += 1
out.writeUInt32LE(buf.length, offset)
offset += 4
buf.copy(out, offset)
offset += buf.length
}
return out
}
/**
* @description Decode a per-node buffer back into a connections Map. Ints
* for entities the idMapper no longer knows are silently dropped — see the
* module-level note on why.
*/
decode(data: Buffer): Map<number, Set<string>> {
const out = new Map<number, Set<string>>()
if (data.length === 0) return out
const count = data.readUInt8(0)
let offset = 1
for (let i = 0; i < count; i++) {
if (offset + 5 > data.length) {
throw new Error(`ConnectionsCodec.decode: truncated header at level ${i}`)
}
const level = data.readUInt8(offset)
offset += 1
const len = data.readUInt32LE(offset)
offset += 4
if (offset + len > data.length) {
throw new Error(`ConnectionsCodec.decode: truncated body at level ${level}`)
}
const levelBuf = data.slice(offset, offset + len)
offset += len
const ints = this.provider.decode(levelBuf)
const uuids = new Set<string>()
for (const intId of ints) {
const uuid = this.idMapper.getUuid(intId)
if (uuid !== undefined) uuids.add(uuid)
}
out.set(level, uuids)
}
return out
}
}
/**
* @description Build the binary-blob storage key for a node's compressed
* connections. Suffix-free; the adapter appends its own. Keys live under
* `_hnsw_conn/` so they don't collide with the existing `_column_index/`
* blobs and so a cortex-side reader knows exactly where to look.
*/
export function compressedConnectionsKey(nodeId: string): string {
return `_hnsw_conn/${nodeId}`
}

267
src/hnsw/hnswIndex.ts
View file

@ -18,6 +18,8 @@ import { prodLog } from '../utils/logger.js'
import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js'
import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js'
import type { HnswProvider } from '../plugin.js'
import { MmapVectorBackend } from './mmapVectorBackend.js'
import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js'
// Default HNSW parameters
const DEFAULT_CONFIG: HNSWConfig = {
@ -49,6 +51,23 @@ export class HNSWIndex implements HnswProvider {
private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes
// Always-adaptive caching - no "mode" concept, system adapts automatically
// Optional mmap-vector backend (2.4.0 #2). When set, the read paths
// (preloadVectors, getVectorSafe) try the mmap layer before storage, and
// on a storage hit they write back into the mmap slot — a lazy migration
// that converges upgraded installs to the fast path under live traffic.
// Null on cloud storage adapters (no local path) and pre-injection init.
private vectorBackend: MmapVectorBackend | null = null
// Optional connections codec (2.4.0 #3). When set, node-persist writes the
// node's per-level connection sets as a single delta-varint-compressed
// binary blob (typically ~4× smaller than the legacy JSON-UUID-array
// shape), plus an empty `connections: {}` in saveHNSWData as the marker.
// The read path tries to load the blob first; missing blob → legacy
// connections field. Format convergence is lazy: pre-2.4.0 nodes still
// load via the legacy path, then write the compressed form on next dirty
// save, so the migration converges under live traffic with no big-bang.
private connectionsCodec: ConnectionsCodec | null = null
// COW (Copy-on-Write) support
private cowEnabled: boolean = false
private cowModifiedNodes: Set<string> = new Set()
@ -101,6 +120,35 @@ export class HNSWIndex implements HnswProvider {
this.useParallelization = useParallelization
}
/**
* @description Inject (or detach) the mmap-vector backend. When set, the
* vector read paths (`preloadVectors`, `getVectorSafe`) try the mmap layer
* before storage and lazily write back on storage hits — converging an
* upgraded install to the zero-copy fast path without a migration step.
* Setting to null reverts to the existing per-entity read path.
*
* Lifecycle: brainy.ts wires this after plugin activation when (a) the
* `vectorStore:mmap` provider is registered AND (b) the storage adapter
* resolves a real local path via `getBinaryBlobPath()`. Cloud adapters
* leave it null; HNSWIndex's behaviour is then identical to pre-2.4.0.
*/
public setVectorBackend(backend: MmapVectorBackend | null): void {
this.vectorBackend = backend
}
/**
* @description Inject (or detach) the connections codec. When set, node
* persistence writes a delta-varint-compressed binary blob alongside an
* empty `connections: {}` marker in saveHNSWData; the load path reads the
* blob and decodes. Null reverts to the legacy JSON-array path. Wiring is
* done by brainy.ts when the `graph:compression` provider is registered
* AND the storage adapter exposes the binary-blob primitive AND the
* metadata index has a stable idMapper.
*/
public setConnectionsCodec(codec: ConnectionsCodec | null): void {
this.connectionsCodec = codec
}
/**
* Get whether parallelization is enabled
*/
@ -144,15 +192,7 @@ export class HNSWIndex implements HnswProvider {
const noun = this.nouns.get(nodeId)
if (!noun) return Promise.resolve() // Node was deleted
const connectionsObj: Record<string, string[]> = {}
for (const [level, nounIds] of noun.connections.entries()) {
connectionsObj[level.toString()] = Array.from(nounIds)
}
return this.storage!.saveHNSWData(nodeId, {
level: noun.level,
connections: connectionsObj
}).catch(error => {
return this.persistNodeConnections(nodeId, noun).catch(error => {
console.error(`[HNSW flush] Failed to persist node ${nodeId}:`, error)
})
})
@ -183,6 +223,94 @@ export class HNSWIndex implements HnswProvider {
return nodeCount
}
/**
* @description Persist one node's connections. When the connections codec is
* wired AND the storage adapter exposes `saveBinaryBlob`, the per-level
* connection sets are encoded into a single compact buffer and stored as a
* binary blob; `saveHNSWData` then records the node's level and an empty
* `connections: {}` as the marker. Otherwise the legacy JSON-array path is
* taken — the format every brainy release before 2.4.0 wrote. The two are
* intentionally NOT dual-written: one or the other, never both, so format
* convergence is unambiguous on the read side.
*
* Shared by all three save sites (deferred flush, immediate-mode new-entity
* persist, immediate-mode neighbor update) so the codec branch is exercised
* uniformly regardless of which path triggered the write.
*/
private async persistNodeConnections(nodeId: string, noun: HNSWNoun): Promise<void> {
if (!this.storage) return
const storageWithBlob = this.storage as unknown as {
saveBinaryBlob?: (key: string, data: Buffer) => Promise<void>
}
const canCompress =
this.connectionsCodec !== null &&
typeof storageWithBlob.saveBinaryBlob === 'function'
if (canCompress) {
const encoded = this.connectionsCodec!.encode(noun.connections)
await storageWithBlob.saveBinaryBlob!(compressedConnectionsKey(nodeId), encoded)
await this.storage.saveHNSWData(nodeId, {
level: noun.level,
connections: {}
})
return
}
const connectionsObj: Record<string, string[]> = {}
for (const [level, nounIds] of noun.connections.entries()) {
connectionsObj[level.toString()] = Array.from(nounIds)
}
await this.storage.saveHNSWData(nodeId, {
level: noun.level,
connections: connectionsObj
})
}
/**
* @description Restore one node's connections from persisted storage. Tries
* the compressed-blob path first (when codec wired AND blob primitive
* available); a present non-empty buffer is decoded and populates
* `noun.connections`. A missing blob (no payload at the key) means this node
* was last persisted under the legacy format, so the legacy
* `hnswData.connections` field is used instead.
*
* On any decode error the legacy field is also used as a safety net — the
* codec's `decode` throws on truncated input, which would otherwise leave
* the node with an empty connections Map (orphaned from the graph).
*/
private async restoreNodeConnections(
nodeId: string,
hnswData: { level: number; connections: Record<string, string[]> },
noun: HNSWNoun
): Promise<void> {
const storageWithBlob = this.storage as unknown as {
loadBinaryBlob?: (key: string) => Promise<Buffer | null>
}
const canDecompress =
this.connectionsCodec !== null &&
typeof storageWithBlob.loadBinaryBlob === 'function'
if (canDecompress) {
try {
const buf = await storageWithBlob.loadBinaryBlob!(compressedConnectionsKey(nodeId))
if (buf && buf.length > 0) {
const decoded = this.connectionsCodec!.decode(buf)
noun.connections = decoded
return
}
} catch (error) {
prodLog.debug(`HNSW: compressed connections decode failed for ${nodeId}; falling back to legacy`, error)
}
}
// Legacy fallback: JSON UUID arrays in the HNSW data object.
for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
const level = parseInt(levelStr, 10)
noun.connections.set(level, new Set<string>(nounIds as string[]))
}
}
/**
* Get the number of dirty (unpersisted) nodes
* Useful for monitoring and debugging
@ -521,18 +649,12 @@ export class HNSWIndex implements HnswProvider {
// In deferred mode, we track dirty nodes instead of persisting immediately
// This reduces GCS operations from 70 to 2-3 per add() (30-50× faster)
if (this.storage && this.persistMode === 'immediate') {
// IMMEDIATE MODE: Original behavior - persist each neighbor update
const neighborConnectionsObj: Record<string, string[]> = {}
for (const [lvl, nounIds] of neighbor.connections.entries()) {
neighborConnectionsObj[lvl.toString()] = Array.from(nounIds)
}
// IMMEDIATE MODE: Original behavior - persist each neighbor update.
// Goes through the per-node helper so the compressed-blob branch
// fires identically here vs. the deferred-flush path.
neighborUpdates.push({
neighborId,
promise: this.storage.saveHNSWData(neighborId, {
level: neighbor.level,
connections: neighborConnectionsObj
})
promise: this.persistNodeConnections(neighborId, neighbor)
})
} else if (this.persistMode === 'deferred') {
// DEFERRED MODE: Track dirty nodes for later batch persistence
@ -619,16 +741,10 @@ export class HNSWIndex implements HnswProvider {
// Persist HNSW graph data to storage
// Respect persistMode setting
if (this.storage && this.persistMode === 'immediate') {
// IMMEDIATE MODE: Original behavior - persist new entity and system data
const connectionsObj: Record<string, string[]> = {}
for (const [level, nounIds] of noun.connections.entries()) {
connectionsObj[level.toString()] = Array.from(nounIds)
}
await this.storage.saveHNSWData(id, {
level: nounLevel,
connections: connectionsObj
}).catch((error) => {
// IMMEDIATE MODE: Original behavior - persist new entity and system data.
// Goes through the per-node helper so the compressed-blob branch fires
// identically here vs. the deferred-flush + neighbor-update paths.
await this.persistNodeConnections(id, noun).catch((error) => {
console.error(`Failed to persist HNSW data for ${id}:`, error)
})
@ -1073,7 +1189,23 @@ export class HNSWIndex implements HnswProvider {
const cacheKey = `hnsw:vector:${noun.id}`
const vector = await this.unifiedCache.get(cacheKey, async () => {
// Cache miss - load from storage
// Try the mmap-vector backend first when wired — zero-copy slot read,
// no per-entity object hydration.
if (this.vectorBackend) {
const fromMmap = this.vectorBackend.readByUuid(noun.id)
if (fromMmap) {
this.unifiedCache.set(
cacheKey,
fromMmap,
'hnsw',
fromMmap.length * 4,
50
)
return fromMmap
}
}
// Storage fallback — the canonical source of truth.
if (!this.storage) {
throw new Error('Storage not available for vector loading')
}
@ -1083,6 +1215,18 @@ export class HNSWIndex implements HnswProvider {
throw new Error(`Vector not found for noun ${noun.id}`)
}
// Lazy migration: write back into the mmap slot so the next read is
// mmap-fast. Idempotent + resumable; a write failure is non-fatal —
// the field still serves this read, and the next storage hit re-tries
// the migration. (See mmapVectorBackend.ts for the contract.)
if (this.vectorBackend) {
try {
this.vectorBackend.writeByUuid(noun.id, loaded)
} catch (error) {
prodLog.debug(`MmapVectorBackend write-back failed for ${noun.id}`, error)
}
}
// Add to UnifiedCache with cost-aware eviction
// This competes fairly with Graph and Metadata indexes
this.unifiedCache.set(
@ -1140,7 +1284,50 @@ export class HNSWIndex implements HnswProvider {
private async preloadVectors(nodeIds: string[]): Promise<void> {
if (nodeIds.length === 0) return
// Use UnifiedCache's request coalescing to prevent duplicate loads
// mmap-vector backend fast path: batched O(1) slot reads + lazy
// write-back migration on miss. Cache populated from both layers so
// subsequent in-session reads are O(1) memory.
if (this.vectorBackend) {
// Filter out ids already cached so we never re-read.
const uncachedIds: string[] = []
for (const id of nodeIds) {
if (this.unifiedCache.getSync(`hnsw:vector:${id}`) === undefined) {
uncachedIds.push(id)
}
}
if (uncachedIds.length === 0) return
const fromMmap = this.vectorBackend.readBatchByUuid(uncachedIds)
const misses: string[] = []
for (let i = 0; i < uncachedIds.length; i++) {
const id = uncachedIds[i]
const v = fromMmap[i]
if (v) {
this.unifiedCache.set(`hnsw:vector:${id}`, v, 'hnsw', v.length * 4, 50)
} else {
misses.push(id)
}
}
if (misses.length === 0) return
// Storage fallback for the misses, with concurrent lazy write-back.
await Promise.all(misses.map(async (id) => {
if (!this.storage) return
const vector = await this.storage.getNounVector(id)
if (!vector) return
if (this.vectorBackend) {
try {
this.vectorBackend.writeByUuid(id, vector)
} catch (error) {
prodLog.debug(`MmapVectorBackend write-back failed for ${id}`, error)
}
}
this.unifiedCache.set(`hnsw:vector:${id}`, vector, 'hnsw', vector.length * 4, 50)
}))
return
}
// Legacy path: per-entity storage load via UnifiedCache request coalescing.
const promises = nodeIds.map(async (id) => {
const cacheKey = `hnsw:vector:${id}`
return this.unifiedCache.get(cacheKey, async () => {
@ -1339,11 +1526,10 @@ export class HNSWIndex implements HnswProvider {
noun.codebookMax = sq8.max
}
// Restore connections from persisted data
for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
const level = parseInt(levelStr, 10)
noun.connections.set(level, new Set<string>(nounIds as string[]))
}
// Restore connections from persisted data — compressed blob path
// first, legacy JSON-array fallback for indexes written before
// graph link compression landed.
await this.restoreNodeConnections(nounData.id, hnswData, noun)
// Add to in-memory index
this.nouns.set(nounData.id, noun)
@ -1424,11 +1610,10 @@ export class HNSWIndex implements HnswProvider {
noun.codebookMax = sq8.max
}
// Restore connections from persisted data
for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) {
const level = parseInt(levelStr, 10)
noun.connections.set(level, new Set<string>(nounIds as string[]))
}
// Restore connections from persisted data — compressed blob path
// first, legacy JSON-array fallback for indexes written before
// graph link compression landed.
await this.restoreNodeConnections(nounData.id, hnswData, noun)
// Add to in-memory index
this.nouns.set(nounData.id, noun)

174
src/hnsw/mmapVectorBackend.ts Normal file
View file

@ -0,0 +1,174 @@
/**
* @module hnsw/mmapVectorBackend
* @description Disk-resident vector cache backing HNSWIndex, wrapping the
* `vectorStore:mmap` provider (cortex's NativeMmapVectorStore in production).
*
* **Why this exists.** Brainy's per-entity vector storage hits two ceilings as
* datasets grow: (1) every `HNSWIndex.preloadVectors` does a sequential
* `storage.getNounVector` per id (no batch), and (2) cloud and FS storage both
* serialize each vector through JSON / msgpack, blocking zero-copy reads. An
* mmap-backed single-file vector store, addressed by the stable int from
* `EntityIdMapper`, gives O(1) batch reads + OS-page-cache prefetch and is the
* foundation 2.4.0 #2 needs.
*
* **Lazy migration on miss.** First read of an unmigrated UUID falls back to
* `storage.getNounVector` and writes the bytes into the mmap slot before
* returning — idempotent, resumable, and lets an upgraded install converge to
* the fast path under live traffic without a big-bang migration step. The
* per-entity store remains the canonical source of truth; nothing in the per-
* entity layer is rewritten or deleted here.
*
* **Local-FS only in 2.4.0.** Activated when the storage adapter exposes a
* real local path via `getBinaryBlobPath()`. Cloud adapters (S3 / R2 / GCS)
* return null and the backend is not constructed, so HNSWIndex falls back to
* the existing per-entity read path. Chunked vector segments for remote
* storage are the planned 2.4.0 #2 follow-up.
*/
import { mkdir } from 'node:fs/promises'
import { dirname } from 'node:path'
import type { Vector } from '../coreTypes.js'
import type {
EntityIdMapperProvider,
VectorStoreMmapInstance,
VectorStoreMmapProvider
} from '../plugin.js'
/**
* @description Bridge between brainy's UUID-keyed vector API and cortex's
* int-slot mmap vector store. Translates UUIDs to stable int slots via the
* `EntityIdMapper`, grows the file on demand, and handles read misses by
* surfacing `null` (HNSWIndex's caller does the lazy storage fallback +
* write-back). The class never touches per-entity storage directly — it owns
* only the mmap layer.
*/
export class MmapVectorBackend {
private readonly store: VectorStoreMmapInstance
private readonly idMapper: EntityIdMapperProvider
private constructor(store: VectorStoreMmapInstance, idMapper: EntityIdMapperProvider) {
this.store = store
this.idMapper = idMapper
}
/**
* @description Open or create the mmap vector file at `path`. If the file
* does not exist, it is created with the given dim + initial capacity. If
* the directory containing `path` does not exist, it is created recursively
* (the cortex provider does not auto-create parent directories).
* @param provider - The `vectorStore:mmap` provider (cortex
* `NativeMmapVectorStore` in production).
* @param path - Absolute path to the vector file. Derived by the caller
* from `storage.getBinaryBlobPath()` on local-FS adapters; cloud adapters
* return null and this method is not called.
* @param dim - Vector dimensionality. Must match the embedding model.
* @param initialCapacity - Slot capacity to allocate when creating a new
* file. The file grows automatically on writes beyond capacity.
* @param idMapper - Stable UUID↔int mapper (post-2.4.0 #1 semantics — never
* renumbers across rebuild). The mmap slots are addressed by these ints,
* so id stability is the load-bearing invariant of the layout.
* @returns A ready-to-use backend.
*/
static async open(
provider: VectorStoreMmapProvider,
path: string,
dim: number,
initialCapacity: number,
idMapper: EntityIdMapperProvider
): Promise<MmapVectorBackend> {
await mkdir(dirname(path), { recursive: true })
let store: VectorStoreMmapInstance
try {
store = provider.open(path)
} catch {
store = provider.create(path, dim, Math.max(initialCapacity, 16))
}
return new MmapVectorBackend(store, idMapper)
}
/** Vector dimensionality of the underlying mmap file. */
get dim(): number {
return this.store.dim
}
/**
* @description Read a vector by UUID. Returns `null` for any of:
* (1) UUID not yet assigned an int (entity never written through this
* backend or storage), (2) int beyond the highest written slot, or
* (3) the underlying mmap read throwing (corrupt / out-of-bounds).
* The null path is the caller's signal to fall back to per-entity storage
* and trigger a write-back migration.
*/
readByUuid(uuid: string): Vector | null {
const intId = this.idMapper.getInt(uuid)
if (intId === undefined || intId >= this.store.count) return null
try {
return this.store.readVector(intId) as unknown as Vector
} catch {
return null
}
}
/**
* @description Batch read vectors by UUID. Returns an array aligned with
* `uuids` where each entry is the vector or `null` for a miss. Order-
* preserving. Internally collapses to a single batched mmap read of only the
* indices that resolve to a written slot, then redistributes the flat output
* back to the caller's order — so missing slots cost only a couple of map
* lookups, not a full N-element read.
*/
readBatchByUuid(uuids: string[]): (Vector | null)[] {
const indices: number[] = []
const slotByUuid = new Map<string, number>()
for (const uuid of uuids) {
const intId = this.idMapper.getInt(uuid)
if (intId !== undefined && intId < this.store.count) {
slotByUuid.set(uuid, indices.length)
indices.push(intId)
}
}
if (indices.length === 0) return uuids.map(() => null)
const flat = this.store.readVectorsBatch(indices)
const dim = this.store.dim
return uuids.map(uuid => {
const slot = slotByUuid.get(uuid)
if (slot === undefined) return null
return flat.slice(slot * dim, (slot + 1) * dim) as unknown as Vector
})
}
/**
* @description Write a vector at the slot for `uuid`. Assigns a new int if
* the UUID is unseen (stable, append-only). Grows the underlying file (slot
* capacity doubled until the target index fits) before writing past
* capacity, so the caller need not worry about file sizing.
*/
writeByUuid(uuid: string, vector: Vector): void {
const intId = this.idMapper.getOrAssign(uuid)
if (intId >= this.store.capacity) {
let newCap = Math.max(this.store.capacity, 16)
while (intId >= newCap) newCap *= 2
this.store.resize(newCap)
}
this.store.writeVector(intId, Array.from(vector as ArrayLike<number>))
}
/**
* @description madvise(WILLNEED) hint for the slots backing `uuids`. Cheap;
* silently drops UUIDs not yet in the mmap. Used by HNSW search to overlap
* disk I/O with graph traversal.
*/
prefetchByUuid(uuids: string[]): void {
const indices: number[] = []
for (const uuid of uuids) {
const intId = this.idMapper.getInt(uuid)
if (intId !== undefined && intId < this.store.count) indices.push(intId)
}
if (indices.length > 0) this.store.prefetch(indices)
}
/** msync — flush dirty pages to disk. */
flush(): void {
this.store.flush()
}
}

103
src/indexes/columnStore/ColumnStore.ts
View file

@ -125,12 +125,29 @@ export class ColumnStore implements ColumnStoreProvider {
this.manifests.set(fieldName, manifest)
this.fieldTypes.set(fieldName, manifest.valueType)
// Load global deleted bitmap if it exists
// Load global deleted bitmap if it exists. Raw blob preferred
// (2.4.0 #4 cortex-shared format); legacy envelope fallback for
// indexes written before format unification.
try {
let buf: Buffer | null = null
const adapter = storage as unknown as {
loadBinaryBlob?: (key: string) => Promise<Buffer | null>
readObjectFromPath: (path: string) => Promise<any>
}
if (typeof adapter.loadBinaryBlob === 'function') {
const key = `${this.basePath}/${fieldName}/DELETED`
const blob = await adapter.loadBinaryBlob(key)
if (blob && blob.length > 0) buf = blob
}
if (!buf) {
const deletedPath = `${this.basePath}/${fieldName}/DELETED.bin`
const stored = await (storage as any).readObjectFromPath(deletedPath)
const stored = await adapter.readObjectFromPath(deletedPath)
if (stored && stored._binary && stored.data) {
const buf = Buffer.from(stored.data, 'base64')
buf = Buffer.from(stored.data, 'base64')
}
}
if (buf) {
this.deletedEntities.set(fieldName, RoaringBitmap32.deserialize(buf, true))
}
} catch {
@ -408,6 +425,23 @@ export class ColumnStore implements ColumnStoreProvider {
/**
* Flush a single field's tail buffer to a new L0 segment.
*
* Storage path (2.4.0 #4 / cortex-interchange contract):
* When the storage adapter exposes the binary-blob primitive
* (`saveBinaryBlob` — every brainy adapter ≥7.25.0 does), the segment
* bytes are written as a raw blob at the shared cortex key
* `_column_index/<field>/L<level>-NNNNNN` (suffix-free; the adapter
* appends its own). This matches byte-for-byte what cortex's
* `NativeColumnStore` writes, so JS- and native-written indexes
* interchange without re-encoding.
*
* When the adapter doesn't (older custom adapters that didn't follow
* the 7.25.0 primitive surface), we fall back to the legacy
* `writeObjectToPath` envelope — `{ _binary, data: <base64> }` at
* `_column_index/<field>/L<level>-NNNNNN.cidx`. Cortex's 2.3.1
* read-side fallback handles the legacy envelope on its end, so the
* format mismatch is transparent in both directions during the
* transition.
*/
private async flushBuffer(field: string, buffer: ColumnTailBuffer): Promise<void> {
const { values, entityIds, pendingRemovals } = buffer.drain()
@ -416,10 +450,15 @@ export class ColumnStore implements ColumnStoreProvider {
const manifest = this.manifests.get(field)
if (!manifest) return
const storage = this.storage as unknown as {
saveBinaryBlob?: (key: string, data: Buffer) => Promise<void>
writeObjectToPath: (path: string, value: unknown) => Promise<void>
}
const canUseBlob = typeof storage.saveBinaryBlob === 'function'
// Write segment if we have values
if (values.length > 0) {
const segId = manifest.nextSegmentId()
const segPath = manifest.segmentPath(0, segId)
const segBuffer = writeSegmentToBuffer(
{
@ -435,13 +474,22 @@ export class ColumnStore implements ColumnStoreProvider {
entityIds
)
// Store segment as binary data
await (this.storage as any).writeObjectToPath(segPath, {
if (canUseBlob) {
// Raw blob, cortex-shared key convention.
const key = `${this.basePath}/${field}/L0-${String(segId).padStart(6, '0')}`
await storage.saveBinaryBlob!(key, segBuffer)
} else {
// Legacy envelope path for adapters without the binary-blob primitive.
const segPath = manifest.segmentPath(0, segId)
await storage.writeObjectToPath(segPath, {
_binary: true,
data: segBuffer.toString('base64')
})
}
// Register in manifest
// Register in manifest. The `.cidx` file name in the manifest is the
// legacy-format file the cortex 2.3.1 read-side fallback looks for; the
// raw-blob key derives from segId at read time. Both paths converge.
manifest.addSegment({
id: segId,
level: 0,
@ -465,16 +513,22 @@ export class ColumnStore implements ColumnStoreProvider {
for (const id of pendingRemovals) deleted.add(id)
}
// Persist the global deleted bitmap alongside the manifest
// Persist the global deleted bitmap alongside the manifest. Same
// raw-blob preference as segments — shared cortex key `<base>/<field>/DELETED`.
const deleted = this.deletedEntities.get(field)
if (deleted && deleted.size > 0) {
const deletedPath = `${this.basePath}/${field}/DELETED.bin`
const serialized = deleted.serialize(true)
await (this.storage as any).writeObjectToPath(deletedPath, {
if (canUseBlob) {
const deletedKey = `${this.basePath}/${field}/DELETED`
await storage.saveBinaryBlob!(deletedKey, Buffer.from(serialized))
} else {
const deletedPath = `${this.basePath}/${field}/DELETED.bin`
await storage.writeObjectToPath(deletedPath, {
_binary: true,
data: Buffer.from(serialized).toString('base64')
})
}
}
// Save manifest
await manifest.save(this.storage)
@ -508,18 +562,36 @@ export class ColumnStore implements ColumnStoreProvider {
/**
* Load a segment from storage and create a cursor.
*
* Reads the raw-blob layout first (the 2.4.0 #4 shared-with-cortex format),
* then falls back to the legacy `{ _binary, base64 }` envelope at the
* `.cidx` object-path so indexes written before the format unification keep
* loading correctly. Mirror of cortex's 2.3.1 read-side fallback.
*/
private async loadSegmentCursor(field: string, seg: SegmentMeta): Promise<ColumnSegmentCursor | null> {
const manifest = this.manifests.get(field)
if (!manifest) return null
try {
const segPath = manifest.segmentPath(seg.level, seg.id)
const stored = await (this.storage as any).readObjectFromPath(segPath)
if (!stored) return null
let buf: Buffer | null = null
// Handle binary storage format
let buf: Buffer
const storage = this.storage as unknown as {
loadBinaryBlob?: (key: string) => Promise<Buffer | null>
readObjectFromPath: (path: string) => Promise<any>
}
// Preferred: raw blob at the cortex-shared key.
if (typeof storage.loadBinaryBlob === 'function') {
const key = `${this.basePath}/${field}/L${seg.level}-${String(seg.id).padStart(6, '0')}`
const blob = await storage.loadBinaryBlob(key)
if (blob && blob.length > 0) buf = blob
}
// Legacy fallback: `{ _binary, base64 }` envelope at the .cidx object-path.
if (!buf) {
const segPath = manifest.segmentPath(seg.level, seg.id)
const stored = await storage.readObjectFromPath(segPath)
if (!stored) return null
if (stored._binary && stored.data) {
buf = Buffer.from(stored.data, 'base64')
} else if (Buffer.isBuffer(stored)) {
@ -527,6 +599,7 @@ export class ColumnStore implements ColumnStoreProvider {
} else {
return null
}
}
const parsed = readSegmentFromBuffer(buf)
return new ColumnSegmentCursor(

75
src/plugin.ts
View file

@ -253,6 +253,81 @@ export interface CacheProvider {
// uses at the `getProvider('embeddings')` call site. No separate interface is
// added here to avoid a duplicate, unwired contract.
/**
* The `'graph:compression'` provider — pure-function encode/decode for HNSW
* connection lists as compact delta-varint byte sequences (cortex's
* `encodeConnections` / `decodeConnections`).
*
* Brainy's `HNSWIndex` consumes this via a `ConnectionsCodec` that translates
* UUIDs to stable int slots via the `EntityIdMapper`, encodes, and persists
* the compressed bytes through the binary-blob primitive. On load, the blob
* is fetched + decoded back into UUID sets — `setConnectionsCodec()` on
* `HNSWIndex` is the injection point. Read path is dual-format: when no blob
* exists for a node, the connections fall back to the legacy JSON-array path
* embedded in `saveHNSWData`, so pre-2.4.0 indexes keep loading unchanged
* and convergence to the compressed form happens lazily on next save.
*
* Activated only when the storage adapter exposes the binary-blob primitive
* AND the metadata index resolves a stable idMapper. Cloud adapters that
* lack a real local-path resolution still benefit, since the blob primitive
* itself works across every adapter as of brainy 7.25.0.
*/
export interface GraphCompressionProvider {
/** Encode a list of u32 ints to compact delta-varint bytes. Sorts internally. */
encode(ids: number[]): Buffer
/** Decode delta-varint bytes back to a u32 list. */
decode(data: Buffer): number[]
}
/**
* The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed
* vector file. Brainy's HNSWIndex uses the static `.create()` / `.open()`
* factories to open a single-file store at a derived path on disk-resident
* storage adapters; non-FS adapters with no local-path support skip the mmap
* layer and fall back to per-entity vector reads.
*
* Cortex registers `NativeMmapVectorStore` as this provider — a Rust mmap'd
* f32 array with a header, batch read/write APIs, and madvise prefetch. Slot
* indices are the stable ints from `EntityIdMapper`; lazy migration on read
* miss writes back from per-entity storage into the slot, converging an
* upgraded install to the fast path without a big-bang step.
*/
export interface VectorStoreMmapProvider {
/** Create a new vector file with pre-allocated slot capacity. */
create(path: string, dim: number, capacity: number): VectorStoreMmapInstance
/** Open an existing vector file for read + write. */
open(path: string): VectorStoreMmapInstance
/** Open an existing vector file for read-only access. */
openReadOnly(path: string): VectorStoreMmapInstance
}
/**
* An open mmap-vector-store instance — the surface brainy's HNSWIndex consumes
* for batch vector reads + lazy migration from per-entity storage on miss.
*/
export interface VectorStoreMmapInstance {
/** Number of vectors written (highest written index + 1). */
readonly count: number
/** Maximum slot capacity before resize is needed. */
readonly capacity: number
/** Vector dimensionality. */
readonly dim: number
/** Write a single vector at `index`. Vector is f64 in JS, stored as f32. */
writeVector(index: number, vector: number[]): void
/** Write a flat batch starting at `startIndex`. Returns vectors written. */
writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number
/** Read a single vector at `index`. Returns f64[]. */
readVector(index: number): number[]
/** Read multiple vectors by index. Returns flat f64[] of length n × dim. */
readVectorsBatch(indices: number[]): number[]
/** Prefetch via madvise(WILLNEED); no-op on unsupported platforms. */
prefetch(indices: number[]): void
/** Grow the vector file to a larger slot capacity. Re-maps the file. */
resize(newCapacity: number): void
/** msync — flush pending writes to disk. */
flush(): void
}
/**
* Storage adapter factory — plugins register these to provide
* new storage backends that users reference by name.

34
src/utils/entityIdMapper.ts
View file

@ -1,14 +1,34 @@
/**
* EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs for roaring bitmaps
* EntityIdMapper - Bidirectional mapping between UUID strings and integer IDs.
*
* Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings as entity IDs.
* This class provides efficient bidirectional mapping with persistence support.
* Roaring bitmaps require 32-bit unsigned integers, but Brainy uses UUID strings
* as canonical entity IDs. This class provides efficient O(1) bidirectional
* mapping with persistence — and, importantly, a stability guarantee that any
* persisted int-keyed data can rely on.
*
* **Stability guarantee (the foundation 2.4.0 vector-mmap, graph-link-compression,
* and column-store interchange all key off):**
*
* - `getOrAssign(uuid)` is **append-only**: once a UUID is assigned an int, the
* mapping never changes. Subsequent `getOrAssign` calls for the same UUID
* return the same int.
* - `nextId` is **monotonically increasing**. New UUIDs always get an int greater
* than any previously assigned, so a removed-then-re-added UUID is treated as
* a fresh entity (and gets a fresh int — there is no automatic "revive").
* - `remove(uuid)` removes the mapping but does **not** decrement `nextId` or
* recycle the int. The removed int becomes a permanent hole in `intToUuid` —
* downstream consumers seeing `getUuid(int) === undefined` know the entity
* was deleted.
* - A metadata-index `rebuild()` does **not** clear the mapper (the rebuild path
* re-iterates entities via `getOrAssign`, which returns existing ints unchanged).
* Only the explicit `clear()` method renumbers — used by `clearAllIndexData()`
* as the nuclear recovery path with a documented warning.
*
* Features:
* - O(1) lookup in both directions
* - Persistent storage via storage adapter
* - Atomic counter for next ID
* - Serialization/deserialization support
* - O(1) lookup in both directions.
* - Persistent storage via storage adapter.
* - Atomic, monotonic, append-only int counter.
* - Serialization/deserialization support.
*
* @module utils/entityIdMapper
*/

24
src/utils/metadataIndex.ts
View file

@ -2635,13 +2635,20 @@ export class MetadataIndexManager implements MetadataIndexProvider {
this.verbCountsByTypeFixed.fill(0)
this.typeFieldAffinity.clear()
// Clear EntityIdMapper
// Clear EntityIdMapper. This is the explicit destructive path: the caller
// asked for nuclear recovery of a corrupted index, so renumbering UUIDs is
// intentional. Persisted int-keyed data (vector-mmap slots, graph
// link-compression encodings) is invalidated by this op — the warning
// below makes that explicit. Rebuild on its own does NOT clear the mapper.
await this.idMapper.clear()
// Clear chunk manager cache
this.chunkManager.clearCache()
prodLog.info(`✅ Cleared ${deletedCount} field indexes and all in-memory state`)
prodLog.warn('⚠️ EntityIdMapper was cleared — any persisted int-keyed data ' +
'(vector mmap slots, graph link-compression encodings, etc.) is now stale ' +
'and must be rebuilt from canonical sources.')
prodLog.info('⚠️ Run brain.index.rebuild() to recreate the index from entity data')
}
@ -3032,8 +3039,19 @@ export class MetadataIndexManager implements MetadataIndexProvider {
prodLog.info(`Cleared ${existingFields.length} field indexes from storage`)
}
// Clear EntityIdMapper to start fresh
await this.idMapper.clear()
// EntityIdMapper is intentionally NOT cleared here. Rebuild re-iterates
// every entity in storage and calls idMapper.getOrAssign(uuid), which
// returns the existing int for known UUIDs (no renumbering). This is the
// foundational stability guarantee — vector-mmap slot indices, graph
// link-compression encodings, and any other persisted int-keyed data
// remain valid across a rebuild. Previously this line reset nextId to 1
// and renumbered every UUID by re-insertion order, silently breaking
// any consumer that had persisted int-keyed data against the old map.
// Stale entries for UUIDs no longer in storage persist (harmless memory
// overhead); a dedicated prune step can be added if it ever matters.
// The destructive wipe is still available via clearAllIndexData() →
// idMapper.clear(), which is the explicit "recovery" path with the
// appropriate warning about invalidating persisted int-keyed data.
// Clear chunk manager cache
this.chunkManager.clearCache()

137
tests/unit/hnsw/connections-codec.test.ts Normal file
View file

@ -0,0 +1,137 @@
/**
* @module hnsw/connectionsCodec.test
* @description Unit tests for `ConnectionsCodec` — the per-node buffer format
* brainy uses to persist HNSW connection sets as delta-varint-compressed
* binary blobs (2.4.0 #3).
*
* Mocks the `graph:compression` provider so the tests run without cortex
* installed. The real cross-language byte-format integration is exercised
* when cortex's `encodeConnections` / `decodeConnections` are wired in.
*
* Coverage:
* 1. Round-trip a Map of `level → Set<UUID>` through encode + decode and
* verify the result is equal to the input (level-by-level Set equality).
* 2. Empty connections encode to a single-byte buffer (`0` = zero levels) and
* decode back to an empty Map — no `undefined` reads, no decode errors.
* 3. Multiple levels round-trip, including a level with a single UUID and a
* level with many UUIDs (the typical L0 fan-out).
* 4. UUIDs whose ints are no longer in the idMapper at decode time are
* silently dropped — matches the pre-2.4.0 behaviour of a missing
* `getUuid()` lookup in the legacy load path.
*/
import { describe, it, expect } from 'vitest'
import { ConnectionsCodec } from '../../../src/hnsw/connectionsCodec.js'
import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js'
import type { GraphCompressionProvider } from '../../../src/plugin.js'
const stubStorage = {
getMetadata: async () => undefined,
saveMetadata: async () => {},
getNouns: async () => ({ totalCount: 0, items: [] })
} as any
/**
* Mock provider — encodes ints as a JSON byte array, decodes back. NOT the
* delta-varint format cortex produces, but format-agnostic round-trip
* verification is the unit-test concern here.
*/
const mockProvider: GraphCompressionProvider = {
encode(ids: number[]): Buffer {
const sorted = [...ids].sort((a, b) => a - b)
return Buffer.from(JSON.stringify(sorted), 'utf8')
},
decode(data: Buffer): number[] {
if (data.length === 0) return []
return JSON.parse(data.toString('utf8')) as number[]
}
}
async function makeIdMapper(): Promise<EntityIdMapper> {
const mapper = new EntityIdMapper({ storage: stubStorage })
await mapper.init()
return mapper
}
function setsEqual<T>(a: Set<T>, b: Set<T>): boolean {
if (a.size !== b.size) return false
for (const v of a) if (!b.has(v)) return false
return true
}
describe('ConnectionsCodec (2.4.0 #3 — wraps graph:compression provider)', () => {
it('round-trips a single-level Set of UUIDs through encode + decode', async () => {
const idMapper = await makeIdMapper()
const codec = new ConnectionsCodec(mockProvider, idMapper)
const input = new Map<number, Set<string>>([
[0, new Set(['a', 'b', 'c', 'd'])]
])
const buf = codec.encode(input)
const out = codec.decode(buf)
expect(out.size).toBe(1)
expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true)
})
it('empty connections encode to a one-byte buffer and decode to an empty Map', async () => {
const idMapper = await makeIdMapper()
const codec = new ConnectionsCodec(mockProvider, idMapper)
const buf = codec.encode(new Map())
expect(buf.length).toBe(1)
expect(buf.readUInt8(0)).toBe(0)
expect(codec.decode(buf).size).toBe(0)
})
it('round-trips multiple levels with very different fan-outs', async () => {
const idMapper = await makeIdMapper()
const codec = new ConnectionsCodec(mockProvider, idMapper)
const manyUuids = Array.from({ length: 30 }, (_, i) => `u-${i}`)
const input = new Map<number, Set<string>>([
[0, new Set(manyUuids)],
[1, new Set(['high-1', 'high-2'])],
[2, new Set(['solo'])]
])
const buf = codec.encode(input)
const out = codec.decode(buf)
expect(out.size).toBe(3)
expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true)
expect(setsEqual(out.get(1)!, input.get(1)!)).toBe(true)
expect(setsEqual(out.get(2)!, input.get(2)!)).toBe(true)
})
it('drops UUIDs whose ints the idMapper no longer recognises on decode', async () => {
const encodingMapper = await makeIdMapper()
const encodeCodec = new ConnectionsCodec(mockProvider, encodingMapper)
// Encode through one mapper.
const input = new Map<number, Set<string>>([[0, new Set(['present', 'gone'])]])
const buf = encodeCodec.encode(input)
// Decode through a FRESH mapper that only knows about 'present'.
const decodingMapper = await makeIdMapper()
decodingMapper.getOrAssign('present')
// Manually re-map 'present' to the same int it had on encode so the
// present UUID round-trips; 'gone' simply has no mapping → silent drop.
// We line up the int spaces by asserting equality below: only 'present'
// survives, 'gone' is missing — no errors, no `undefined` Set entries.
const decodeCodec = new ConnectionsCodec(mockProvider, decodingMapper)
// Inject the same int for 'present' so it round-trips one-for-one. (The
// encoding mapper assigns ints in insertion order: 'present'=1, 'gone'=2.
// The decoding mapper's first getOrAssign also returns 1, so 'present'
// matches; 'gone' (encoded as 2) has no mapping in the decoder.)
expect(encodingMapper.getInt('present')).toBe(1)
expect(decodingMapper.getInt('present')).toBe(1)
const out = decodeCodec.decode(buf)
const l0 = out.get(0)!
expect(l0.has('present')).toBe(true)
expect(l0.has('gone')).toBe(false)
expect(l0.size).toBe(1)
})
})

242
tests/unit/hnsw/mmap-vector-backend.test.ts Normal file
View file

@ -0,0 +1,242 @@
/**
* @module hnsw/mmapVectorBackend.test
* @description Unit tests for the `MmapVectorBackend` bridge — the brainy-side
* wrapper that translates UUID-keyed vector reads/writes into stable int slot
* ops against an `vectorStore:mmap` provider.
*
* Mocks the provider so the tests run without cortex installed (cortex is a
* downstream consumer of brainy, not a dev dep). The real integration with
* cortex's `NativeMmapVectorStore` is exercised when cortex 2.4.0 picks up
* this brainy release and re-runs its cross-language parity suite.
*
* Coverage:
* 1. Open-then-write-then-read round-trips for a single vector.
* 2. Batch reads return an array aligned with the input UUIDs, with `null`
* entries for misses interleaved among hits — order preserved.
* 3. Slot assignment is stable across multiple writes for the same UUID
* (no re-slot, no overwrite of an adjacent slot).
* 4. Writes beyond the initial capacity grow the file (doubling) without
* losing the vectors already written.
* 5. `readByUuid` returns `null` for both unknown UUIDs and UUIDs in the map
* whose slot has not yet been written.
* 6. Open is idempotent — opening an already-existing file reuses it.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { tmpdir } from 'node:os'
import { mkdtemp, rm } from 'node:fs/promises'
import { join } from 'node:path'
import { MmapVectorBackend } from '../../../src/hnsw/mmapVectorBackend.js'
import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js'
import type {
VectorStoreMmapInstance,
VectorStoreMmapProvider
} from '../../../src/plugin.js'
/**
* Minimal storage stub for the EntityIdMapper. The mapper only touches storage
* in init/flush; for these tests the mapper starts empty and is never flushed.
*/
const stubStorage = {
getMetadata: async () => undefined,
saveMetadata: async () => {},
getNouns: async () => ({ totalCount: 0, items: [] })
} as any
/**
* Pure in-memory mmap store. Mirrors cortex's NativeMmapVectorStore surface
* just closely enough to exercise the backend's contract — no real mmap, no
* file I/O, no f32 round-trip narrowing (the precision check is out of scope
* here; that's covered by cortex's parity suite).
*/
class MockMmapStore implements VectorStoreMmapInstance {
private readonly vectors: Array<number[] | undefined> = []
private highestWritten = -1
constructor(
public readonly dim: number,
private _capacity: number
) {}
get count(): number {
return this.highestWritten + 1
}
get capacity(): number {
return this._capacity
}
writeVector(index: number, vector: number[]): void {
if (index >= this._capacity) {
throw new Error(`Slot ${index} >= capacity ${this._capacity}`)
}
if (vector.length !== this.dim) {
throw new Error(`Dim mismatch: expected ${this.dim}, got ${vector.length}`)
}
this.vectors[index] = [...vector]
if (index > this.highestWritten) this.highestWritten = index
}
writeVectorsBatch(startIndex: number, vectorsFlat: number[]): number {
if (vectorsFlat.length % this.dim !== 0) {
throw new Error('vectorsFlat length not a multiple of dim')
}
const n = vectorsFlat.length / this.dim
for (let i = 0; i < n; i++) {
this.writeVector(startIndex + i, vectorsFlat.slice(i * this.dim, (i + 1) * this.dim))
}
return n
}
readVector(index: number): number[] {
const v = this.vectors[index]
if (!v) throw new Error(`Slot ${index} not written`)
return [...v]
}
readVectorsBatch(indices: number[]): number[] {
const flat: number[] = []
for (const i of indices) {
const v = this.vectors[i]
if (!v) throw new Error(`Slot ${i} not written`)
for (let k = 0; k < this.dim; k++) flat.push(v[k])
}
return flat
}
prefetch(_indices: number[]): void {
/* no-op in mock */
}
resize(newCapacity: number): void {
if (newCapacity < this._capacity) throw new Error('Cannot shrink')
this._capacity = newCapacity
}
flush(): void {
/* no-op in mock */
}
}
/**
* Mock provider — keeps one MockMmapStore per path. open() throws if the path
* doesn't exist yet (matches cortex semantics: open() is for existing files
* only); create() throws if it does (cortex doesn't, but the brainy backend
* does open-first-then-create, so the throw path is exercised).
*/
class MockMmapProvider implements VectorStoreMmapProvider {
private files = new Map<string, MockMmapStore>()
create(path: string, dim: number, capacity: number): VectorStoreMmapInstance {
if (this.files.has(path)) throw new Error(`File exists at ${path}`)
const store = new MockMmapStore(dim, capacity)
this.files.set(path, store)
return store
}
open(path: string): VectorStoreMmapInstance {
const store = this.files.get(path)
if (!store) throw new Error(`No file at ${path}`)
return store
}
openReadOnly(path: string): VectorStoreMmapInstance {
return this.open(path)
}
/** Test helper — peek at the underlying store. */
_peek(path: string): MockMmapStore | undefined {
return this.files.get(path)
}
}
describe('MmapVectorBackend (2.4.0 #2 — wraps vectorStore:mmap provider)', () => {
let dir: string
let path: string
let idMapper: EntityIdMapper
let provider: MockMmapProvider
beforeEach(async () => {
dir = await mkdtemp(join(tmpdir(), 'brainy-mmap-vec-'))
path = join(dir, 'vectors.bin')
idMapper = new EntityIdMapper({ storage: stubStorage })
await idMapper.init()
provider = new MockMmapProvider()
})
afterEach(async () => {
await rm(dir, { recursive: true, force: true }).catch(() => {})
})
it('open creates a new file when none exists, then round-trips a vector by UUID', async () => {
const backend = await MmapVectorBackend.open(provider, path, 4, 16, idMapper)
expect(backend.dim).toBe(4)
backend.writeByUuid('alpha', [1, 2, 3, 4])
expect(backend.readByUuid('alpha')).toEqual([1, 2, 3, 4])
})
it('reads return null for unknown UUIDs and for UUIDs in the map but not yet written', async () => {
const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper)
// Unknown — never seen by the mapper.
expect(backend.readByUuid('ghost')).toBeNull()
// Known to the mapper but no slot ever written. We assign through the
// mapper directly so the backend itself has not touched the slot.
idMapper.getOrAssign('reserved')
expect(backend.readByUuid('reserved')).toBeNull()
})
it('batch read returns an array aligned to the input UUIDs (nulls preserved in place)', async () => {
const backend = await MmapVectorBackend.open(provider, path, 3, 16, idMapper)
backend.writeByUuid('a', [1, 1, 1])
backend.writeByUuid('b', [2, 2, 2])
backend.writeByUuid('c', [3, 3, 3])
// Interleave hits + a never-seen UUID + a duplicate hit. Order preserved.
const result = backend.readBatchByUuid(['b', 'missing', 'a', 'c', 'missing-too', 'b'])
expect(result).toEqual([
[2, 2, 2],
null,
[1, 1, 1],
[3, 3, 3],
null,
[2, 2, 2]
])
})
it('writes for the same UUID land in the same slot (stable id, no re-slotting)', async () => {
const backend = await MmapVectorBackend.open(provider, path, 2, 8, idMapper)
backend.writeByUuid('persistent', [1, 1])
const slotAfterFirstWrite = idMapper.getInt('persistent')
backend.writeByUuid('persistent', [9, 9]) // overwrite the same slot
expect(idMapper.getInt('persistent')).toBe(slotAfterFirstWrite)
expect(backend.readByUuid('persistent')).toEqual([9, 9])
// Another UUID gets a different slot, and the first vector is undisturbed.
backend.writeByUuid('other', [5, 5])
expect(idMapper.getInt('other')).not.toBe(slotAfterFirstWrite)
expect(backend.readByUuid('persistent')).toEqual([9, 9])
expect(backend.readByUuid('other')).toEqual([5, 5])
})
it('grows the file (doubling) when a write lands beyond capacity, without losing prior data', async () => {
// Start at the smallest sane initial capacity (clamped to 16 by the backend).
const backend = await MmapVectorBackend.open(provider, path, 2, 1, idMapper)
const store = provider._peek(path)!
expect(store.capacity).toBe(16) // backend floor
// Write 20 vectors so capacity must double at least once (16 → 32).
const uuids: string[] = []
for (let i = 0; i < 20; i++) {
const uuid = `u-${i}`
uuids.push(uuid)
backend.writeByUuid(uuid, [i, i * 2])
}
expect(store.capacity).toBeGreaterThanOrEqual(32)
// Every vector survived the growth — no slot got overwritten or lost.
for (let i = 0; i < uuids.length; i++) {
expect(backend.readByUuid(uuids[i])).toEqual([i, i * 2])
}
})
it('opens an existing file idempotently when the second open hits the same path', async () => {
// First open creates.
const backend1 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper)
backend1.writeByUuid('persisted', [7, 7])
// Second open against the same path. The provider's create() throws on
// collision; the backend's open-first behaviour means we reuse the file.
const backend2 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper)
expect(backend2.readByUuid('persisted')).toEqual([7, 7])
})
})

191
tests/unit/indexes/columnStore/column-store-interchange.test.ts Normal file
View file

@ -0,0 +1,191 @@
/**
* @module column-store-interchange.test
* @description 2.4.0 #4 — column-store JS↔native interchange. Pins down that
* brainy's JS `ColumnStore` writes segments + DELETED bitmaps via the binary-
* blob primitive at the cortex-shared key convention, AND that indexes
* persisted under the pre-2.4.0 envelope (`{ _binary, base64 }` at the
* `.cidx` object-path) still load correctly via the legacy-read fallback.
*
* Together these properties make the JS and native engines byte-compatible
* at the segment payload, so writing through one and reading through the
* other is a no-op format-wise — closing the gap the cortex 2.3.1 read-side
* fallback opened on the cortex side.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { ColumnStore } from '../../../../src/indexes/columnStore/ColumnStore.js'
import { MemoryStorage } from '../../../../src/storage/adapters/memoryStorage.js'
import { EntityIdMapper } from '../../../../src/utils/entityIdMapper.js'
import { writeSegmentToBuffer } from '../../../../src/indexes/columnStore/ColumnSegmentFormat.js'
import { ColumnManifest } from '../../../../src/indexes/columnStore/ColumnManifest.js'
import { RoaringBitmap32 } from '../../../../src/utils/roaring/index.js'
describe('ColumnStore — JS↔native interchange (2.4.0 #4)', () => {
let storage: MemoryStorage
let idMapper: EntityIdMapper
let store: ColumnStore
beforeEach(async () => {
storage = new MemoryStorage()
await storage.init()
idMapper = new EntityIdMapper({ storage, storageKey: 'test:idMapper' })
await idMapper.init()
store = new ColumnStore({ flushThreshold: 10 })
await store.init(storage, idMapper)
})
afterEach(async () => {
await store.close()
})
// -----------------------------------------------------------------------
// WRITE side: raw blob at the cortex-shared key, no legacy envelope
// -----------------------------------------------------------------------
it('flushBuffer writes segment bytes as a raw blob at the cortex-shared key', async () => {
for (let i = 0; i < 5; i++) {
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 })
}
await store.flush()
// The raw blob exists at the suffix-free cortex key.
const key = '_column_index/score/L0-000001'
const raw = await (storage as any).loadBinaryBlob(key)
expect(Buffer.isBuffer(raw)).toBe(true)
expect(raw!.length).toBeGreaterThan(0)
// The legacy envelope at the `.cidx` object-path is NOT written (no
// 33% base64 bloat, no dual-write).
const legacyPath = '_column_index/score/L0-000001.cidx'
const legacyEnvelope = await (storage as any).readObjectFromPath(legacyPath)
expect(legacyEnvelope).toBeFalsy()
})
it('flushBuffer writes the DELETED bitmap as a raw blob too (not an envelope)', async () => {
for (let i = 0; i < 3; i++) {
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i })
}
store.removeEntity(idMapper.getInt('u-1')!)
await store.flush()
const rawDel = await (storage as any).loadBinaryBlob('_column_index/score/DELETED')
expect(Buffer.isBuffer(rawDel)).toBe(true)
expect(rawDel!.length).toBeGreaterThan(0)
const legacyEnvelope = await (storage as any).readObjectFromPath(
'_column_index/score/DELETED.bin'
)
expect(legacyEnvelope).toBeFalsy()
})
// -----------------------------------------------------------------------
// READ side: pre-2.4.0 legacy envelope still loads via the fallback
// -----------------------------------------------------------------------
it('loads a segment persisted as the pre-2.4.0 envelope (legacy fallback)', async () => {
// Seed a legacy-format index DIRECTLY on storage — the on-disk shape
// brainy 7.20.0..7.25.0 wrote before this commit.
const legacyStore = new ColumnStore()
await legacyStore.init(storage, idMapper)
// Build a segment buffer using the shared cidx format.
const values: number[] = [10, 20, 30]
const entityIds: number[] = [
idMapper.getOrAssign('legacy-a'),
idMapper.getOrAssign('legacy-b'),
idMapper.getOrAssign('legacy-c')
]
const segBuffer = writeSegmentToBuffer(
{
fieldName: 'score',
fieldNameLength: 5,
valueType: 0,
level: 0,
codec: 0,
flags: 0,
count: values.length
},
values,
entityIds
)
// Persist via the LEGACY envelope path (writeObjectToPath, base64 wrap).
await (storage as any).writeObjectToPath('_column_index/score/L0-000001.cidx', {
_binary: true,
data: segBuffer.toString('base64')
})
const manifest = new ColumnManifest('score', '_column_index')
manifest.valueType = 0
manifest.multiValue = false
manifest.addSegment({
id: 1,
level: 0,
count: values.length,
minValue: values[0],
maxValue: values[values.length - 1],
file: 'L0-000001.cidx'
})
await manifest.save(storage)
await legacyStore.close()
// NEW ColumnStore re-discovers the manifest at init, then the legacy
// segment loads via the readObjectFromPath fallback (no raw blob exists
// at the cortex key, so it falls through). sortTopK returns the entities
// in the order the segment recorded — proves the bytes round-tripped.
const readStore = new ColumnStore()
await readStore.init(storage, idMapper)
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
expect(sortedUuids).toEqual(['legacy-a', 'legacy-b', 'legacy-c'])
await readStore.close()
})
it('loads a DELETED bitmap persisted as the pre-2.4.0 envelope (legacy fallback)', async () => {
// Manually seed a legacy DELETED.bin envelope, then re-init and confirm
// the entity it marks gone is excluded from filter/sort results.
const bm = new RoaringBitmap32()
bm.add(idMapper.getOrAssign('dead'))
const serialized = Buffer.from(bm.serialize(true))
await (storage as any).writeObjectToPath('_column_index/score/DELETED.bin', {
_binary: true,
data: serialized.toString('base64')
})
// Add a couple of live entities so a manifest exists (init only loads
// DELETED for fields with a manifest on disk).
store.addEntity(idMapper.getOrAssign('alive-1'), { score: 100 })
store.addEntity(idMapper.getOrAssign('alive-2'), { score: 200 })
await store.flush()
await store.close()
const readStore = new ColumnStore()
await readStore.init(storage, idMapper)
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
// The dead entity must NOT appear; the alive ones must.
expect(sortedUuids).not.toContain('dead')
expect(sortedUuids).toContain('alive-1')
expect(sortedUuids).toContain('alive-2')
await readStore.close()
})
// -----------------------------------------------------------------------
// ROUND-TRIP: write-then-read in the new format
// -----------------------------------------------------------------------
it('write (new format) → re-init → read returns the same data', async () => {
for (let i = 0; i < 5; i++) {
store.addEntity(idMapper.getOrAssign(`u-${i}`), { score: i * 10 })
}
await store.flush()
await store.close()
const readStore = new ColumnStore()
await readStore.init(storage, idMapper)
const sortedInts = await readStore.sortTopK('score', 'asc', 10)
const sortedUuids = sortedInts.map(i => idMapper.getUuid(i))
expect(sortedUuids).toEqual(['u-0', 'u-1', 'u-2', 'u-3', 'u-4'])
await readStore.close()
})
})

161
tests/unit/utils/entity-id-mapper-stability.test.ts Normal file
View file

@ -0,0 +1,161 @@
/**
* Regression test: EntityIdMapper stability across rebuild.
*
* The foundation 2.4.0 (vector mmap store, graph link compression, column-store
* JS↔native interchange) all key off UUID→int mappings that **must not change**
* across a metadata-index rebuild. Previously `metadataIndex.rebuild()` called
* `idMapper.clear()` which reset `nextId` to 1 and renumbered every UUID by
* re-insertion order, silently invalidating any consumer that had persisted
* int-keyed data against the old map.
*
* This test pins down the stability contract:
*
* 1. UUID→int mappings persist across a single rebuild.
* 2. Mappings persist across many consecutive rebuilds.
* 3. New entities added after rebuild get fresh ints greater than any prior
* assignment — no collisions with existing UUIDs' ints.
* 4. Removed entities leave a permanent hole — new entities don't recycle the
* gap, even across a rebuild.
* 5. `clearAllIndexData()` is the explicit, intentional nuclear path — it DOES
* renumber. This is the only documented way to invalidate the int space, and
* a warning is logged so consumers know persisted int-keyed data is now stale.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../../src/brainy.js'
const DIM = 384
const makeVec = (seed = 1) =>
new Float32Array(DIM).map((_, i) => ((i + seed) % DIM) / DIM)
describe('EntityIdMapper stability (foundation for 2.4.0)', () => {
let brain: Brainy
beforeEach(async () => {
brain = new Brainy({ storage: { type: 'memory' }, silent: true })
await brain.init()
})
afterEach(async () => {
await brain.close()
})
async function addEntity(name: string, seed: number): Promise<string> {
return brain.add({
data: name,
vector: makeVec(seed),
type: 'thing' as any,
metadata: { name }
})
}
function getInt(uuid: string): number | undefined {
return (brain as any).metadataIndex.idMapper.getInt(uuid)
}
async function rebuild(): Promise<void> {
await (brain as any).metadataIndex.rebuild()
}
it('UUID→int mappings persist across a single metadata-index rebuild', async () => {
const ids = [
await addEntity('a', 1),
await addEntity('b', 2),
await addEntity('c', 3),
await addEntity('d', 4),
await addEntity('e', 5)
]
const before = ids.map(id => getInt(id))
expect(before.every(i => typeof i === 'number' && (i as number) > 0)).toBe(true)
await rebuild()
const after = ids.map(id => getInt(id))
expect(after).toEqual(before)
})
it('mappings stay byte-for-byte stable across many consecutive rebuilds', async () => {
const ids = [
await addEntity('a', 1),
await addEntity('b', 2),
await addEntity('c', 3)
]
const before = ids.map(id => getInt(id))
for (let i = 0; i < 5; i++) {
await rebuild()
const after = ids.map(id => getInt(id))
expect(after).toEqual(before)
}
})
it('entities added after rebuild get fresh monotonic ints (no collision with existing)', async () => {
const priorIds = [
await addEntity('a', 1),
await addEntity('b', 2),
await addEntity('c', 3)
]
const priorInts = priorIds.map(id => getInt(id) as number)
const maxPrior = Math.max(...priorInts)
await rebuild()
const newId = await addEntity('d', 4)
const newInt = getInt(newId) as number
expect(newInt).toBeGreaterThan(maxPrior)
// Prior entities' ints didn't drift.
expect(priorIds.map(id => getInt(id))).toEqual(priorInts)
})
it('removed entities leave a permanent hole — new entities never recycle the gap', async () => {
const ids = [
await addEntity('a', 1),
await addEntity('b', 2),
await addEntity('c', 3),
await addEntity('d', 4),
await addEntity('e', 5)
]
const beforeInts = ids.map(id => getInt(id) as number)
const deletedId = ids[2]
const deletedInt = beforeInts[2]
const maxBefore = Math.max(...beforeInts)
await brain.delete(deletedId)
expect(getInt(deletedId)).toBeUndefined()
const newId = await addEntity('f', 6)
const newInt = getInt(newId) as number
expect(newInt).not.toBe(deletedInt)
expect(newInt).toBeGreaterThan(maxBefore)
// Surviving ids keep their ints across the deletion + the add.
const survivors = ids.filter((_, i) => i !== 2)
const survivorIntsBefore = beforeInts.filter((_, i) => i !== 2)
expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore)
// Survivors' ints also survive a rebuild after the delete.
await rebuild()
expect(survivors.map(id => getInt(id))).toEqual(survivorIntsBefore)
// The deleted id is still gone after rebuild (no resurrection).
expect(getInt(deletedId)).toBeUndefined()
})
it('clearAllIndexData() is the explicit nuclear path that DOES renumber', async () => {
const id1 = await addEntity('a', 1)
const id2 = await addEntity('b', 2)
const priorInts = [getInt(id1) as number, getInt(id2) as number]
expect(priorInts.every(i => i >= 1)).toBe(true)
// Nuclear recovery: explicit destructive op. The warning logged here is
// the only documented way to invalidate the canonical int space.
await (brain as any).metadataIndex.clearAllIndexData()
// Both UUIDs are gone from the mapper.
expect(getInt(id1)).toBeUndefined()
expect(getInt(id2)).toBeUndefined()
// The int counter restarted from 1: the next add() gets int 1.
const idAfter = await addEntity('c', 3)
expect(getInt(idAfter)).toBe(1)
})
})