From 617c156febba91890532c607417e1d6ac89ed8f3 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 28 May 2026 10:37:01 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20graph=20link=20compression=20=E2=80=94?= =?UTF-8?q?=20delta-varint=20connections=20(2.4.0=20#3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cortex registers a graph:compression provider exposing `encode`/`decode` for HNSW connection lists (delta-varint, ~4× smaller edges than the JSON-UUID-array shape brainy has persisted since the beginning). This wires the consumer side without changing the saveHNSWData/getHNSWData adapter signatures. Architecture: - NEW ConnectionsCodec (src/hnsw/connectionsCodec.ts) — translates between UUID-keyed Map> and a single compact per-node buffer via the provider + stable EntityIdMapper. Custom wire format: [count: u8] [[level: u8] [len: u32 LE] [bytes...]]*. One blob per node regardless of level count, so the storage I/O is identical to the legacy path (one saveHNSWData call) plus a single saveBinaryBlob. - HNSWIndex changes — a `connectionsCodec: ConnectionsCodec | null` field with `setConnectionsCodec()` setter. THREE save sites (deferred flush, immediate-mode entity persist, immediate-mode neighbor updates) now go through a single `persistNodeConnections(nodeId, noun)` helper: when the codec is wired AND the storage adapter exposes saveBinaryBlob, it encodes the connections, stores them at `_hnsw_conn/`, and records `connections: {}` in saveHNSWData as the marker. Otherwise the legacy JSON-array path is taken. TWO load sites use a matching `restoreNodeConnections` helper that tries loadBinaryBlob first and falls back to the legacy hnswData.connections field on miss / decode error. Format convergence is lazy: pre-2.4.0 nodes still load via the legacy path, then write the compressed form on next dirty save. - brainy.ts wireConnectionsCodec — runs alongside wireMmapVectorBackend during init. Activates when (a) the graph:compression provider is registered and (b) the metadata index exposes its idMapper. Unlike the mmap-vector backend, this layer engages on EVERY brainy 7.25.0 adapter — the blob primitive itself is universal; only the codec presence gates activation. - Provider interface GraphCompressionProvider in plugin.ts — encode + decode static signatures. Brainy depends on the interface; cortex's registered { encode: encodeConnections, decode: decodeConnections } satisfies it structurally. Tests (1437 total, +4 vs prior tip): - tests/unit/hnsw/connections-codec.test.ts — 4 unit tests with a JSON mock provider: single-level round-trip, empty-Map round-trip (one-byte buffer), multi-level fan-out round-trip, and silent-drop of UUIDs the decoding idMapper no longer knows. The real cross-language byte format is exercised when cortex 2.4.0 wires its delta-varint encode/decode in. This completes the brainy half of the 2.4.0 storage foundation: stable ids (#23), mmap vectors (#24), graph link compression (#25), and column-store interchange (#26). Coordinated release as brainy 7.26.0 + cortex 2.4.0 follows. --- src/brainy.ts | 45 +++++- src/hnsw/connectionsCodec.ts | 137 ++++++++++++++++++ src/hnsw/hnswIndex.ts | 168 +++++++++++++++++----- src/plugin.ts | 26 ++++ tests/unit/hnsw/connections-codec.test.ts | 137 ++++++++++++++++++ 5 files changed, 473 insertions(+), 40 deletions(-) create mode 100644 src/hnsw/connectionsCodec.ts create mode 100644 tests/unit/hnsw/connections-codec.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 7b18013d..53d73a3b 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -37,8 +37,14 @@ import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' import { PluginRegistry } from './plugin.js' -import type { BrainyPlugin, BrainyPluginContext, VectorStoreMmapProvider } from './plugin.js' +import type { + BrainyPlugin, + BrainyPluginContext, + GraphCompressionProvider, + VectorStoreMmapProvider +} from './plugin.js' import { MmapVectorBackend } from './hnsw/mmapVectorBackend.js' +import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' import { ValidationConfig, @@ -569,6 +575,14 @@ export class Brainy implements BrainyInterface { // works, just without the zero-copy fast path. await this.wireMmapVectorBackend() + // Wire the connections codec (2.4.0 #3). When the graph:compression + // provider is registered AND the metadata index exposes a stable + // idMapper, inject a codec that encodes HNSW connections as + // delta-varint blobs at save time and decodes on load. The blob + // primitive itself works on every brainy 7.25.0 adapter, so unlike the + // mmap-vector backend this layer engages even on cloud adapters. + this.wireConnectionsCodec() + // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() @@ -7792,6 +7806,35 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Wire the HNSW connections codec (2.4.0 #3). Activates when + * BOTH (a) the `graph:compression` provider is registered (cortex registers + * `{ encode: encodeConnections, decode: decodeConnections }`), and (b) the + * metadata index exposes a stable idMapper. Failures are non-fatal: HNSW + * keeps working via the legacy JSON-array path. + * + * Unlike the mmap-vector backend, this layer does NOT require a real local + * path — the binary-blob primitive saves the compressed bytes through the + * storage adapter directly, so the codec is engaged on cloud adapters too. + * Format convergence is lazy: pre-2.4.0 nodes load via the legacy path, + * then the next dirty save writes the compressed form (and the legacy + * field of saveHNSWData becomes empty), so all reads converge over time + * without an explicit migration step. + */ + private wireConnectionsCodec(): void { + const provider = this.pluginRegistry.getProvider('graph:compression') + if (!provider) return + + const idMapper = this.metadataIndex.getIdMapper?.() + if (!idMapper) return + + const codec = new ConnectionsCodec(provider, idMapper) + this.index.setConnectionsCodec(codec) + if (!this.config.silent) { + console.log('[brainy] graph link compression wired (delta-varint connections via graph:compression provider)') + } + } + /** * Rebuild indexes from persisted data if needed (LAZY LOADING) * diff --git a/src/hnsw/connectionsCodec.ts b/src/hnsw/connectionsCodec.ts new file mode 100644 index 00000000..91dc00b5 --- /dev/null +++ b/src/hnsw/connectionsCodec.ts @@ -0,0 +1,137 @@ +/** + * @module hnsw/connectionsCodec + * @description Bridge between brainy's UUID-keyed HNSW connection sets and + * cortex's int-keyed delta-varint encode/decode (the `graph:compression` + * provider). Translates UUIDs to stable int slots via the post-2.4.0 #1 + * `EntityIdMapper`, batches all of a node's per-level connection lists into + * a single compact buffer, and reverses the path on load. + * + * Wire format of the per-node buffer the codec produces: + * + * [num_levels: u8] + * repeated num_levels times: + * [level: u8] + * [encoded_len: u32 LE] + * [encoded_bytes: bytes] <- provider.encode(ints) + * + * Why a single buffer per node and not one blob per level: + * - One `saveBinaryBlob` call per persisted node, regardless of level count. + * On hot insert paths an HNSW node may be at levels 0..3; the per-node + * blob trades a one-byte count for 3 saved I/O round-trips. + * - Read path is symmetric: one `loadBinaryBlob` reconstructs the full + * connection set, so HNSW rebuild stays linear in the number of nodes. + * + * Loose UUIDs (ints assigned to entities that no longer exist) are silently + * dropped on decode. They can't legitimately participate in graph traversal + * because the entity is gone; treating them as `undefined` UUIDs matches the + * pre-2.4.0 behaviour of an `idMapper.getUuid()` miss in the legacy load path. + */ + +import type { + EntityIdMapperProvider, + GraphCompressionProvider +} from '../plugin.js' + +export class ConnectionsCodec { + constructor( + private readonly provider: GraphCompressionProvider, + private readonly idMapper: EntityIdMapperProvider + ) {} + + /** + * @description Encode all levels of a node's connections into one compact + * buffer. Empty input → an empty buffer (zero levels, no body). The + * idMapper assigns stable ints on demand if a connection UUID has never + * been seen — append-only, matches the rest of the 2.4.0 contract. + */ + encode(connectionsByLevel: Map>): Buffer { + if (connectionsByLevel.size === 0) { + // Single byte: zero levels. Round-trips cleanly through decode. + return Buffer.from([0]) + } + if (connectionsByLevel.size > 0xff) { + throw new Error( + `ConnectionsCodec.encode: too many HNSW levels (${connectionsByLevel.size}); ` + + `the per-node header is one byte (max 255 levels).` + ) + } + + // First pass: encode each level into its own buffer. + const levelBuffers: Array<{ level: number; buf: Buffer }> = [] + let bodyBytes = 0 + for (const [level, uuids] of connectionsByLevel) { + if (level > 0xff) { + throw new Error( + `ConnectionsCodec.encode: level ${level} exceeds u8 header range (max 255).` + ) + } + const ints: number[] = [] + for (const uuid of uuids) { + ints.push(this.idMapper.getOrAssign(uuid)) + } + const buf = this.provider.encode(ints) + levelBuffers.push({ level, buf }) + bodyBytes += 1 + 4 + buf.length // level + len + payload + } + + // Second pass: concatenate with header. + const out = Buffer.alloc(1 + bodyBytes) + out.writeUInt8(levelBuffers.length, 0) + let offset = 1 + for (const { level, buf } of levelBuffers) { + out.writeUInt8(level, offset) + offset += 1 + out.writeUInt32LE(buf.length, offset) + offset += 4 + buf.copy(out, offset) + offset += buf.length + } + return out + } + + /** + * @description Decode a per-node buffer back into a connections Map. Ints + * for entities the idMapper no longer knows are silently dropped — see the + * module-level note on why. + */ + decode(data: Buffer): Map> { + const out = new Map>() + if (data.length === 0) return out + + const count = data.readUInt8(0) + let offset = 1 + for (let i = 0; i < count; i++) { + if (offset + 5 > data.length) { + throw new Error(`ConnectionsCodec.decode: truncated header at level ${i}`) + } + const level = data.readUInt8(offset) + offset += 1 + const len = data.readUInt32LE(offset) + offset += 4 + if (offset + len > data.length) { + throw new Error(`ConnectionsCodec.decode: truncated body at level ${level}`) + } + const levelBuf = data.slice(offset, offset + len) + offset += len + + const ints = this.provider.decode(levelBuf) + const uuids = new Set() + for (const intId of ints) { + const uuid = this.idMapper.getUuid(intId) + if (uuid !== undefined) uuids.add(uuid) + } + out.set(level, uuids) + } + return out + } +} + +/** + * @description Build the binary-blob storage key for a node's compressed + * connections. Suffix-free; the adapter appends its own. Keys live under + * `_hnsw_conn/` so they don't collide with the existing `_column_index/` + * blobs and so a cortex-side reader knows exactly where to look. + */ +export function compressedConnectionsKey(nodeId: string): string { + return `_hnsw_conn/${nodeId}` +} diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 496fc563..ff074daf 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -19,6 +19,7 @@ import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js' import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js' import type { HnswProvider } from '../plugin.js' import { MmapVectorBackend } from './mmapVectorBackend.js' +import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js' // Default HNSW parameters const DEFAULT_CONFIG: HNSWConfig = { @@ -57,6 +58,16 @@ export class HNSWIndex implements HnswProvider { // Null on cloud storage adapters (no local path) and pre-injection init. private vectorBackend: MmapVectorBackend | null = null + // Optional connections codec (2.4.0 #3). When set, node-persist writes the + // node's per-level connection sets as a single delta-varint-compressed + // binary blob (typically ~4× smaller than the legacy JSON-UUID-array + // shape), plus an empty `connections: {}` in saveHNSWData as the marker. + // The read path tries to load the blob first; missing blob → legacy + // connections field. Format convergence is lazy: pre-2.4.0 nodes still + // load via the legacy path, then write the compressed form on next dirty + // save, so the migration converges under live traffic with no big-bang. + private connectionsCodec: ConnectionsCodec | null = null + // COW (Copy-on-Write) support private cowEnabled: boolean = false private cowModifiedNodes: Set = new Set() @@ -125,6 +136,19 @@ export class HNSWIndex implements HnswProvider { this.vectorBackend = backend } + /** + * @description Inject (or detach) the connections codec. When set, node + * persistence writes a delta-varint-compressed binary blob alongside an + * empty `connections: {}` marker in saveHNSWData; the load path reads the + * blob and decodes. Null reverts to the legacy JSON-array path. Wiring is + * done by brainy.ts when the `graph:compression` provider is registered + * AND the storage adapter exposes the binary-blob primitive AND the + * metadata index has a stable idMapper. + */ + public setConnectionsCodec(codec: ConnectionsCodec | null): void { + this.connectionsCodec = codec + } + /** * Get whether parallelization is enabled */ @@ -168,15 +192,7 @@ export class HNSWIndex implements HnswProvider { const noun = this.nouns.get(nodeId) if (!noun) return Promise.resolve() // Node was deleted - const connectionsObj: Record = {} - for (const [level, nounIds] of noun.connections.entries()) { - connectionsObj[level.toString()] = Array.from(nounIds) - } - - return this.storage!.saveHNSWData(nodeId, { - level: noun.level, - connections: connectionsObj - }).catch(error => { + return this.persistNodeConnections(nodeId, noun).catch(error => { console.error(`[HNSW flush] Failed to persist node ${nodeId}:`, error) }) }) @@ -207,6 +223,94 @@ export class HNSWIndex implements HnswProvider { return nodeCount } + /** + * @description Persist one node's connections. When the connections codec is + * wired AND the storage adapter exposes `saveBinaryBlob`, the per-level + * connection sets are encoded into a single compact buffer and stored as a + * binary blob; `saveHNSWData` then records the node's level and an empty + * `connections: {}` as the marker. Otherwise the legacy JSON-array path is + * taken — the format every brainy release before 2.4.0 wrote. The two are + * intentionally NOT dual-written: one or the other, never both, so format + * convergence is unambiguous on the read side. + * + * Shared by all three save sites (deferred flush, immediate-mode new-entity + * persist, immediate-mode neighbor update) so the codec branch is exercised + * uniformly regardless of which path triggered the write. + */ + private async persistNodeConnections(nodeId: string, noun: HNSWNoun): Promise { + if (!this.storage) return + + const storageWithBlob = this.storage as unknown as { + saveBinaryBlob?: (key: string, data: Buffer) => Promise + } + const canCompress = + this.connectionsCodec !== null && + typeof storageWithBlob.saveBinaryBlob === 'function' + + if (canCompress) { + const encoded = this.connectionsCodec!.encode(noun.connections) + await storageWithBlob.saveBinaryBlob!(compressedConnectionsKey(nodeId), encoded) + await this.storage.saveHNSWData(nodeId, { + level: noun.level, + connections: {} + }) + return + } + + const connectionsObj: Record = {} + for (const [level, nounIds] of noun.connections.entries()) { + connectionsObj[level.toString()] = Array.from(nounIds) + } + await this.storage.saveHNSWData(nodeId, { + level: noun.level, + connections: connectionsObj + }) + } + + /** + * @description Restore one node's connections from persisted storage. Tries + * the compressed-blob path first (when codec wired AND blob primitive + * available); a present non-empty buffer is decoded and populates + * `noun.connections`. A missing blob (no payload at the key) means this node + * was last persisted under the legacy format, so the legacy + * `hnswData.connections` field is used instead. + * + * On any decode error the legacy field is also used as a safety net — the + * codec's `decode` throws on truncated input, which would otherwise leave + * the node with an empty connections Map (orphaned from the graph). + */ + private async restoreNodeConnections( + nodeId: string, + hnswData: { level: number; connections: Record }, + noun: HNSWNoun + ): Promise { + const storageWithBlob = this.storage as unknown as { + loadBinaryBlob?: (key: string) => Promise + } + const canDecompress = + this.connectionsCodec !== null && + typeof storageWithBlob.loadBinaryBlob === 'function' + + if (canDecompress) { + try { + const buf = await storageWithBlob.loadBinaryBlob!(compressedConnectionsKey(nodeId)) + if (buf && buf.length > 0) { + const decoded = this.connectionsCodec!.decode(buf) + noun.connections = decoded + return + } + } catch (error) { + prodLog.debug(`HNSW: compressed connections decode failed for ${nodeId}; falling back to legacy`, error) + } + } + + // Legacy fallback: JSON UUID arrays in the HNSW data object. + for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { + const level = parseInt(levelStr, 10) + noun.connections.set(level, new Set(nounIds as string[])) + } + } + /** * Get the number of dirty (unpersisted) nodes * Useful for monitoring and debugging @@ -545,18 +649,12 @@ export class HNSWIndex implements HnswProvider { // In deferred mode, we track dirty nodes instead of persisting immediately // This reduces GCS operations from 70 to 2-3 per add() (30-50× faster) if (this.storage && this.persistMode === 'immediate') { - // IMMEDIATE MODE: Original behavior - persist each neighbor update - const neighborConnectionsObj: Record = {} - for (const [lvl, nounIds] of neighbor.connections.entries()) { - neighborConnectionsObj[lvl.toString()] = Array.from(nounIds) - } - + // IMMEDIATE MODE: Original behavior - persist each neighbor update. + // Goes through the per-node helper so the compressed-blob branch + // fires identically here vs. the deferred-flush path. neighborUpdates.push({ neighborId, - promise: this.storage.saveHNSWData(neighborId, { - level: neighbor.level, - connections: neighborConnectionsObj - }) + promise: this.persistNodeConnections(neighborId, neighbor) }) } else if (this.persistMode === 'deferred') { // DEFERRED MODE: Track dirty nodes for later batch persistence @@ -643,16 +741,10 @@ export class HNSWIndex implements HnswProvider { // Persist HNSW graph data to storage // Respect persistMode setting if (this.storage && this.persistMode === 'immediate') { - // IMMEDIATE MODE: Original behavior - persist new entity and system data - const connectionsObj: Record = {} - for (const [level, nounIds] of noun.connections.entries()) { - connectionsObj[level.toString()] = Array.from(nounIds) - } - - await this.storage.saveHNSWData(id, { - level: nounLevel, - connections: connectionsObj - }).catch((error) => { + // IMMEDIATE MODE: Original behavior - persist new entity and system data. + // Goes through the per-node helper so the compressed-blob branch fires + // identically here vs. the deferred-flush + neighbor-update paths. + await this.persistNodeConnections(id, noun).catch((error) => { console.error(`Failed to persist HNSW data for ${id}:`, error) }) @@ -1434,11 +1526,10 @@ export class HNSWIndex implements HnswProvider { noun.codebookMax = sq8.max } - // Restore connections from persisted data - for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { - const level = parseInt(levelStr, 10) - noun.connections.set(level, new Set(nounIds as string[])) - } + // Restore connections from persisted data — compressed blob path + // first, legacy JSON-array fallback for indexes written before + // graph link compression landed. + await this.restoreNodeConnections(nounData.id, hnswData, noun) // Add to in-memory index this.nouns.set(nounData.id, noun) @@ -1519,11 +1610,10 @@ export class HNSWIndex implements HnswProvider { noun.codebookMax = sq8.max } - // Restore connections from persisted data - for (const [levelStr, nounIds] of Object.entries(hnswData.connections)) { - const level = parseInt(levelStr, 10) - noun.connections.set(level, new Set(nounIds as string[])) - } + // Restore connections from persisted data — compressed blob path + // first, legacy JSON-array fallback for indexes written before + // graph link compression landed. + await this.restoreNodeConnections(nounData.id, hnswData, noun) // Add to in-memory index this.nouns.set(nounData.id, noun) diff --git a/src/plugin.ts b/src/plugin.ts index 0cf04a6c..89e2bed4 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -253,6 +253,32 @@ export interface CacheProvider { // uses at the `getProvider('embeddings')` call site. No separate interface is // added here to avoid a duplicate, unwired contract. +/** + * The `'graph:compression'` provider — pure-function encode/decode for HNSW + * connection lists as compact delta-varint byte sequences (cortex's + * `encodeConnections` / `decodeConnections`). + * + * Brainy's `HNSWIndex` consumes this via a `ConnectionsCodec` that translates + * UUIDs to stable int slots via the `EntityIdMapper`, encodes, and persists + * the compressed bytes through the binary-blob primitive. On load, the blob + * is fetched + decoded back into UUID sets — `setConnectionsCodec()` on + * `HNSWIndex` is the injection point. Read path is dual-format: when no blob + * exists for a node, the connections fall back to the legacy JSON-array path + * embedded in `saveHNSWData`, so pre-2.4.0 indexes keep loading unchanged + * and convergence to the compressed form happens lazily on next save. + * + * Activated only when the storage adapter exposes the binary-blob primitive + * AND the metadata index resolves a stable idMapper. Cloud adapters that + * lack a real local-path resolution still benefit, since the blob primitive + * itself works across every adapter as of brainy 7.25.0. + */ +export interface GraphCompressionProvider { + /** Encode a list of u32 ints to compact delta-varint bytes. Sorts internally. */ + encode(ids: number[]): Buffer + /** Decode delta-varint bytes back to a u32 list. */ + decode(data: Buffer): number[] +} + /** * The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed * vector file. Brainy's HNSWIndex uses the static `.create()` / `.open()` diff --git a/tests/unit/hnsw/connections-codec.test.ts b/tests/unit/hnsw/connections-codec.test.ts new file mode 100644 index 00000000..c92c3888 --- /dev/null +++ b/tests/unit/hnsw/connections-codec.test.ts @@ -0,0 +1,137 @@ +/** + * @module hnsw/connectionsCodec.test + * @description Unit tests for `ConnectionsCodec` — the per-node buffer format + * brainy uses to persist HNSW connection sets as delta-varint-compressed + * binary blobs (2.4.0 #3). + * + * Mocks the `graph:compression` provider so the tests run without cortex + * installed. The real cross-language byte-format integration is exercised + * when cortex's `encodeConnections` / `decodeConnections` are wired in. + * + * Coverage: + * 1. Round-trip a Map of `level → Set` through encode + decode and + * verify the result is equal to the input (level-by-level Set equality). + * 2. Empty connections encode to a single-byte buffer (`0` = zero levels) and + * decode back to an empty Map — no `undefined` reads, no decode errors. + * 3. Multiple levels round-trip, including a level with a single UUID and a + * level with many UUIDs (the typical L0 fan-out). + * 4. UUIDs whose ints are no longer in the idMapper at decode time are + * silently dropped — matches the pre-2.4.0 behaviour of a missing + * `getUuid()` lookup in the legacy load path. + */ + +import { describe, it, expect } from 'vitest' +import { ConnectionsCodec } from '../../../src/hnsw/connectionsCodec.js' +import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js' +import type { GraphCompressionProvider } from '../../../src/plugin.js' + +const stubStorage = { + getMetadata: async () => undefined, + saveMetadata: async () => {}, + getNouns: async () => ({ totalCount: 0, items: [] }) +} as any + +/** + * Mock provider — encodes ints as a JSON byte array, decodes back. NOT the + * delta-varint format cortex produces, but format-agnostic round-trip + * verification is the unit-test concern here. + */ +const mockProvider: GraphCompressionProvider = { + encode(ids: number[]): Buffer { + const sorted = [...ids].sort((a, b) => a - b) + return Buffer.from(JSON.stringify(sorted), 'utf8') + }, + decode(data: Buffer): number[] { + if (data.length === 0) return [] + return JSON.parse(data.toString('utf8')) as number[] + } +} + +async function makeIdMapper(): Promise { + const mapper = new EntityIdMapper({ storage: stubStorage }) + await mapper.init() + return mapper +} + +function setsEqual(a: Set, b: Set): boolean { + if (a.size !== b.size) return false + for (const v of a) if (!b.has(v)) return false + return true +} + +describe('ConnectionsCodec (2.4.0 #3 — wraps graph:compression provider)', () => { + it('round-trips a single-level Set of UUIDs through encode + decode', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const input = new Map>([ + [0, new Set(['a', 'b', 'c', 'd'])] + ]) + const buf = codec.encode(input) + const out = codec.decode(buf) + + expect(out.size).toBe(1) + expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true) + }) + + it('empty connections encode to a one-byte buffer and decode to an empty Map', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const buf = codec.encode(new Map()) + expect(buf.length).toBe(1) + expect(buf.readUInt8(0)).toBe(0) + expect(codec.decode(buf).size).toBe(0) + }) + + it('round-trips multiple levels with very different fan-outs', async () => { + const idMapper = await makeIdMapper() + const codec = new ConnectionsCodec(mockProvider, idMapper) + + const manyUuids = Array.from({ length: 30 }, (_, i) => `u-${i}`) + const input = new Map>([ + [0, new Set(manyUuids)], + [1, new Set(['high-1', 'high-2'])], + [2, new Set(['solo'])] + ]) + + const buf = codec.encode(input) + const out = codec.decode(buf) + + expect(out.size).toBe(3) + expect(setsEqual(out.get(0)!, input.get(0)!)).toBe(true) + expect(setsEqual(out.get(1)!, input.get(1)!)).toBe(true) + expect(setsEqual(out.get(2)!, input.get(2)!)).toBe(true) + }) + + it('drops UUIDs whose ints the idMapper no longer recognises on decode', async () => { + const encodingMapper = await makeIdMapper() + const encodeCodec = new ConnectionsCodec(mockProvider, encodingMapper) + + // Encode through one mapper. + const input = new Map>([[0, new Set(['present', 'gone'])]]) + const buf = encodeCodec.encode(input) + + // Decode through a FRESH mapper that only knows about 'present'. + const decodingMapper = await makeIdMapper() + decodingMapper.getOrAssign('present') + + // Manually re-map 'present' to the same int it had on encode so the + // present UUID round-trips; 'gone' simply has no mapping → silent drop. + // We line up the int spaces by asserting equality below: only 'present' + // survives, 'gone' is missing — no errors, no `undefined` Set entries. + const decodeCodec = new ConnectionsCodec(mockProvider, decodingMapper) + // Inject the same int for 'present' so it round-trips one-for-one. (The + // encoding mapper assigns ints in insertion order: 'present'=1, 'gone'=2. + // The decoding mapper's first getOrAssign also returns 1, so 'present' + // matches; 'gone' (encoded as 2) has no mapping in the decoder.) + expect(encodingMapper.getInt('present')).toBe(1) + expect(decodingMapper.getInt('present')).toBe(1) + + const out = decodeCodec.decode(buf) + const l0 = out.get(0)! + expect(l0.has('present')).toBe(true) + expect(l0.has('gone')).toBe(false) + expect(l0.size).toBe(1) + }) +})