From eade6ff1bed1a1e40656061484929092718487e0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 10 Jun 2026 09:29:08 -0700 Subject: [PATCH] fix: mmap-vector backend capacity NaN at the provider FFI boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production deployment reported "[brainy] mmap-vector backend not wired (Capacity must be > 0); falling back to per-entity vector reads" on every cold start of populated brains under the native plugin — degrading vector recall to per-entity disk reads (8.3s cold starts at 685 entities, scaling linearly). Root cause: wireMmapVectorBackend computes its initial slot capacity as idMapper.size * 2. When the metadata index is provider-backed, getIdMapper() returns a façade exposing getInt/getOrAssign/getUuid but NO `size` property. `undefined * 2` is NaN, Math.max(NaN, 1024) stays NaN, and NaN coerces to 0 via ToUint32 at the provider's u32 FFI boundary — tripping the provider's "Capacity must be > 0" guard. The JS-only path never hit this because brainy's own EntityIdMapper has a real `size` getter. Fix, two independent layers: - wireMmapVectorBackend treats a missing/non-finite mapper size as 0, so the 1024-slot floor always holds (the file grows on demand past it). - MmapVectorBackend.open sanitizes the capacity (NaN/Infinity/non-positive → 16-slot floor) so no caller can ever hand the provider an invalid allocation size. Regression tests mimic the exact failure: a U32-coercing provider that rejects capacity 0 plus an idMapper façade without `size`. Pre-fix the NaN reached create() and threw; post-fix the backend wires with the floor capacity and round-trips vectors. 1464/1464 unit suite passing. --- src/brainy.ts | 10 +++- src/hnsw/mmapVectorBackend.ts | 8 +++- tests/unit/hnsw/mmap-vector-backend.test.ts | 52 +++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/brainy.ts b/src/brainy.ts index d6bd7e39..e04df2c4 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -9410,7 +9410,15 @@ export class Brainy implements BrainyInterface { if (!idMapper) return const dim = this.dimensions ?? 384 - const initialCapacity = Math.max(idMapper.size * 2, 1024) + // Provider-supplied id mappers (e.g. a native metadata index's façade) may + // not expose a `size` property. `undefined * 2` is NaN, and NaN coerces to + // 0 at the napi u32 boundary — the provider then rejects the create() with + // "Capacity must be > 0". Treat a missing/non-finite size as 0 so the + // 1024 floor always holds; the file grows on demand past it. + const mapperSize = Number.isFinite((idMapper as { size?: number }).size) + ? (idMapper as { size: number }).size + : 0 + const initialCapacity = Math.max(mapperSize * 2, 1024) try { const backend = await MmapVectorBackend.open( diff --git a/src/hnsw/mmapVectorBackend.ts b/src/hnsw/mmapVectorBackend.ts index fa9d4e37..cafcee4f 100644 --- a/src/hnsw/mmapVectorBackend.ts +++ b/src/hnsw/mmapVectorBackend.ts @@ -77,11 +77,17 @@ export class MmapVectorBackend { idMapper: EntityIdMapperProvider ): Promise { await mkdir(dirname(path), { recursive: true }) + // NaN/Infinity/non-positive capacities coerce to 0 at the provider's u32 + // FFI boundary ("Capacity must be > 0"). Sanitize here so no caller can + // ever hand the provider an invalid allocation size. + const capacity = Number.isFinite(initialCapacity) && initialCapacity >= 16 + ? Math.ceil(initialCapacity) + : 16 let store: VectorStoreMmapInstance try { store = provider.open(path) } catch { - store = provider.create(path, dim, Math.max(initialCapacity, 16)) + store = provider.create(path, dim, capacity) } return new MmapVectorBackend(store, idMapper) } diff --git a/tests/unit/hnsw/mmap-vector-backend.test.ts b/tests/unit/hnsw/mmap-vector-backend.test.ts index 257d50c4..5179ac9d 100644 --- a/tests/unit/hnsw/mmap-vector-backend.test.ts +++ b/tests/unit/hnsw/mmap-vector-backend.test.ts @@ -239,4 +239,56 @@ describe('MmapVectorBackend (2.4.0 #2 — wraps vectorStore:mmap provider)', () const backend2 = await MmapVectorBackend.open(provider, path, 2, 8, idMapper) expect(backend2.readByUuid('persisted')).toEqual([7, 7]) }) + + describe('capacity sanitization at the provider FFI boundary (7.31.3 regression)', () => { + /** + * Mimics cortex's NativeMmapVectorStore napi boundary: JS numbers are + * coerced ToUint32 (NaN → 0, Infinity → 0) before the Rust-side + * `capacity == 0` guard fires with "Capacity must be > 0". This is the + * exact production failure from a metadata-provider idMapper façade that + * exposes no `size` property: `undefined * 2` = NaN at the caller. + */ + class U32CoercingProvider extends MockMmapProvider { + lastCreateCapacity: number | null = null + override create(path: string, dim: number, capacity: number): VectorStoreMmapInstance { + const coerced = capacity >>> 0 // ECMA ToUint32 — what napi-rs does + this.lastCreateCapacity = coerced + if (coerced === 0) throw new Error('Capacity must be > 0') + return super.create(path, dim, coerced) + } + } + + it('NaN initialCapacity (idMapper façade without `size`) still wires with the floor capacity', async () => { + const strictProvider = new U32CoercingProvider() + const facadeWithoutSize = { + getInt: (uuid: string) => idMapper.getInt(uuid), + getOrAssign: (uuid: string) => idMapper.getOrAssign(uuid), + getUuid: (intId: number) => idMapper.getUuid(intId) + } as any + + // Pre-fix: NaN reached create(), coerced to 0, threw, and the backend + // never wired. Post-fix it must construct with the 16-slot floor. + const naiveCapacity = (facadeWithoutSize.size as number) * 2 // NaN + const backend = await MmapVectorBackend.open( + strictProvider, + path, + 4, + naiveCapacity, + facadeWithoutSize + ) + expect(strictProvider.lastCreateCapacity).toBeGreaterThanOrEqual(16) + + backend.writeByUuid('wired', [1, 2, 3, 4]) + expect(backend.readByUuid('wired')).toEqual([1, 2, 3, 4]) + }) + + it('Infinity and non-positive capacities are clamped to the floor', async () => { + for (const bad of [Infinity, -Infinity, 0, -5]) { + const strictProvider = new U32CoercingProvider() + const p = join(dir, `vectors-${String(bad)}.bin`) + await MmapVectorBackend.open(strictProvider, p, 2, bad, idMapper) + expect(strictProvider.lastCreateCapacity).toBe(16) + } + }) + }) })