From 0de7665930ac23fb48cff0c0034f4fb475527727 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 13:53:09 -0700 Subject: [PATCH] fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The engine-pair seam law: a zero-norm vector never crosses an engine boundary. The index belt already refused to insert one, but the canonical write and the vectored-noun ledger still counted it, so a near-empty store whose only vectored row was zero-norm read "1 canonical vectored vs 0 indexed" and threw a not-ready error at open, and a store's own legacy zero-norm VFS root could trip the same gate before its VFS-init-time cure ever ran. - add()/update() (single and transact()) now normalize an explicit real all-zero vector to the unvectored [] shape before the dimension pin, the ledger flag, and the index ops ever see it (loud, one warn per write, canonical write still succeeds). - The legacy counts.json derivation walk (scanVectoredNounCount) excludes a persisted zero-norm row, matching the live ledger's definition. - A legacy zero-norm VFS root now migrates at open, before the vector-leg gate evaluates, via one O(1) fixed-path read (torn-tolerant — skips rather than aborting init on a torn root, letting the recovery walk heal it) — independent of whether a VirtualFileSystem is ever constructed this session. - update({ id, vector: [] }) (and the same op inside transact()) is now the sanctioned, idempotent unvector door: index removal, exactly-once ledger decrement, no re-embed, and it clears a pending deferred-embed marker rather than leaving it to re-vectorize the row later. The combination with deferEmbedding is a typed refusal. - JsHnswVectorIndex.rebuild() now skips a zero-norm/empty persisted vector when repopulating from canonical (the same belt the live add/replace paths already had), and health()'s index-parity check now compares HNSW size against the vectored-noun ledger rather than the raw metadata-entry count, since a store's VFS root is permanently unvectored by design. Co-Authored-By: Claude Fable 5 --- src/brainy.ts | 351 +++++++++++++-- src/hnsw/hnswIndex.ts | 29 +- src/storage/adapters/fileSystemStorage.ts | 30 +- src/utils/paramValidation.ts | 23 +- tests/integration/vfs-root-zero-norm.test.ts | 31 +- .../zero-norm-unvector-door.test.ts | 399 ++++++++++++++++++ 6 files changed, 810 insertions(+), 53 deletions(-) create mode 100644 tests/integration/zero-norm-unvector-door.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index bfac8c08..ed958a67 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -396,6 +396,15 @@ interface PlannedTransact { * marker outlives its write. */ markerRecords: FactMarkerRecord[] + /** + * Ids the batch's `{ op: 'update' }` unvector door (`vector: []`) needs to + * decrement on the vectored-noun ledger — consumed by `transact()` with a + * proper `await this.storage.noteVectorUnlanded?.(id)` per id, AFTER + * `commitTransaction` resolves (never for a rejected batch). Kept separate + * from `postCommit` (`Array<() => void>`, called synchronously, fire-and- + * forget) because the ledger hook is async and must be awaited. + */ + vectorUnlands: string[] } /** @@ -1504,6 +1513,15 @@ export class Brainy implements BrainyInterface { }).backfillBlobHistoryRefCountsIfNeeded() } + // LEG C (zero-norm/unvector-door law): migrate a legacy zero-norm VFS + // root BEFORE the vector-leg open gate below ever compares the + // canonical vectored-noun count against the vector index's size — see + // migrateLegacyZeroNormVfsRootIfNeeded's JSDoc for why this is a safe + // O(1) exception to "nothing at open may scale with brain size", and + // why it must run here rather than waiting on VirtualFileSystem's own + // (VFS-instance-gated) lazy migration. + await this.migrateLegacyZeroNormVfsRootIfNeeded() + // Rebuild indexes if needed for existing data. Runs to completion before // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true @@ -1518,8 +1536,9 @@ export class Brainy implements BrainyInterface { // cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph // index construction, the eager cold-load, id-resolver + connections- // codec wiring, crash-recovery index rebuild, the replay-gap check, - // legacy VFS blob adoption, blob-history backfill, and the - // rebuildIndexesIfNeeded() gate + migration check. + // legacy VFS blob adoption, blob-history backfill, the legacy + // zero-norm VFS root migration, and the rebuildIndexesIfNeeded() gate + // + migration check. markPhase('index-init-gate') // Register shutdown hooks for graceful count flushing (once globally) @@ -2912,10 +2931,34 @@ export class Brainy implements BrainyInterface { // vector shape, is structurally impossible). The background worker // embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed + let vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) + // THE ZERO-NORM LAW (canonical write side): a zero-norm vector is not a + // vector — it never crosses an engine boundary (the engine pair's seam + // law). This engine's own cosine distance treats an all-zero vector + // safely (a zero-norm operand always scores MAXIMUM distance — see + // isZeroNormVector's JSDoc), but a downstream engine serving squared- + // euclidean distance cannot tell it apart from a legitimate origin + // point — a false attractor that silently darkened 150+ rows in a + // production deployment. The index belt (AddToVectorIndexOperation) + // already refuses to INDEX a zero-norm vector, but until now the + // CANONICAL write still persisted it and the vectored-noun ledger + // counted it — so a near-empty store whose only vectored row was + // zero-norm read "canonical vectored > 0, index size 0" and threw a + // not-ready error at open. Normalize HERE, before the dimension pin, + // the vectored-ledger flag (`SaveNounMetadataOperation`'s `hasVector`), + // and the index ops below ever see it, so it persists as the sanctioned + // "unvectored" `[]` shape instead — the canonical write still succeeds. + if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { + prodLog.warn( + `[Brainy] add(): entity ${id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + vector = [] + } + // Ensure dimensions are set (a deferred-embed stub carries no dimension // information — the worker's real vector goes through the same guard). // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose @@ -3623,25 +3666,53 @@ export class Brainy implements BrainyInterface { // often the host writes. const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged + + // THE ZERO-NORM LAW (canonical write side) — see add()'s matching + // comment: an explicit REAL all-zero vector is not a vector. Normalize + // to the sanctioned "unvectored" `[]` shape BEFORE the dimension + // check, the unvector-door decision below, and the index ops ever see + // it — a local copy; `params.vector` itself is never mutated. + let explicitVector = params.vector + if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { + prodLog.warn( + `[Brainy] update(): entity ${params.id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + explicitVector = [] + } + + // THE SANCTIONED UNVECTOR DOOR: `explicitVector` at length 0 (an + // explicit `vector: []`, or a real all-zero vector just normalized + // above) is an instruction to remove the vector NOW — never "please + // embed". `validateUpdateParams` already refuses combining it with + // `deferEmbedding: true` (an empty array is truthy, so that guard + // fires unconditionally on any explicit `vector`). Idempotent on an + // already-unvectored row: the ledger decrement near the end of this + // method is gated on the PRIOR vector actually having been real. + const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 + // MT5 deferred re-embedding: the OLD vector keeps serving semantic // search — stale-but-present, never absent (the flicker law) — until // the background worker embeds the new data and swaps it atomically. const deferringEmbed = - params.deferEmbedding === true && hasNewData && !params.vector - if (params.vector) { - if (this.dimensions && params.vector.length !== this.dimensions) { + params.deferEmbedding === true && hasNewData && !explicitVector + if (explicitVector) { + // A length-0 explicit vector (the unvector door) carries no + // dimension information — exempt from the check, mirroring add()'s + // own `vector.length > 0` gate on the dimension pin. + if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` ) } - vector = params.vector + vector = explicitVector } else if (hasNewData && !deferringEmbed) { vector = await this.embed(params.data) } // A deferred data change does NOT reindex now (the vector is unchanged; // the worker's atomic swap carries the real reindex later). const needsReindexing = Boolean( - (hasNewData && !deferringEmbed) || params.type || params.vector + (hasNewData && !deferringEmbed) || params.type || explicitVector ) // Always update the noun with new metadata @@ -3735,6 +3806,22 @@ export class Brainy implements BrainyInterface { ? [this.enqueuePendingEmbed(params.id)] : undefined + // Leg D — the unvector door clears a PENDING deferred-embed marker: + // without this, the worker would later embed this row's current data + // and silently re-vector it, defeating the caller's explicit "remove + // the vector now" instruction. The clear rides THIS SAME commit fact + // (an `embed.landed` record with an empty vector — the recovery fold + // disarms a pending marker on ANY `embed.landed` for the id, + // regardless of the vector it carries), so a crash between the write + // and the in-memory clear below still recovers disarmed. Mutually + // exclusive with `embedMarkers` above: `deferringEmbed` requires an + // ABSENT `explicitVector`, so the two branches never both apply. + const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) + const commitRecords: FactMarkerRecord[] | undefined = + embedMarkers ?? (clearsPendingEmbed + ? [{ type: 'embed.landed', id: params.id, vector: [] }] + : undefined) + // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { @@ -3823,7 +3910,33 @@ export class Brainy implements BrainyInterface { } } ] - : undefined, embedMarkers) + : undefined, commitRecords) + + // Leg D continued — the in-memory pending-embed clear runs only AFTER + // the commit above actually succeeded (an aborted update must not + // disarm a marker whose durable `embed.landed` twin was never + // written). + if (clearsPendingEmbed) { + this.clearPendingEmbed(params.id) + prodLog.warn( + `[Brainy] update(): entity ${params.id} had a pending deferred embed — ` + + `the unvector door cleared it ('vector: []' is an explicit instruction, ` + + `never "please embed").` + ) + } + + // Leg D — vectored-ledger decrement for the sanctioned unvector door. + // update()'s own metadata write goes through UpdateNounMetadataOperation + // (isNew=false), so the saveNounMetadata(..., hasVector) seam never + // fires here — noteVectorUnlanded is the ONLY seam, the same + // sanctioned hook unvectorNounForRootMigration() uses. Gated on the + // PRIOR vector having actually been real (non-empty, non-zero-norm): + // an already-unvectored row's second call is a true no-op — no + // decrement, matching the ledger-exactness law (never double-count, + // never drift negative). + if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { + await this.storage.noteVectorUnlanded?.(params.id) + } // Aggregation hook (outside transaction — derived data). `existing` is // the full get() view — every reserved field top-level — and must be @@ -9179,6 +9292,15 @@ export class Brainy implements BrainyInterface { hook() } + // Leg D — vectored-ledger decrements for this batch's unvector-door + // updates (see planTxUpdate's matching comment), applied after the + // commit point and properly awaited (unlike `postCommit`'s synchronous + // fire-and-forget hooks) — each is the same sanctioned hook + // unvectorNounForRootMigration() uses. + for (const id of plan.vectorUnlands) { + await this.storage.noteVectorUnlanded?.(id) + } + // Change feed: the batch's events share its single committed generation. // A rejected batch throws at commitTransaction and never reaches here. this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) @@ -10422,7 +10544,8 @@ export class Brainy implements BrainyInterface { casUpdates: [], createdNouns: new Set(), changeEvents: [], - markerRecords: [] + markerRecords: [], + vectorUnlands: [] } for (const op of ops) { @@ -10544,9 +10667,22 @@ export class Brainy implements BrainyInterface { // marker-less committed row would be a silently missing vector, which is // the disallowed direction). The background worker embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed + let vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) + + // THE ZERO-NORM LAW — see the single-add() insert path's matching + // comment (a zero-norm vector is not a vector; never crosses an engine + // boundary). Normalized here BEFORE the dimension pin and the + // vectored-ledger `hasVector` flag below ever see it. + if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { + prodLog.warn( + `[Brainy] transact add: entity ${id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + vector = [] + } + // Gated on `vector.length > 0` — see the single-add() insert path's // matching comment: an explicit `vector: []` carries no dimension // information either, deferred or not. @@ -10717,17 +10853,62 @@ export class Brainy implements BrainyInterface { const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged let vector = existing.vector - if (params.vector) { - if (this.dimensions && params.vector.length !== this.dimensions) { + + // THE ZERO-NORM LAW + THE SANCTIONED UNVECTOR DOOR — transact() mirror + // of update()'s matching block: an explicit REAL all-zero vector + // normalizes to `[]` (never crosses an engine boundary), and an + // explicit `vector: []` (post-normalization) is the sanctioned unvector + // instruction, exempt from the dimension check. `validateUpdateParams` + // already refuses combining it with `deferEmbedding: true`. + let explicitVector = params.vector + if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { + prodLog.warn( + `[Brainy] transact update: entity ${params.id} was given an explicit all-zero ` + + `vector — a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + explicitVector = [] + } + const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 + + if (explicitVector) { + if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` ) } - vector = params.vector + vector = explicitVector } else if (hasNewData) { vector = await this.embed(params.data) } - const needsReindexing = Boolean(hasNewData || params.type || params.vector) + const needsReindexing = Boolean(hasNewData || params.type || explicitVector) + + // Leg D — the unvector door clears a PENDING deferred-embed marker (see + // update()'s matching comment for the full rationale): the durable + // clear (an `embed.landed` record, empty vector) rides the batch's ONE + // commit fact via `plan.markerRecords`; the in-memory clear is deferred + // to `plan.postCommit` so an aborted batch never disarms a marker whose + // durable twin was never written. + const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) + if (clearsPendingEmbed) { + plan.markerRecords.push({ type: 'embed.landed', id: params.id, vector: [] }) + plan.postCommit.push(() => { + this.clearPendingEmbed(params.id) + prodLog.warn( + `[Brainy] transact update: entity ${params.id} had a pending deferred embed — ` + + `the unvector door cleared it ('vector: []' is an explicit instruction, ` + + `never "please embed").` + ) + }) + } + + // Leg D — vectored-ledger decrement for the sanctioned unvector door, + // deferred to `plan.vectorUnlands` (consumed with a proper `await` in + // `transact()`, AFTER the commit succeeds — see its matching comment). + // Gated on the PRIOR vector having actually been real (non-empty, + // non-zero-norm): idempotent on an already-unvectored row. + if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { + plan.vectorUnlands.push(params.id) + } const newMetadata = params.merge !== false @@ -12431,21 +12612,49 @@ export class Brainy implements BrainyInterface { const metadataStats = await this.metadataIndex.getStats() const graphSize = await this.graphIndex.size() - // 1. Index size parity. HNSW must hold at least one node per indexed entity. - if (hnswSize === metadataStats.totalEntries) { + // 1. Index size parity. HNSW must hold one node per VECTORED noun — the + // vectored-noun ledger (`getCanonicalCounts().vectors.all`), NOT the raw + // metadata-entry count: every store's VFS root is PERMANENTLY unvectored + // (`vector: []` by design — a zero-norm/empty vector never crosses into + // the index, see AddToVectorIndexOperation/JsHnswVectorIndex.rebuild()'s + // matching belts), and a not-yet-landed deferred embed is unvectored + // too. Comparing against total entries counted the always-unvectored + // root as a permanent 1-node "drift" on every VFS-having store — a false + // warn on an otherwise perfectly healthy handoff. `vectors.all` is + // already the documented coverage denominator for exactly this + // comparison (see `CanonicalCounts.vectors`'s JSDoc). Falls back to the + // metadata-entry count when the ledger is unavailable or suspect (a + // storage adapter without the optional hook, or an unrecounted store) — + // never worse than the prior behavior in that case. + const vectorLedgerForParity = await this.storage.getCanonicalCounts?.() + const vectorParityTarget = + vectorLedgerForParity && !vectorLedgerForParity.suspect + ? vectorLedgerForParity.vectors.all + : metadataStats.totalEntries + if (hnswSize === vectorParityTarget) { checks.push({ name: 'index-parity', status: 'pass', - message: `HNSW (${hnswSize}) and metadata index (${metadataStats.totalEntries}) agree.`, - details: { hnswSize, metadataEntries: metadataStats.totalEntries, graphRelationships: graphSize } + message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) agree.`, + details: { + hnswSize, + vectoredNouns: vectorParityTarget, + metadataEntries: metadataStats.totalEntries, + graphRelationships: graphSize + } }) } else { - const drift = Math.abs(hnswSize - metadataStats.totalEntries) + const drift = Math.abs(hnswSize - vectorParityTarget) checks.push({ name: 'index-parity', - status: drift > Math.max(10, metadataStats.totalEntries * 0.01) ? 'fail' : 'warn', - message: `HNSW (${hnswSize}) and metadata (${metadataStats.totalEntries}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, - details: { hnswSize, metadataEntries: metadataStats.totalEntries, drift } + status: drift > Math.max(10, vectorParityTarget * 0.01) ? 'fail' : 'warn', + message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, + details: { + hnswSize, + vectoredNouns: vectorParityTarget, + metadataEntries: metadataStats.totalEntries, + drift + } }) } @@ -16066,6 +16275,79 @@ export class Brainy implements BrainyInterface { return !this.pluginRegistry.hasProvider('embeddings') } + /** + * @description LEG C of the zero-norm/unvector-door law — migrate a + * legacy zero-norm VFS root BEFORE the vector-leg open gate + * ({@link rebuildIndexesIfNeeded}'s `vectorCoverageGap` check) ever + * compares the canonical vectored-noun count against the vector index's + * size. A pre-fix store may have persisted the VFS root (the fixed + * all-zeros UUID) with a REAL all-zero placeholder vector — lawful inside + * brainy (`cosineDistance` treats a zero-norm operand as MAXIMUM distance, + * see {@link isZeroNormVector}'s JSDoc) but never indexed (the index belt + * refuses to insert a zero-norm vector) and never meant to cross an + * engine boundary. Left unmigrated, the canonical ledger still counts it + * as vectored while the vector index correctly holds nothing for it — a + * near-empty store whose ONLY vectored row is this zero-norm root reads + * "canonical vectored 1, index size 0" and throws + * `VectorIndexNotReadyError` at open, going DARK instead of serving. + * + * THE LIFECYCLE LAW: nothing at open may scale with brain size. This step + * is safe under that law BECAUSE the VFS root lives at a FIXED, + * well-known id (`00000000-0000-0000-0000-000000000000` — mirrors + * `VirtualFileSystem.VFS_ROOT_ID`; kept as a literal here, the same + * convention as the other reserved-root literals in this file and in + * `db/factLog.ts`/`db/portableGraph.ts` — `brainy.ts` cannot import + * `VirtualFileSystem.ts`, which itself imports `Brainy`) — this is ONE + * direct canonical read by id (`storage.getNoun`, the same O(1) + * fixed-path lookup {@link unvectorNounForRootMigration} itself uses + * internally), NEVER a listing or a walk over `entities/nouns/**`. An + * absent root (a store that has never used the VFS) is a no-op, no error. + * + * Runs UNCONDITIONALLY at every open, independent of whether a + * `VirtualFileSystem` is ever constructed this session — the vector-leg + * gate this fixes runs during Brainy's OWN init, before any + * `VirtualFileSystem` instance exists to run its own lazy migration at + * `doInitializeRoot()` (kept in place as the second line of defense for a + * VFS actually opened this session — belt AND suspenders, never either + * alone). + */ + private async migrateLegacyZeroNormVfsRootIfNeeded(): Promise { + const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + // TORN-TOLERANT: a torn root record is a recovery-walk healer's job + // (see tests/integration/recovery-walk-tolerance.test.ts — an init-time + // walk that meets a torn record narrates+counts, via the adapter's own + // loud floor at the read site, and heals PAST it; the open itself must + // still succeed), not this O(1) migration check's. Skip this open's + // migration attempt rather than aborting init(): this leg is a + // defensive EXTRA (the index belt + VirtualFileSystem's own + // doInitializeRoot() migration still stand as the other lines of + // defense), and it retries harmlessly at a later open once the root + // heals. + let root: HNSWNounWithMetadata | null + try { + root = await this.storage.getNoun(VFS_ROOT_ID) + } catch (err) { + if ((err as { code?: string }).code !== 'TORN_RECORD') throw err + prodLog.warn( + `[Brainy] open(): the VFS root's record is TORN — skipping the zero-norm root ` + + `migration check this open (the recovery walk is the healer; this migration ` + + `retries harmlessly once the root heals).` + ) + return + } + if (!root || !Array.isArray(root.vector) || root.vector.length === 0) return + if (!isZeroNormVector(root.vector)) return + const migrated = await this.unvectorNounForRootMigration(VFS_ROOT_ID) + if (migrated) { + prodLog.warn( + `[Brainy] open(): migrated the VFS root's legacy all-zero placeholder vector to ` + + `the unvectored shape (zero-norm vectors never cross an engine boundary) — run ` + + `before the vector-leg open gate compares canonical-vectored-count against the ` + + `vector index, so a near-empty store never reads a false coverage gap.` + ) + } + } + /** * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's * persisted vector from a real (non-empty) vector to the "unvectored" @@ -16075,13 +16357,18 @@ export class Brainy implements BrainyInterface { * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the * coverage ledger never silently drifts. * - * Exists SOLELY for the VFS root zero-norm migration (see - * `VirtualFileSystem.doInitializeRoot()`, which detects a persisted root - * whose vector is the legacy all-zero placeholder and calls this once per - * store). This is NOT a general-purpose "clear my vector" API — ordinary - * application data has no sanctioned path from vectored back to - * unvectored (`update()` refuses an empty vector as a dimension mismatch, - * by design). Never call this outside the VFS root migration. + * Exists SOLELY for the VFS root zero-norm migration, called from two + * sites that detect the same legacy shape (a persisted root whose vector + * is the legacy all-zero placeholder): {@link migrateLegacyZeroNormVfsRootIfNeeded} + * (this brain's own init sequence, BEFORE the vector-leg open gate — Leg + * C of the zero-norm/unvector-door law) and + * `VirtualFileSystem.doInitializeRoot()` (the second line of defense, for + * a VFS actually constructed this session). This is NOT the general- + * purpose unvector API — ordinary application data uses the sanctioned + * unvector DOOR instead (`update({ id, vector: [] })` / the same op inside + * `transact()`), which decrements the ledger and clears any pending + * deferred-embed marker inline; it does not call this method. Never call + * this outside a VFS root migration. * * Idempotent: a noun already unvectored (`vector.length === 0`) or absent * is a no-op — safe to call on every `init()`. diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 77e4f84d..1ed8f9ea 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -10,7 +10,7 @@ import { Vector, VectorDocument } from '../coreTypes.js' -import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' +import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' @@ -1768,6 +1768,33 @@ export class JsHnswVectorIndex implements VectorIndexProvider { // Process all nouns at once for (const nounData of result.items) { try { + // THE ZERO-NORM LAW / THE INDEX BELT — bulk-rebuild leg: a row + // holding no REAL vector (the "unvectored" `[]` shape — a + // deferred embed's stub, or a row the sanctioned unvector door + // rewrote) must never enter the index, mirroring the guard + // AddToVectorIndexOperation/ReplaceInVectorIndexOperation already + // enforce on the live transactional write paths. This matters + // HERE specifically: a row can be unvectored (canonical vector + // rewritten to `[]`) while its PERSISTED HNSW graph metadata + // (`getVectorIndexData` — level/connections) is still present + // from before the unvector, so `hnswData`'s mere presence below + // is not proof the row belongs in the index — only the + // canonical vector itself is authoritative. Checked against + // `nounData.vector` (the FULL loaded vector), never the + // to-be-truncated `noun.vector` below, so this holds regardless + // of the eager/lazy preload strategy chosen further down. + if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) { + continue + } + if (isZeroNormVector(nounData.vector)) { + prodLog.warn( + `[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` + + `zero-norm (a zero-norm vector is not a vector and never crosses an ` + + `engine boundary)` + ) + continue + } + // Load HNSW graph data for this entity const hnswData = await this.storage.getVectorIndexData(nounData.id) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 817e5e36..4f2a43b0 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -19,6 +19,7 @@ import { import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' import { prodLog } from '../../utils/logger.js' +import { isZeroNormVector } from '../../utils/distance.js' import { TornRecordError, isUnparseablePayloadError, @@ -2841,14 +2842,20 @@ export class FileSystemStorage extends BaseStorage { } /** - * Count canonical nouns holding a REAL (non-empty) vector — the vectored- - * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence - * cannot be decided from the id-directory listing alone: a deferred-embed - * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`) - * until its embed LANDS, so this walk reads every noun's `vectors.json` - * CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time - * legacy-counts.json derivation or a lost/corrupted counts.json recovery; - * the result is persisted so this scan never repeats. + * Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector — + * the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, + * presence cannot be decided from the id-directory listing alone: a + * deferred-embed noun's `vectors.json` EXISTS (written at `add()` time + * with `vector: []`) until its embed LANDS, so this walk reads every + * noun's `vectors.json` CONTENT — O(nouns) reads, not O(ids) listing. + * ZERO-NORM LAW: a real all-zero vector is not a vector — it never counts + * here either (Brainy's write paths normalize an explicit zero-norm + * vector to `[]` at write time, but a store created before that fix may + * still carry legacy all-zero rows on disk; this derivation must agree + * with the live ledger's definition of "vectored" regardless of when the + * row was written). Used ONLY for a one-time legacy-counts.json derivation + * or a lost/corrupted counts.json recovery; the result is persisted so + * this scan never repeats. */ private async scanVectoredNounCount(): Promise { const base = path.join(this.rootDir, 'entities', 'nouns') @@ -2862,7 +2869,12 @@ export class FileSystemStorage extends BaseStorage { for (const entry of ids) { if (!entry.isDirectory()) continue const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) - if (record && Array.isArray(record.vector) && record.vector.length > 0) { + if ( + record && + Array.isArray(record.vector) && + record.vector.length > 0 && + !isZeroNormVector(record.vector) + ) { vectored++ } } diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 606516ac..00790a4a 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -613,6 +613,17 @@ export function validateUpdateParams(params: UpdateParams): void { // null/undefined means "no new data was given". const hasData = params.data !== undefined && params.data !== null if ((params as UpdateParams & { deferEmbedding?: boolean }).deferEmbedding === true) { + if (params.vector && params.vector.length === 0) { + // The nonsensical combination Leg D of the zero-norm/unvector-door law + // refuses: `vector: []` is the SANCTIONED UNVECTOR DOOR — an explicit + // instruction to remove the vector NOW, never "please embed" — so it + // cannot be paired with a request to defer an embed. + throw new Error( + `update(): 'vector: []' (the unvector door) cannot be combined with ` + + `'deferEmbedding: true' — an unvector is an explicit instruction to remove ` + + `the vector now, not a request to defer an embed. Drop one of the two.` + ) + } if (params.vector) { throw new Error( `update(): deferEmbedding cannot be combined with an explicit 'vector' — ` + @@ -649,8 +660,16 @@ export function validateUpdateParams(params: UpdateParams): void { throw new Error(`invalid NounType: ${params.type}`) } - // Validate vector dimensions if provided - if (params.vector) { + // Validate vector dimensions if provided. A length-0 vector is the + // SANCTIONED UNVECTOR DOOR (see brainy.ts update()'s matching comment): an + // explicit `vector: []` — or a real all-zero vector, normalized to `[]` + // upstream by the zero-norm law — carries no dimension information, + // exactly like validateAddParams's identical exemption, so it is exempt + // from the dimension check rather than refused as a "0-dimensional + // vector". (The `deferEmbedding` combination above already refuses + // `vector: []` paired with `deferEmbedding: true` — an empty array is + // truthy, so that guard fires unconditionally on any explicit `vector`.) + if (params.vector && params.vector.length > 0) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) diff --git a/tests/integration/vfs-root-zero-norm.test.ts b/tests/integration/vfs-root-zero-norm.test.ts index 00260eb4..577ae7ee 100644 --- a/tests/integration/vfs-root-zero-norm.test.ts +++ b/tests/integration/vfs-root-zero-norm.test.ts @@ -18,9 +18,16 @@ * harness reproduces exactly what a pre-fix store looked like on disk) * is rewritten to `[]` on the next `init()`, the ledger is decremented * through the sanctioned path, and a second `init()` is a no-op. - * (c) THE BELT at the live provider-write seam: an entity added with an - * EXPLICIT all-zero vector (any dimension) still lands its canonical - * write, but the vector-index insert is refused loudly. + * (c) THE CANONICAL-WRITE NORMALIZATION (Leg A of the follow-up + * zero-norm/unvector-door fix): an entity added with an EXPLICIT + * all-zero vector (any dimension) is normalized to the "unvectored" + * `[]` shape BEFORE the canonical write, the ledger flag, and the index + * ops ever see it — the canonical write still succeeds, loudly, and the + * vector-index insert never happens (nothing to index). Supersedes the + * original "canonical keeps the zero vector, only the index refuses" + * shape: a downstream engine's health-report gate reads the canonical + * ledger directly, so leaving a zero-norm vector on the canonical side + * re-opened the exact false-attractor risk this whole fix closes. * (d) the migrated root never surfaces in `find()` results (it was already * hidden behind `visibility: 'system'` — this pin holds regardless). */ @@ -142,7 +149,7 @@ describe('VFS root zero-norm cure', () => { await brain.close() }) - it('(c) the live-write belt: an entity added with an explicit all-zero vector lands its canonical write, but the vector-index insert is refused loudly', async () => { + it('(c) canonical-write normalization: an entity added with an explicit all-zero vector persists UNVECTORED ([]), loudly, and never reaches the vector index', async () => { const dir = mkTmp() const brain = openBrain(dir) await brain.init() @@ -150,20 +157,26 @@ describe('VFS root zero-norm cure', () => { const warnSpy = vi.spyOn(prodLog, 'warn') const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const ledgerBefore = await brain.storage.getCanonicalCounts() const zeroVector = new Array(384).fill(0) const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) - // The canonical write succeeded — the entity is fully readable with its - // (real, all-zero) vector intact. + // The canonical write succeeded — but the zero-norm vector was + // normalized to the "unvectored" `[]` shape BEFORE it was persisted + // (Leg A: a zero-norm vector is not a vector — it never crosses an + // engine boundary, canonical side included). const entity = await brain.get(id, { includeVectors: true }) expect(entity).not.toBeNull() - expect(entity.vector).toEqual(zeroVector) + expect(entity.vector).toEqual([]) - // The vector-index insert was skipped — the index size never moved. + // Nothing to index — the vector-index size never moved, and the + // vectored-noun ledger never counted this row. const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size expect(sizeAfter).toBe(sizeBefore) + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - // The refusal was LOUD and named the entity. + // The normalization was LOUD and named the entity. const loudCall = warnSpy.mock.calls.find( (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') ) diff --git a/tests/integration/zero-norm-unvector-door.test.ts b/tests/integration/zero-norm-unvector-door.test.ts new file mode 100644 index 00000000..d3290010 --- /dev/null +++ b/tests/integration/zero-norm-unvector-door.test.ts @@ -0,0 +1,399 @@ +/** + * @module tests/integration/zero-norm-unvector-door + * @description THE SEAM LAW, GENERALIZED: "a zero-norm vector is not a + * vector — it never crosses an engine boundary." `tests/integration/ + * vfs-root-zero-norm.test.ts` pins the VFS-root-specific cure; this file + * pins the follow-up that generalizes it to every write path plus the + * sanctioned door for shedding a vector on purpose. + * + * Four legs pinned here: + * (A) THE CANONICAL WRITE NORMALIZES ZERO-NORM TO `[]` — `add()` (single and + * `transact()`) persists an explicit real all-zero vector as the + * "unvectored" `[]` shape, loudly, before the ledger flag/dimension + * pin/index ops ever see it. The canonical write still succeeds. + * (B) THE LEGACY DERIVATION IS ZERO-NORM-AWARE — a lost/corrupted + * `counts.json`'s one-time re-derivation walk excludes a persisted + * zero-norm row from the vectored-noun scalar, matching the live + * ledger's definition of "vectored". + * (C) THE LEGACY VFS ROOT MIGRATES AT OPEN, BEFORE THE GATE, IN O(1) — a + * store whose ONLY vectored row is a legacy all-zero VFS root opens + * clean (no `VectorIndexNotReadyError`), via one fixed-path read, never + * a listing. + * (D) THE UNVECTOR DOOR — `update({ id, vector: [] })` (and the same op + * inside `transact()`) is the sanctioned, idempotent way to shed a + * vector on purpose: ledger decrement exactly once, index removal, no + * re-embed, and a pending deferred-embed marker is cleared rather than + * left to re-vectorize the row later. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { prodLog } from '../../src/utils/logger.js' +import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' +import { BaseStorage } from '../../src/storage/baseStorage.js' + +const ROOT_ID = '00000000-0000-0000-0000-000000000000' + +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-zero-norm-unvector-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + vi.restoreAllMocks() + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) +} + +const countsPath = (root: string) => path.join(root, '_system', 'counts.json') + +describe('zero-norm canonical write + the sanctioned unvector door', () => { + it('(A1) add() with an explicit all-zero vector persists [], warns loudly, never indexes, and the ledger is unchanged', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const warnSpy = vi.spyOn(prodLog, 'warn') + + const zeroVector = new Array(384).fill(0) + const id = await brain.add({ data: 'zero-norm add', type: NounType.Document, vector: zeroVector }) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore) + + const loud = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') + ) + expect(loud).toBeDefined() + + await brain.close() + }) + + it('(A2) transact() add with an explicit all-zero vector — the same canonical normalization', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const warnSpy = vi.spyOn(prodLog, 'warn') + const zeroVector = new Array(384).fill(0) + const id = 'aaaaaaaa-0000-4000-8000-000000000001' + + await brain.transact([ + { op: 'add', id, type: NounType.Document, data: 'zero-norm transact add', vector: zeroVector } + ]) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + const loud = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') + ) + expect(loud).toBeDefined() + + await brain.close() + }) + + it('(B) the legacy counts.json derivation excludes a persisted zero-norm row from the vectored-noun scalar', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() + + // The VFS root alone (unvectored — []) — the floor. + const baseline = (await brain.storage.getCanonicalCounts()).vectors.all + + const realId = await brain.add({ data: 'a real vectored document', type: NounType.Document }) + + // Plant the legacy all-zero shape BY HAND: a genuine identity record + // (via add(), so it has real metadata) whose vector leg is then + // overwritten directly through the raw storage primitive — bypassing + // Leg A's canonical-write normalization entirely (brain.storage.saveNoun + // is not Brainy.add()/update()'s normalized path) — reproducing exactly + // what a pre-fix store could have persisted on disk. + const zeroId = await brain.add({ data: 'a legacy zero-norm document', type: NounType.Document }) + const zeroVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: zeroId, vector: zeroVector, connections: new Map(), level: 0 }) + + await brain.flush() + await brain.close() + + // Remove counts.json so the next open re-derives from scratch (the + // one-time legacy/lost-file derivation path — Leg B). + fs.rmSync(countsPath(dir), { force: true }) + + brain = openBrain(dir) + await brain.init() + const ledger = await brain.storage.getCanonicalCounts() + // Only realId counts; zeroId's persisted all-zero vector does not. + expect(ledger.vectors.all).toBe(baseline + 1) + + await brain.close() + }) + + it('(C) a legacy all-zero VFS root as the ONLY vectored row: open succeeds with no not-ready error, via an O(1) fixed-path read (no entities-tree readdir), and the ledger is 0 after open', async () => { + const dir = mkTmp() + + // SESSION 1 — build the legacy shape: the root is a REAL all-zero + // 384-dim vector, genuinely indexed and genuinely ledgered — exactly + // what a pre-fix store's root looked like on disk (see + // vfs-root-zero-norm.test.ts pin (b) for the identical harness). + // `index.addItem` is called directly (bypassing the transactional + // zero-norm belt) because the pre-fix code path had no such belt — this + // harness must match history, not the cure. No other entity is added, + // so the root is the store's ONLY vectored row. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.flush() + expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(1) + await brain.close() + + // SESSION 2 — reopen with a FAKE native vector provider that claims + // `serving: true` at `size()===0` (the exact shape a downstream + // engine's own health report can legitimately carry — same technique as + // tests/integration/vector-leg-open-build.test.ts). This is the ONLY + // codepath where the vector-leg open gate's FAIL-TYPED throw + // (VectorIndexNotReadyError) can fire; the built-in JS engine alone + // never reaches it (the size-heuristic branch just rebuilds instead) — + // so this is the faithful reproduction of the incident Leg C closes. + const readdirCalls: string[] = [] + const originalReaddir = fs.promises.readdir.bind(fs.promises) + vi.spyOn(fs.promises, 'readdir').mockImplementation(((...args: any[]) => { + readdirCalls.push(String(args[0])) + return (originalReaddir as any)(...args) + }) as any) + + // Spy at the PROTOTYPE level (BaseStorage.getNoun) — the new brain's + // storage instance does not exist until init() runs, so an + // instance-level spy cannot be installed beforehand. Records the + // readdir-call delta across the FIRST call made with the root id — + // Leg C's own fixed-path read — proving it needs no directory listing. + let readdirDeltaDuringRootRead: number | null = null + const originalGetNoun = BaseStorage.prototype.getNoun + vi.spyOn(BaseStorage.prototype, 'getNoun').mockImplementation(async function ( + this: unknown, + id: string + ) { + const before = readdirCalls.length + const result = await originalGetNoun.call(this as BaseStorage, id) + if (id === ROOT_ID && readdirDeltaDuringRootRead === null) { + readdirDeltaDuringRootRead = readdirCalls.length - before + } + return result + }) + + brain = openBrain(dir) + brain.use({ + name: 'fake-native-vector-unledgered-coverage', + activate: async (ctx: any) => { + ctx.registerProvider('vector', (config: any, distance: any, options: any) => { + const real = new JsHnswVectorIndex(config, distance, options) + let rebuilt = false + const originalRebuild = real.rebuild.bind(real) + ;(real as any).rebuild = async (...args: any[]) => { + const r = await originalRebuild(...args) + rebuilt = true + return r + } + const originalSize = real.size.bind(real) + ;(real as any).size = () => (rebuilt ? originalSize() : 0) + ;(real as any).healthReport = () => ({ + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 0, + generation: 1, + unledgered: ['vector-coverage'] + }) + return real + }) + return true + } + }) + + // Must NOT throw VectorIndexNotReadyError (or anything else) — a + // near-empty store whose only vectored row is the zero-norm root must + // never go dark. + await brain.init() + + const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) + expect(migratedRoot.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(0) + + expect(readdirDeltaDuringRootRead).toBe(0) + + await brain.close() + }) + + describe('the sanctioned unvector door', () => { + it('(D1) update({ id, vector: [] }) unvectors a real vectored row — canonical [], removed from the index, ledger decremented by exactly 1, no embed call', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + await brain.flush() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + + const embedSpy = vi.spyOn(brain, 'embed') + await brain.update({ id, vector: [] }) + expect(embedSpy).not.toHaveBeenCalled() + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore - 1) + + await brain.close() + }) + + it('(D2) idempotent: a second update({ id, vector: [] }) on an already-unvectored row is a true no-op — no error, no further decrement', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + await brain.flush() + + await brain.update({ id, vector: [] }) + const ledgerAfterFirst = await brain.storage.getCanonicalCounts() + + await brain.update({ id, vector: [] }) + const ledgerAfterSecond = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfterFirst.vectors.all) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + await brain.close() + }) + + it('(D3) a PENDING deferred-embed row: the unvector door clears the marker; awaitPendingEmbeds() then leaves it unvectored', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + // Prevent the background worker from ever actually running — it is + // fire-and-forget from add(), and a real run would race this test's + // own assertions (see tests/integration/vector-leg-open-build.test.ts + // for the same concern). This isolates exactly the marker-clearing + // behavior under test. + vi.spyOn(brain as any, 'kickEmbedWorker').mockImplementation(() => {}) + + const id = await brain.add({ + data: 'deferred content, never embedded', + type: NounType.Document, + deferEmbedding: true + }) + expect(brain.pendingEmbedCount()).toBe(1) + + const warnSpy = vi.spyOn(prodLog, 'warn') + await brain.update({ id, vector: [] }) + + expect(brain.pendingEmbedCount()).toBe(0) + const clearedWarn = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('pending') + ) + expect(clearedWarn).toBeDefined() + + // The barrier must not hang and must not re-vectorize the row — the + // worker (still mocked to a no-op) never runs again. + await brain.awaitPendingEmbeds() + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + await brain.close() + }) + + it('(D4) update({ vector: [], deferEmbedding: true }) is a typed refusal — the unvector door cannot be paired with a deferred embed', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + const before = await brain.get(id, { includeVectors: true }) + + await expect( + brain.update({ id, vector: [], deferEmbedding: true }) + ).rejects.toThrow(/unvector door/i) + + // Refused before any write — the row is untouched. + const after = await brain.get(id, { includeVectors: true }) + expect(after.vector).toEqual(before.vector) + + await brain.close() + }) + + it('(D5) the transact() twin of the unvector door decrements the ledger exactly once, and is idempotent on a second call', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document for transact unvector', type: NounType.Document }) + await brain.flush() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + + await brain.transact([{ op: 'update', id, vector: [] }]) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore - 1) + + // Idempotent through transact() too. + await brain.transact([{ op: 'update', id, vector: [] }]) + const ledgerAfterSecond = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfter.vectors.all) + + await brain.close() + }) + }) +})