From c6cc0de955e396ab3b65adc80ad92564cacbbf3e Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 09:28:44 -0700 Subject: [PATCH] fix(vfs): the VFS root never persists a zero-norm vector A zero-norm vector is lawful inside brainy (cosine distance scores it at maximum, never a false top hit) but a false attractor for a downstream engine serving squared-euclidean distance, which cannot tell a real all-zero vector apart from a legitimate origin point. - The VFS root now persists with vector [] (the existing "unvectored" shape) instead of a real all-zero 384-dim placeholder, and is never routed into the deferred-embed pipeline. - A one-time migration in the root-init path detects a pre-fix store's all-zero placeholder root (by norm, not length) and rewrites it to [] through a new sanctioned Brainy method that keeps the canonical vectored-noun ledger honest and removes the row from the vector index. - The vector-index write seam (AddToVectorIndexOperation, ReplaceInVectorIndexOperation, and the generation materializer's direct insert) now refuses any real all-zero vector before it reaches a provider, loudly naming the entity, while the canonical write still lands. - add()'s dimension-pinning and HNSW-insert gates, and the add-params validator, now treat any empty vector as carrying no dimension information, closing a latent trap where an explicit `vector: []` would have pinned dimensions to 0. --- src/brainy.ts | 105 ++++++++- src/coreTypes.ts | 21 ++ src/storage/adapters/baseStorageAdapter.ts | 18 ++ src/transaction/operations/IndexOperations.ts | 55 +++++ src/utils/distance.ts | 23 ++ src/utils/paramValidation.ts | 10 +- src/vfs/VirtualFileSystem.ts | 100 ++++++--- .../canonical-count-ledger.test.ts | 12 +- .../integration/vector-leg-open-build.test.ts | 33 +-- tests/integration/vfs-root-zero-norm.test.ts | 203 ++++++++++++++++++ 10 files changed, 516 insertions(+), 64 deletions(-) create mode 100644 tests/integration/vfs-root-zero-norm.test.ts diff --git a/src/brainy.ts b/src/brainy.ts index 6289e8f2..b498e632 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -25,6 +25,7 @@ import { } from './storage/brainFormat.js' import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' +import { isZeroNormVector } from './utils/distance.js' import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, @@ -2913,7 +2914,13 @@ export class Brainy implements BrainyInterface { // Ensure dimensions are set (a deferred-embed stub carries no dimension // information — the worker's real vector goes through the same guard). - if (!deferringEmbed) { + // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose + // vector is the "unvectored" empty-array shape carries no dimension + // information, deferred or not — an explicit `vector: []` (e.g. the VFS + // root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must + // never pin `this.dimensions` to 0, which would poison every subsequent + // real embed's dimension check for the life of the store. + if (!deferringEmbed && vector.length > 0) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -3029,10 +3036,15 @@ export class Brainy implements BrainyInterface { }, true) ) - // Operation 3: Add to HNSW index (after entity saved). A deferred - // embed has nothing to index yet — the worker's atomic update - // inserts the real vector. - if (!deferringEmbed) { + // Operation 3: Add to HNSW index (after entity saved). Gated on + // `vector.length > 0`, not `!deferringEmbed`: a deferred embed has + // nothing to index yet (the worker's atomic update inserts the real + // vector later), and an explicit `vector: []` insert (the VFS root's + // zero-norm fix — permanently unvectored plumbing, never embedded) + // is exactly the same "nothing to index yet" shape. The zero-norm + // BELT (a real all-zero vector, non-empty) is enforced inside + // AddToVectorIndexOperation itself — see its JSDoc. + if (vector.length > 0) { tx.addOperation( new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration) ) @@ -10245,6 +10257,18 @@ export class Brainy implements BrainyInterface { for (const id of nounIds) { const noun = await snapshotStorage.getNoun(id) if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) { + // THE ZERO-NORM LAW: a direct provider-write seam (this materializer + // inserts one-by-one, bypassing AddToVectorIndexOperation's own + // belt) — apply the same refusal here rather than handing a false + // attractor to the ephemeral reader's index. + if (isZeroNormVector(noun.vector)) { + prodLog.warn( + `[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` + + `entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` + + `engine boundary (the materialized record is unaffected)` + ) + continue + } await reader.index.addItem({ id: noun.id, vector: noun.vector }) } } @@ -10519,7 +10543,10 @@ export class Brainy implements BrainyInterface { const vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) - if (!deferringEmbed) { + // Gated on `vector.length > 0` — see the single-add() insert path's + // matching comment: an explicit `vector: []` carries no dimension + // information either, deferred or not. + if (!deferringEmbed && vector.length > 0) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -10601,9 +10628,12 @@ export class Brainy implements BrainyInterface { // for a deferred embed (stub vector `[]`; counted later at landing). new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), - ...(deferringEmbed - ? [] - : [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]), + // Gated on `vector.length > 0` — see the single-add() insert path's + // matching comment: an explicit `vector: []` has nothing to index + // either, deferred or not. + ...(vector.length > 0 + ? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)] + : []), new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration) ) plan.touchedNouns.push(id) @@ -16032,6 +16062,63 @@ export class Brainy implements BrainyInterface { return !this.pluginRegistry.hasProvider('embeddings') } + /** + * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's + * persisted vector from a real (non-empty) vector to the "unvectored" + * empty-array shape: the vector record is rewritten to `[]`, the row is + * removed from the vector index (if present), and the vectored-noun + * ledger (`getCanonicalCounts().vectors.all`) is decremented through the + * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the + * coverage ledger never silently drifts. + * + * Exists SOLELY for the VFS root zero-norm migration (see + * `VirtualFileSystem.doInitializeRoot()`, which detects a persisted root + * whose vector is the legacy all-zero placeholder and calls this once per + * store). This is NOT a general-purpose "clear my vector" API — ordinary + * application data has no sanctioned path from vectored back to + * unvectored (`update()` refuses an empty vector as a dimension mismatch, + * by design). Never call this outside the VFS root migration. + * + * Idempotent: a noun already unvectored (`vector.length === 0`) or absent + * is a no-op — safe to call on every `init()`. + * + * @param id - The canonical noun id to migrate. + * @returns `true` if a migration write happened, `false` if the noun was + * already unvectored (or absent) — a no-op. + */ + async unvectorNounForRootMigration(id: string): Promise { + const noun = await this.storage.getNoun(id) + if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false + + await this.persistSingleOp({ nouns: [id] }, async (tx) => { + // Rewrite the vector leg to the unvectored shape. Placeholder adjacency + // (mirrors update()'s own SaveNounOperation staging) — the op preserves + // stored graph state when `connections.size === 0`. + tx.addOperation( + new SaveNounOperation(this.storage, { + id, + vector: [], + connections: new Map(), + level: 0 + }) + ) + // Remove from the vector index — safe even if the row was never + // actually indexed (RemoveFromVectorIndexOperation's removeItem is a + // no-op when the id is absent). + tx.addOperation( + new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration) + ) + }) + + // Vectored-noun ledger: this migration carries a vector write with no + // accompanying metadata operation (metadata is untouched), so the + // saveNounMetadata(..., hasVector) seam never fires for it — mirrors the + // deferred-embed LANDING path's use of the narrow storage hook, in + // reverse. + await this.storage.noteVectorUnlanded?.(id) + return true + } + /** * Setup embedder */ diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 8112afd2..4b018e94 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -872,6 +872,27 @@ export interface StorageAdapter { */ noteVectorLanded?(id: string): Promise + /** + * OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}: + * record that a canonical noun's vector was just REMOVED — rewritten from + * a real (non-empty) vector to the "unvectored" empty-array shape. Exists + * for the ONE sanctioned reverse migration this engine supports: the VFS + * root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and + * `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix + * store's all-zero placeholder root vector to `[]` and must decrement + * `vectors.all` through this hook so the coverage ledger never drifts. + * NOT a general-purpose "I removed a vector" callback — ordinary + * application data has no sanctioned path from vectored back to + * unvectored (`update()` refuses an empty vector as a dimension + * mismatch by design). Callers MUST call this only when the noun held a + * REAL vector immediately before this write (the caller already holds + * that fact for free, from its own pre-write read — never an added read). + * A backend without vectored-noun tracking is a no-op via this method's + * absence (feature-detected). + * @param id - The noun whose vector was just removed. + */ + noteVectorUnlanded?(id: string): Promise + /** * Get noun with metadata combined * @returns Combined HNSWNounWithMetadata or null diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 22bf1366..88018a66 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1152,6 +1152,24 @@ export abstract class BaseStorageAdapter implements StorageAdapter { }) } + /** + * OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}): + * the mirror of {@link noteVectorLanded} — record a noun's vector was just + * REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a + * caller that (incorrectly) fires this for a noun already unvectored would + * otherwise drive the ledger negative — clamped defensively, matching the + * delete path's `if (this.totalVectoredNounCount > 0)` guard. + * @param id - The noun whose vector was just removed (retained for a + * future narration seam; the count itself needs no id-keyed state). + */ + async noteVectorUnlanded(id: string): Promise { + void id + if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount-- + this.scheduleCountPersist().catch(() => { + // Ignore persist errors — the in-memory count is authoritative; a later op retries. + }) + } + /** * Increment count for entity type - O(1) operation. * Concurrency is handled by the process-global mutex diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 139c67fe..cfee5074 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -13,6 +13,8 @@ import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' +import { isZeroNormVector } from '../../utils/distance.js' +import { prodLog } from '../../utils/logger.js' /** * Backend identity stamped into an operation's emitted `name` string (e.g. @@ -88,6 +90,30 @@ export class AddToVectorIndexOperation implements Operation { } async execute(): Promise { + // THE ZERO-NORM LAW (the live provider-write seam's belt): a zero-norm + // vector is not a vector — it never crosses an engine boundary. This + // engine's own cosine distance treats an all-zero vector safely (a + // zero-norm operand always scores MAXIMUM distance, see + // {@link isZeroNormVector}'s JSDoc), but a downstream engine serving + // squared-euclidean distance cannot tell it apart from a legitimate + // origin point — a false attractor that silently darkens real results. + // The canonical write already landed (SaveNoun/SaveNounMetadata + // operations are staged ahead of this one in every caller) — only the + // INDEX INSERT is refused here, loudly, never a throw. A length-0 + // vector is the unrelated "unvectored" shape and is skipped silently + // (the same contract callers already rely on for deferred embeds). + if (this.vector.length === 0) { + return async () => {} + } + if (isZeroNormVector(this.vector)) { + prodLog.warn( + `[vector-index] refusing to index a zero-norm vector for entity ${this.id} — ` + + `a zero-norm vector is not a vector and never crosses an engine boundary ` + + `(the canonical write is unaffected; only the vector-index insert is skipped)` + ) + return async () => {} + } + // Check if item already exists (for rollback decision) const existed = await this.itemExists(this.id) @@ -263,6 +289,35 @@ export class ReplaceInVectorIndexOperation implements Operation { // One commit generation for the whole replace (both branches + rollback). const generation = this.generationFn?.() + // THE ZERO-NORM LAW (see AddToVectorIndexOperation's matching JSDoc): a + // real all-zero replacement vector must never land in the index — refuse + // loudly, canonical write unaffected. The row must not be left stale + // either: if it was genuinely indexed under `oldVector`, remove it + // rather than pretend the old vector still describes the row. A + // length-0 `newVector` (the unrelated "unvectored" shape) is handled the + // same way, silently — no caller today reaches this with an empty + // replacement (update() rejects a dimension-mismatched empty vector), + // but the seam stays consistent in case one ever legitimately does. + if (isZeroNormVector(this.newVector) || this.newVector.length === 0) { + const wasIndexed = this.oldVector.length > 0 && !isZeroNormVector(this.oldVector) + if (isZeroNormVector(this.newVector)) { + prodLog.warn( + `[vector-index] refusing to replace with a zero-norm vector for entity ${this.id} — ` + + `a zero-norm vector is not a vector and never crosses an engine boundary ` + + `(the canonical write is unaffected; the row is removed from the vector index instead)` + ) + } + if (wasIndexed) { + await this.index.removeItem(this.id, generation) + } + return async () => { + // Restore the declared before-state. + if (wasIndexed) { + await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + } + } + } + if (typeof index.updateItem === 'function') { // Atomic path: one in-place call, the row never leaves the index. await index.updateItem({ id: this.id, vector: this.newVector }, generation) diff --git a/src/utils/distance.ts b/src/utils/distance.ts index 36e9e8e5..d61bc12e 100644 --- a/src/utils/distance.ts +++ b/src/utils/distance.ts @@ -65,6 +65,29 @@ export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number = return 1 - similarity } +/** + * True when `vector` is a REAL (non-empty) all-zero vector — the "false + * attractor" shape this engine's own cosine distance treats safely (a + * zero-norm operand always scores the MAXIMUM distance, see + * {@link cosineDistance}) but a downstream engine serving squared-euclidean + * distance cannot distinguish from a legitimate origin point. THE LAW: a + * zero-norm vector is not a vector — it never crosses an engine boundary + * (never handed to a vector-index provider as a searchable item). + * + * A length-0 vector is the UNRELATED "unvectored, not yet embedded" shape + * (the deferred-embed stub, a permanently-vectorless system row) and is + * deliberately NOT zero-norm here — callers checking for "nothing to index" + * should test `vector.length === 0` separately; this only flags the + * dangerous non-empty all-zero case. + */ +export function isZeroNormVector(vector: readonly number[]): boolean { + if (vector.length === 0) return false + for (let i = 0; i < vector.length; i++) { + if (vector[i] !== 0) return false + } + return true +} + /** * Calculates the Manhattan (L1) distance between two vectors. * Lower values indicate higher similarity. diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index d43559dc..606516ac 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -588,8 +588,14 @@ export function validateAddParams(params: AddParams): void { ) } - // Validate vector dimensions if provided - if (params.vector) { + // Validate vector dimensions if provided. A length-0 vector is the + // "unvectored" shape — an explicit `vector: []` (e.g. the VFS root's + // permanently-vectorless creation, see + // VirtualFileSystem.doInitializeRoot()'s zero-norm fix) carries no + // dimension information, exactly like an absent vector or a deferred + // embed's internal stub, so it is exempt from the dimension check rather + // than refused as a "0-dimensional vector". + if (params.vector && params.vector.length > 0) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 90018863..59a16be4 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -11,6 +11,7 @@ import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' import { Entity, AddParams, RelateParams, FindParams, Relation } from '../types/brainy.types.js' import { NounType, VerbType } from '../types/graphTypes.js' +import { isZeroNormVector } from '../utils/distance.js' import { PathResolver } from './PathResolver.js' import { mimeDetector } from './MimeTypeDetector.js' import { @@ -89,16 +90,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Uses deterministic UUID format for storage compatibility private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' - // OPEN-PATH FIX: the dimension of the placeholder vector given to the VFS - // root when it is first created (see `doInitializeRoot`). Mirrors the - // built-in WASM embedding engine's fixed, hardcoded output size - // (all-MiniLM-L6-v2 — see src/embeddings/candle-wasm/src/lib.rs - // HIDDEN_SIZE and src/embeddings/EmbeddingManager.ts). Deliberately NOT - // derived from `brain.dimensions` — this constant only applies to the - // default-embedder branch, where the true output dimension is this fixed - // value by construction, never a moving target. - private static readonly VFS_ROOT_VECTOR_DIMENSIONS = 384 - /** * Construct a VFS bound to a Brainy instance. * @@ -241,9 +232,11 @@ export class VirtualFileSystem implements IVirtualFileSystem { private async doInitializeRoot(): Promise { const rootId = VirtualFileSystem.VFS_ROOT_ID - // Try to get existing root by fixed ID (O(1) lookup, not query) + // Try to get existing root by fixed ID (O(1) lookup, not query). + // includeVectors: true — the zero-norm migration below (leg 2) needs to + // inspect the persisted vector to detect the legacy placeholder shape. try { - const existingRoot = await this.brain.get(rootId) + const existingRoot = await this.brain.get(rootId, { includeVectors: true }) if (existingRoot) { // Root exists - verify metadata is correct @@ -260,6 +253,34 @@ export class VirtualFileSystem implements IVirtualFileSystem { }) } + // ZERO-NORM ROOT MIGRATION (one-time): a pre-fix store persisted the + // root with a REAL all-zero placeholder vector — lawful inside + // brainy (cosineDistance treats a zero-norm operand as MAXIMUM + // distance, see src/utils/distance.ts) but a "false attractor" for a + // downstream engine serving squared-euclidean distance, which cannot + // tell an all-zero vector apart from a legitimate origin point (a + // production incident silently darkened 150+ rows in a partner + // engine's index this way). THE LAW: a zero-norm vector is not a + // vector — it never crosses an engine boundary. Detect the legacy + // shape via NORM, not length or dimension (any real all-zero vector + // qualifies, not just the historical 384-dim one), and rewrite it to + // the "unvectored" `[]` shape through the sanctioned migration path + // (Brainy.unvectorNounForRootMigration — see its JSDoc), which keeps + // `getCanonicalCounts().vectors.all` honest and removes the row from + // the vector index. Idempotent: a store already on the new shape + // (vector.length === 0) takes the false branch below on every + // subsequent init() — a permanent no-op, not a one-time flag. + const existingVector = existingRoot.vector ?? [] + if (existingVector.length > 0 && isZeroNormVector(existingVector)) { + const migrated = await this.brain.unvectorNounForRootMigration(rootId) + if (migrated) { + console.log( + 'VFS: migrated root vector from the legacy all-zero placeholder to the ' + + 'unvectored shape (zero-norm vectors never cross an engine boundary)' + ) + } + } + return rootId } } catch (error) { @@ -277,31 +298,42 @@ export class VirtualFileSystem implements IVirtualFileSystem { // meant every writer's FIRST-EVER open forced the process-global WASM // engine to cold-compile its model (measured 90-140s on throttled // CPUs) before the brain could even finish init(). This branch only - // runs once per store (the root already exists — with a real vector — - // in every previously-opened production store; reopening never re-adds - // it), so the fix applies only to brand-new stores. + // runs once per store (a previously-opened store already has a root — + // see the migration above for the pre-fix shape — reopening never + // re-adds it), so the fix applies only to brand-new stores. // - // Chosen fix: an explicit all-zero vector, not `deferEmbedding: true`. - // `deferEmbedding` looked attractive (ack now, embed later) but its - // landing path (`kickEmbedWorker()`, called synchronously right after - // commit — see brainy.ts add()/update()) would still force the WASM - // engine to cold-compile within milliseconds of open, just off the - // awaited path instead of never paying it at all — worse than the - // explicit-vector path, which never touches the engine for this row. - // An all-zero vector is safe: `cosineDistance` (src/utils/distance.ts) - // explicitly returns the MAXIMUM distance whenever either operand's - // norm is zero, so the root can never rank ahead of real content in a - // similarity search, and HNSW indexes it like any other vector. + // ZERO-NORM LAW (current shape, superseding the historical all-zero + // placeholder): the root's vector is `[]` — the SAME "unvectored" + // empty-array shape used for a deferred embed's stub and any other + // not-yet-embedded row — never a real all-zero vector. A zero-norm + // vector is lawful inside brainy (`cosineDistance`, see + // src/utils/distance.ts, returns the MAXIMUM distance whenever either + // operand's norm is zero) but is a "false attractor" for a downstream + // engine serving squared-euclidean distance, which cannot tell a real + // all-zero vector apart from a legitimate origin point — it silently + // darkened 150+ rows in a partner engine's index in production. THE + // LAW: a zero-norm vector is not a vector — it never crosses an engine + // boundary. `vector: []` achieves the SAME cold-compile avoidance the + // original placeholder did (`add()`'s dimension-pin and HNSW-insert + // gates both key off `vector.length > 0`, so an empty vector never + // calls embed(), never pins `brain.dimensions`, and never reaches the + // vector index — see brainy.ts add()'s matching comments) while never + // persisting a searchable zero vector for a downstream engine to trip + // over. Deliberately NOT `deferEmbedding: true`: that flag's landing + // path (`kickEmbedWorker()`, called synchronously right after commit — + // see brainy.ts add()/update()) would still force the WASM engine to + // cold-compile within milliseconds of open (just off the awaited path + // instead of never paying it at all) AND would eventually embed the + // root's data for real, which this fix forbids — the root must NEVER + // be embedded, not merely "not yet". // - // Only the default WASM engine gets this treatment — its output - // dimension (384) is fixed and hardcoded, so the placeholder can never - // mis-pin `brain.dimensions` for it. A plugin-registered native - // 'embeddings' provider has no cold-compile cost AND may use a - // different dimension, so it keeps embedding the root for real (same - // as before this fix) rather than risk pinning the wrong dimension - // ahead of the caller's own first real embed. + // Only the default WASM engine gets this treatment — a plugin- + // registered native 'embeddings' provider has no cold-compile cost and + // may use a different dimension, so it keeps embedding the root for + // real (same as before this fix) rather than leave Brainy's own + // plumbing permanently unvectored on a store where embedding is cheap. const rootVector = this.brain.usesDefaultWasmEmbedder() - ? new Array(VirtualFileSystem.VFS_ROOT_VECTOR_DIMENSIONS).fill(0) + ? ([] as number[]) : undefined await this.brain.add({ diff --git a/tests/integration/canonical-count-ledger.test.ts b/tests/integration/canonical-count-ledger.test.ts index f3c8ce20..3d292e97 100644 --- a/tests/integration/canonical-count-ledger.test.ts +++ b/tests/integration/canonical-count-ledger.test.ts @@ -225,9 +225,12 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s } /** Baseline vectored count right after a fresh open() — init() creates a - * hidden system VFS-root noun that itself carries a real vector, so a - * brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off - * this baseline rather than hardcoding it away. */ + * hidden system VFS-root noun, but (the zero-norm root cure) it is + * deliberately UNVECTORED (`vector: []`, never a real all-zero + * placeholder — a zero-norm vector never crosses an engine boundary), so + * a brand-new store's `vectors.all` is 0. Tests still assert DELTAS off + * this baseline rather than hardcoding it away, in case that ever + * changes again. */ let baseline: number beforeEach(async () => { @@ -235,6 +238,7 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-')) brain = await open() baseline = (await brain.storage.getCanonicalCounts()).vectors.all + expect(baseline).toBe(0) // the unvectored VFS root contributes nothing }) afterEach(async () => { vi.restoreAllMocks() @@ -348,7 +352,7 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s brain = await open() const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun + expect(ledger.vectors.all).toBe(baseline + 1) // just the one non-deferred noun — the root is unvectored expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) expect(persisted.totalVectoredNounCount).toBe(baseline + 1) diff --git a/tests/integration/vector-leg-open-build.test.ts b/tests/integration/vector-leg-open-build.test.ts index a1ccaefa..ea841545 100644 --- a/tests/integration/vector-leg-open-build.test.ts +++ b/tests/integration/vector-leg-open-build.test.ts @@ -176,21 +176,22 @@ describe('vector-leg open-build (two-engine gate, last red)', () => { }) it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => { - // ARCHITECTURAL NOTE (found while building this pin): every brainy store - // carries ONE permanent, always-vectored noun beyond user data — the VFS - // root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, - // src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero - // (but non-empty, length-384) vector on EVERY store's first open — never - // deferred (a deliberate WASM-cold-compile-avoidance fix, see that - // file's comment) — and VFS init unconditionally re-creates it if - // missing, before the rebuild gate ever runs. A literal "0 vectored - // nouns" store is therefore unreachable through the public API; a - // brand-new store's `vectors.all` floor is 1, not 0. This pin verifies - // the law the task names in the ACHIEVABLE form: nouns whose embed is - // still deferred/unlanded contribute NOTHING to the vectored-noun ledger - // — the coverage-gap comparison sees exactly the root (1), never - // root+deferred — and semantic search over deferred-only user content - // honestly returns `[]` (no error, no false "coverage restored" claim). + // ARCHITECTURAL NOTE (updated by the zero-norm root cure): every brainy + // store carries ONE permanent VFS root noun beyond user data + // (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, + // src/vfs/VirtualFileSystem.ts), created (or, on a pre-fix store, + // migrated) on every open — but it is deliberately UNVECTORED (vector + // `[]`), never a real all-zero placeholder: a zero-norm vector is not a + // vector and never crosses an engine boundary (see that file's + // doInitializeRoot() comment). It therefore contributes NOTHING to the + // vectored-noun ledger — a brand-new store's `vectors.all` floor is 0, + // not 1. This pin verifies the law the task names in the ACHIEVABLE + // form: nouns whose embed is still deferred/unlanded contribute NOTHING + // to the vectored-noun ledger either — the coverage-gap comparison sees + // exactly the baseline (the root, contributing 0), never + // baseline+deferred — and semantic search over deferred-only user + // content honestly returns `[]` (no error, no false "coverage restored" + // claim). const dir = mkTmp() const build: any = new Brainy({ @@ -202,6 +203,8 @@ describe('vector-leg open-build (two-engine gate, last red)', () => { }) await build.init() const rootOnlyLedger = await build.storage.getCanonicalCounts() + // THE NEW LAW: the root is unvectored — a brand-new store's floor is 0. + expect(rootOnlyLedger.vectors.all).toBe(0) // Block the embedder permanently so every add below stays deferred and // unlanded for the rest of this test (a fast deterministic embedder // could otherwise land it before we ever observe the "still 0 extra" diff --git a/tests/integration/vfs-root-zero-norm.test.ts b/tests/integration/vfs-root-zero-norm.test.ts new file mode 100644 index 00000000..00260eb4 --- /dev/null +++ b/tests/integration/vfs-root-zero-norm.test.ts @@ -0,0 +1,203 @@ +/** + * @module tests/integration/vfs-root-zero-norm + * @description THE ZERO-NORM ROOT CURE — a production incident traced 150+ + * darkened rows in a downstream engine's index to the VFS root's persisted + * ALL-ZERO placeholder vector: lawful inside brainy (`cosineDistance` + * treats a zero-norm operand as MAXIMUM distance, src/utils/distance.ts) + * but a "false attractor" for an engine serving squared-euclidean distance, + * which cannot tell a real all-zero vector apart from a legitimate origin + * point. THE LAW: a zero-norm vector is not a vector — it never crosses an + * engine boundary. + * + * Three legs pinned here: + * (a) the root persists NO zeros — a brand-new store creates it with + * vector `[]` (the "unvectored" shape), absent from the HNSW index, and + * the canonical vectored-noun ledger does not count it. + * (b) a ONE-TIME migration heals an existing (pre-fix) store: an old-shape + * root (a REAL all-zero vector, genuinely indexed and ledgered — the + * harness reproduces exactly what a pre-fix store looked like on disk) + * is rewritten to `[]` on the next `init()`, the ledger is decremented + * through the sanctioned path, and a second `init()` is a no-op. + * (c) THE BELT at the live provider-write seam: an entity added with an + * EXPLICIT all-zero vector (any dimension) still lands its canonical + * write, but the vector-index insert is refused loudly. + * (d) the migrated root never surfaces in `find()` results (it was already + * hidden behind `visibility: 'system'` — this pin holds regardless). + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { prodLog } from '../../src/utils/logger.js' + +const ROOT_ID = '00000000-0000-0000-0000-000000000000' + +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vfs-root-zero-norm-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + vi.restoreAllMocks() + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) +} + +describe('VFS root zero-norm cure', () => { + it('(a) a brand-new store persists the root with vector [], absent from the HNSW index, and the canonical ledger counts it unvectored', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const root = await brain.get(ROOT_ID, { includeVectors: true }) + expect(root).not.toBeNull() + expect(root.vector).toEqual([]) + + const status = await brain.getIndexStatus() + expect(status.hnswIndex.size).toBe(0) + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(0) + + await brain.close() + }) + + it('(b) an old-shape store (a real all-zero placeholder root) migrates to [] exactly once on init; the ledger is decremented through the sanctioned path; a second init is a no-op', async () => { + const dir = mkTmp() + + // SESSION 1 — build the store, then hand-rewrite the root to the LEGACY + // shape: a REAL all-zero 384-dim vector, genuinely inserted into the + // vector index and genuinely counted by the vectored-noun ledger — + // reproducing exactly what a pre-fix store's root looked like on disk + // (the pre-fix add() always indexed + counted it). `index.addItem` is + // called directly (bypassing AddToVectorIndexOperation's own zero-norm + // belt, added by this same fix) precisely because the pre-fix code path + // had no such belt — this harness must match history, not the cure. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.flush() + + const ledgerBeforeMigration = await brain.storage.getCanonicalCounts() + expect(ledgerBeforeMigration.vectors.all).toBe(1) + await brain.close() + + // SESSION 2 — reopen: VFS init must detect the legacy shape and migrate. + // Spy on the sanctioned migration method itself (not console output — + // `silent: true` monkey-patches `console.log` to a no-op INSIDE init(), + // which would silently swallow any pre-installed console spy too). + brain = openBrain(dir) + const migrateSpy = vi.spyOn(brain, 'unvectorNounForRootMigration') + await brain.init() + + expect(migrateSpy).toHaveBeenCalledTimes(1) + expect(migrateSpy).toHaveBeenCalledWith(ROOT_ID) + await expect(migrateSpy.mock.results[0].value).resolves.toBe(true) + + const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) + expect(migratedRoot.vector).toEqual([]) + + const ledgerAfterMigration = await brain.storage.getCanonicalCounts() + expect(ledgerAfterMigration.vectors.all).toBe(0) + + const statusAfterMigration = await brain.getIndexStatus() + expect(statusAfterMigration.hnswIndex.size).toBe(0) + + await brain.flush() + await brain.close() + + // SESSION 3 — reopen again: the migration is a permanent no-op, not a + // one-time flag that silently re-drifts or re-fires. The zero-norm + // detection at the VFS init site never even calls the migration method + // again — the root's vector is already `[]`. + brain = openBrain(dir) + const migrateSpy2 = vi.spyOn(brain, 'unvectorNounForRootMigration') + await brain.init() + + expect(migrateSpy2).not.toHaveBeenCalled() + + const rootAfterSecondInit = await brain.get(ROOT_ID, { includeVectors: true }) + expect(rootAfterSecondInit.vector).toEqual([]) + + const ledgerAfterSecondInit = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecondInit.vectors.all).toBe(0) + + await brain.close() + }) + + it('(c) the live-write belt: an entity added with an explicit all-zero vector lands its canonical write, but the vector-index insert is refused loudly', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const warnSpy = vi.spyOn(prodLog, 'warn') + + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const zeroVector = new Array(384).fill(0) + const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) + + // The canonical write succeeded — the entity is fully readable with its + // (real, all-zero) vector intact. + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual(zeroVector) + + // The vector-index insert was skipped — the index size never moved. + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore) + + // The refusal was LOUD and named the entity. + const loudCall = warnSpy.mock.calls.find( + (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') + ) + expect(loudCall).toBeDefined() + + await brain.close() + }) + + it('(d) find() over a store whose root has been migrated never returns the root (already hidden behind visibility: system — pinned anyway)', async () => { + const dir = mkTmp() + + // Build an old-shape store (same harness as pin (b)) and let it migrate. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.add({ data: 'a document about technology', type: NounType.Document }) + await brain.flush() + await brain.close() + + brain = openBrain(dir) // migrates on init() + await brain.init() + + const results = await brain.find({ query: 'technology', limit: 10 }) + expect(results.some((r: any) => r.id === ROOT_ID)).toBe(false) + + // Even asking explicitly for system-tier entities must never surface the + // root as a semantic-search HIT (it carries no vector to match against). + const resultsIncludingSystem = await brain.find({ query: 'technology', limit: 10, includeSystem: true }) + expect(resultsIncludingSystem.some((r: any) => r.id === ROOT_ID)).toBe(false) + + await brain.close() + }) +})