fix(vfs): the VFS root never persists a zero-norm vector
Some checks failed
CI / Node 22 (push) Successful in 12m23s
CI / Node 24 (push) Successful in 12m12s
CI / Integration + conformance (Node 22) (push) Failing after 14m47s
CI / Bun (latest) (push) Successful in 12m38s

A zero-norm vector is lawful inside brainy (cosine distance scores it at
maximum, never a false top hit) but a false attractor for a downstream
engine serving squared-euclidean distance, which cannot tell a real
all-zero vector apart from a legitimate origin point.

- The VFS root now persists with vector [] (the existing "unvectored"
  shape) instead of a real all-zero 384-dim placeholder, and is never
  routed into the deferred-embed pipeline.
- A one-time migration in the root-init path detects a pre-fix store's
  all-zero placeholder root (by norm, not length) and rewrites it to []
  through a new sanctioned Brainy method that keeps the canonical
  vectored-noun ledger honest and removes the row from the vector index.
- The vector-index write seam (AddToVectorIndexOperation,
  ReplaceInVectorIndexOperation, and the generation materializer's direct
  insert) now refuses any real all-zero vector before it reaches a
  provider, loudly naming the entity, while the canonical write still
  lands.
- add()'s dimension-pinning and HNSW-insert gates, and the add-params
  validator, now treat any empty vector as carrying no dimension
  information, closing a latent trap where an explicit `vector: []`
  would have pinned dimensions to 0.
This commit is contained in:
David Snelling 2026-08-27 09:28:44 -07:00
parent aad9e2eeb1
commit c6cc0de955
10 changed files with 516 additions and 64 deletions

View file

@ -25,6 +25,7 @@ import {
} from './storage/brainFormat.js'
import type { BrainFormat } from './storage/brainFormat.js'
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
import { isZeroNormVector } from './utils/distance.js'
import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
import {
defaultEmbeddingFunction,
@ -2913,7 +2914,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Ensure dimensions are set (a deferred-embed stub carries no dimension
// information — the worker's real vector goes through the same guard).
if (!deferringEmbed) {
// Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose
// vector is the "unvectored" empty-array shape carries no dimension
// information, deferred or not — an explicit `vector: []` (e.g. the VFS
// root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must
// never pin `this.dimensions` to 0, which would poison every subsequent
// real embed's dimension check for the life of the store.
if (!deferringEmbed && vector.length > 0) {
if (!this.dimensions) {
this.dimensions = vector.length
} else if (vector.length !== this.dimensions) {
@ -3029,10 +3036,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}, true)
)
// Operation 3: Add to HNSW index (after entity saved). A deferred
// embed has nothing to index yet — the worker's atomic update
// inserts the real vector.
if (!deferringEmbed) {
// Operation 3: Add to HNSW index (after entity saved). Gated on
// `vector.length > 0`, not `!deferringEmbed`: a deferred embed has
// nothing to index yet (the worker's atomic update inserts the real
// vector later), and an explicit `vector: []` insert (the VFS root's
// zero-norm fix — permanently unvectored plumbing, never embedded)
// is exactly the same "nothing to index yet" shape. The zero-norm
// BELT (a real all-zero vector, non-empty) is enforced inside
// AddToVectorIndexOperation itself — see its JSDoc.
if (vector.length > 0) {
tx.addOperation(
new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)
)
@ -10245,6 +10257,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
for (const id of nounIds) {
const noun = await snapshotStorage.getNoun(id)
if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) {
// THE ZERO-NORM LAW: a direct provider-write seam (this materializer
// inserts one-by-one, bypassing AddToVectorIndexOperation's own
// belt) — apply the same refusal here rather than handing a false
// attractor to the ephemeral reader's index.
if (isZeroNormVector(noun.vector)) {
prodLog.warn(
`[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` +
`entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` +
`engine boundary (the materialized record is unaffected)`
)
continue
}
await reader.index.addItem({ id: noun.id, vector: noun.vector })
}
}
@ -10519,7 +10543,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const vector = deferringEmbed
? []
: params.vector || (await this.embed(params.data))
if (!deferringEmbed) {
// Gated on `vector.length > 0` — see the single-add() insert path's
// matching comment: an explicit `vector: []` carries no dimension
// information either, deferred or not.
if (!deferringEmbed && vector.length > 0) {
if (!this.dimensions) {
this.dimensions = vector.length
} else if (vector.length !== this.dimensions) {
@ -10601,9 +10628,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// for a deferred embed (stub vector `[]`; counted later at landing).
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0),
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
...(deferringEmbed
? []
: [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]),
// Gated on `vector.length > 0` — see the single-add() insert path's
// matching comment: an explicit `vector: []` has nothing to index
// either, deferred or not.
...(vector.length > 0
? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]
: []),
new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration)
)
plan.touchedNouns.push(id)
@ -16032,6 +16062,63 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return !this.pluginRegistry.hasProvider('embeddings')
}
/**
* SANCTIONED, ONE-TIME MIGRATION HOOK rewrite a canonical noun's
* persisted vector from a real (non-empty) vector to the "unvectored"
* empty-array shape: the vector record is rewritten to `[]`, the row is
* removed from the vector index (if present), and the vectored-noun
* ledger (`getCanonicalCounts().vectors.all`) is decremented through the
* sanctioned {@link StorageAdapter.noteVectorUnlanded} hook so the
* coverage ledger never silently drifts.
*
* Exists SOLELY for the VFS root zero-norm migration (see
* `VirtualFileSystem.doInitializeRoot()`, which detects a persisted root
* whose vector is the legacy all-zero placeholder and calls this once per
* store). This is NOT a general-purpose "clear my vector" API ordinary
* application data has no sanctioned path from vectored back to
* unvectored (`update()` refuses an empty vector as a dimension mismatch,
* by design). Never call this outside the VFS root migration.
*
* Idempotent: a noun already unvectored (`vector.length === 0`) or absent
* is a no-op safe to call on every `init()`.
*
* @param id - The canonical noun id to migrate.
* @returns `true` if a migration write happened, `false` if the noun was
* already unvectored (or absent) a no-op.
*/
async unvectorNounForRootMigration(id: string): Promise<boolean> {
const noun = await this.storage.getNoun(id)
if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false
await this.persistSingleOp({ nouns: [id] }, async (tx) => {
// Rewrite the vector leg to the unvectored shape. Placeholder adjacency
// (mirrors update()'s own SaveNounOperation staging) — the op preserves
// stored graph state when `connections.size === 0`.
tx.addOperation(
new SaveNounOperation(this.storage, {
id,
vector: [],
connections: new Map(),
level: 0
})
)
// Remove from the vector index — safe even if the row was never
// actually indexed (RemoveFromVectorIndexOperation's removeItem is a
// no-op when the id is absent).
tx.addOperation(
new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration)
)
})
// Vectored-noun ledger: this migration carries a vector write with no
// accompanying metadata operation (metadata is untouched), so the
// saveNounMetadata(..., hasVector) seam never fires for it — mirrors the
// deferred-embed LANDING path's use of the narrow storage hook, in
// reverse.
await this.storage.noteVectorUnlanded?.(id)
return true
}
/**
* Setup embedder
*/