fix(vfs): the VFS root never persists a zero-norm vector
A zero-norm vector is lawful inside brainy (cosine distance scores it at maximum, never a false top hit) but a false attractor for a downstream engine serving squared-euclidean distance, which cannot tell a real all-zero vector apart from a legitimate origin point. - The VFS root now persists with vector [] (the existing "unvectored" shape) instead of a real all-zero 384-dim placeholder, and is never routed into the deferred-embed pipeline. - A one-time migration in the root-init path detects a pre-fix store's all-zero placeholder root (by norm, not length) and rewrites it to [] through a new sanctioned Brainy method that keeps the canonical vectored-noun ledger honest and removes the row from the vector index. - The vector-index write seam (AddToVectorIndexOperation, ReplaceInVectorIndexOperation, and the generation materializer's direct insert) now refuses any real all-zero vector before it reaches a provider, loudly naming the entity, while the canonical write still lands. - add()'s dimension-pinning and HNSW-insert gates, and the add-params validator, now treat any empty vector as carrying no dimension information, closing a latent trap where an explicit `vector: []` would have pinned dimensions to 0.
This commit is contained in:
parent
aad9e2eeb1
commit
c6cc0de955
10 changed files with 516 additions and 64 deletions
105
src/brainy.ts
105
src/brainy.ts
|
|
@ -25,6 +25,7 @@ import {
|
|||
} from './storage/brainFormat.js'
|
||||
import type { BrainFormat } from './storage/brainFormat.js'
|
||||
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
|
||||
import { isZeroNormVector } from './utils/distance.js'
|
||||
import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
|
||||
import {
|
||||
defaultEmbeddingFunction,
|
||||
|
|
@ -2913,7 +2914,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
|
||||
// Ensure dimensions are set (a deferred-embed stub carries no dimension
|
||||
// information — the worker's real vector goes through the same guard).
|
||||
if (!deferringEmbed) {
|
||||
// Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose
|
||||
// vector is the "unvectored" empty-array shape carries no dimension
|
||||
// information, deferred or not — an explicit `vector: []` (e.g. the VFS
|
||||
// root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must
|
||||
// never pin `this.dimensions` to 0, which would poison every subsequent
|
||||
// real embed's dimension check for the life of the store.
|
||||
if (!deferringEmbed && vector.length > 0) {
|
||||
if (!this.dimensions) {
|
||||
this.dimensions = vector.length
|
||||
} else if (vector.length !== this.dimensions) {
|
||||
|
|
@ -3029,10 +3036,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}, true)
|
||||
)
|
||||
|
||||
// Operation 3: Add to HNSW index (after entity saved). A deferred
|
||||
// embed has nothing to index yet — the worker's atomic update
|
||||
// inserts the real vector.
|
||||
if (!deferringEmbed) {
|
||||
// Operation 3: Add to HNSW index (after entity saved). Gated on
|
||||
// `vector.length > 0`, not `!deferringEmbed`: a deferred embed has
|
||||
// nothing to index yet (the worker's atomic update inserts the real
|
||||
// vector later), and an explicit `vector: []` insert (the VFS root's
|
||||
// zero-norm fix — permanently unvectored plumbing, never embedded)
|
||||
// is exactly the same "nothing to index yet" shape. The zero-norm
|
||||
// BELT (a real all-zero vector, non-empty) is enforced inside
|
||||
// AddToVectorIndexOperation itself — see its JSDoc.
|
||||
if (vector.length > 0) {
|
||||
tx.addOperation(
|
||||
new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)
|
||||
)
|
||||
|
|
@ -10245,6 +10257,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
for (const id of nounIds) {
|
||||
const noun = await snapshotStorage.getNoun(id)
|
||||
if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) {
|
||||
// THE ZERO-NORM LAW: a direct provider-write seam (this materializer
|
||||
// inserts one-by-one, bypassing AddToVectorIndexOperation's own
|
||||
// belt) — apply the same refusal here rather than handing a false
|
||||
// attractor to the ephemeral reader's index.
|
||||
if (isZeroNormVector(noun.vector)) {
|
||||
prodLog.warn(
|
||||
`[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` +
|
||||
`entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` +
|
||||
`engine boundary (the materialized record is unaffected)`
|
||||
)
|
||||
continue
|
||||
}
|
||||
await reader.index.addItem({ id: noun.id, vector: noun.vector })
|
||||
}
|
||||
}
|
||||
|
|
@ -10519,7 +10543,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const vector = deferringEmbed
|
||||
? []
|
||||
: params.vector || (await this.embed(params.data))
|
||||
if (!deferringEmbed) {
|
||||
// Gated on `vector.length > 0` — see the single-add() insert path's
|
||||
// matching comment: an explicit `vector: []` carries no dimension
|
||||
// information either, deferred or not.
|
||||
if (!deferringEmbed && vector.length > 0) {
|
||||
if (!this.dimensions) {
|
||||
this.dimensions = vector.length
|
||||
} else if (vector.length !== this.dimensions) {
|
||||
|
|
@ -10601,9 +10628,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// for a deferred embed (stub vector `[]`; counted later at landing).
|
||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0),
|
||||
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
|
||||
...(deferringEmbed
|
||||
? []
|
||||
: [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]),
|
||||
// Gated on `vector.length > 0` — see the single-add() insert path's
|
||||
// matching comment: an explicit `vector: []` has nothing to index
|
||||
// either, deferred or not.
|
||||
...(vector.length > 0
|
||||
? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]
|
||||
: []),
|
||||
new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration)
|
||||
)
|
||||
plan.touchedNouns.push(id)
|
||||
|
|
@ -16032,6 +16062,63 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return !this.pluginRegistry.hasProvider('embeddings')
|
||||
}
|
||||
|
||||
/**
|
||||
* SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's
|
||||
* persisted vector from a real (non-empty) vector to the "unvectored"
|
||||
* empty-array shape: the vector record is rewritten to `[]`, the row is
|
||||
* removed from the vector index (if present), and the vectored-noun
|
||||
* ledger (`getCanonicalCounts().vectors.all`) is decremented through the
|
||||
* sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the
|
||||
* coverage ledger never silently drifts.
|
||||
*
|
||||
* Exists SOLELY for the VFS root zero-norm migration (see
|
||||
* `VirtualFileSystem.doInitializeRoot()`, which detects a persisted root
|
||||
* whose vector is the legacy all-zero placeholder and calls this once per
|
||||
* store). This is NOT a general-purpose "clear my vector" API — ordinary
|
||||
* application data has no sanctioned path from vectored back to
|
||||
* unvectored (`update()` refuses an empty vector as a dimension mismatch,
|
||||
* by design). Never call this outside the VFS root migration.
|
||||
*
|
||||
* Idempotent: a noun already unvectored (`vector.length === 0`) or absent
|
||||
* is a no-op — safe to call on every `init()`.
|
||||
*
|
||||
* @param id - The canonical noun id to migrate.
|
||||
* @returns `true` if a migration write happened, `false` if the noun was
|
||||
* already unvectored (or absent) — a no-op.
|
||||
*/
|
||||
async unvectorNounForRootMigration(id: string): Promise<boolean> {
|
||||
const noun = await this.storage.getNoun(id)
|
||||
if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false
|
||||
|
||||
await this.persistSingleOp({ nouns: [id] }, async (tx) => {
|
||||
// Rewrite the vector leg to the unvectored shape. Placeholder adjacency
|
||||
// (mirrors update()'s own SaveNounOperation staging) — the op preserves
|
||||
// stored graph state when `connections.size === 0`.
|
||||
tx.addOperation(
|
||||
new SaveNounOperation(this.storage, {
|
||||
id,
|
||||
vector: [],
|
||||
connections: new Map(),
|
||||
level: 0
|
||||
})
|
||||
)
|
||||
// Remove from the vector index — safe even if the row was never
|
||||
// actually indexed (RemoveFromVectorIndexOperation's removeItem is a
|
||||
// no-op when the id is absent).
|
||||
tx.addOperation(
|
||||
new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration)
|
||||
)
|
||||
})
|
||||
|
||||
// Vectored-noun ledger: this migration carries a vector write with no
|
||||
// accompanying metadata operation (metadata is untouched), so the
|
||||
// saveNounMetadata(..., hasVector) seam never fires for it — mirrors the
|
||||
// deferred-embed LANDING path's use of the narrow storage hook, in
|
||||
// reverse.
|
||||
await this.storage.noteVectorUnlanded?.(id)
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup embedder
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue