soulcraftlabs/open-brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 341 Packages Wiki Activity Actions 12

Merge branch 'next/vfs-root-zero-norm'

Browse source
This commit is contained in:
David Snelling 2026-08-27 11:49:50 -07:00
commit 4c7b0fab7a
10 changed files with 516 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

105
src/brainy.ts
View file

@ -25,6 +25,7 @@ import {
} from './storage/brainFormat.js'
import type { BrainFormat } from './storage/brainFormat.js'
import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js'
import { isZeroNormVector } from './utils/distance.js'
import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js'
import {
defaultEmbeddingFunction,
@ -2913,7 +2914,13 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// Ensure dimensions are set (a deferred-embed stub carries no dimension
// information — the worker's real vector goes through the same guard).
if (!deferringEmbed) {
// Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose
// vector is the "unvectored" empty-array shape carries no dimension
// information, deferred or not — an explicit `vector: []` (e.g. the VFS
// root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must
// never pin `this.dimensions` to 0, which would poison every subsequent
// real embed's dimension check for the life of the store.
if (!deferringEmbed && vector.length > 0) {
if (!this.dimensions) {
this.dimensions = vector.length
} else if (vector.length !== this.dimensions) {
@ -3029,10 +3036,15 @@ export class Brainy<T = any> implements BrainyInterface<T> {
}, true)
)
// Operation 3: Add to HNSW index (after entity saved). A deferred
// embed has nothing to index yet — the worker's atomic update
// inserts the real vector.
if (!deferringEmbed) {
// Operation 3: Add to HNSW index (after entity saved). Gated on
// `vector.length > 0`, not `!deferringEmbed`: a deferred embed has
// nothing to index yet (the worker's atomic update inserts the real
// vector later), and an explicit `vector: []` insert (the VFS root's
// zero-norm fix — permanently unvectored plumbing, never embedded)
// is exactly the same "nothing to index yet" shape. The zero-norm
// BELT (a real all-zero vector, non-empty) is enforced inside
// AddToVectorIndexOperation itself — see its JSDoc.
if (vector.length > 0) {
tx.addOperation(
new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)
)
@ -10245,6 +10257,18 @@ export class Brainy<T = any> implements BrainyInterface<T> {
for (const id of nounIds) {
const noun = await snapshotStorage.getNoun(id)
if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) {
// THE ZERO-NORM LAW: a direct provider-write seam (this materializer
// inserts one-by-one, bypassing AddToVectorIndexOperation's own
// belt) — apply the same refusal here rather than handing a false
// attractor to the ephemeral reader's index.
if (isZeroNormVector(noun.vector)) {
prodLog.warn(
`[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` +
`entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` +
`engine boundary (the materialized record is unaffected)`
)
continue
}
await reader.index.addItem({ id: noun.id, vector: noun.vector })
}
}
@ -10519,7 +10543,10 @@ export class Brainy<T = any> implements BrainyInterface<T> {
const vector = deferringEmbed
? []
: params.vector || (await this.embed(params.data))
if (!deferringEmbed) {
// Gated on `vector.length > 0` — see the single-add() insert path's
// matching comment: an explicit `vector: []` carries no dimension
// information either, deferred or not.
if (!deferringEmbed && vector.length > 0) {
if (!this.dimensions) {
this.dimensions = vector.length
} else if (vector.length !== this.dimensions) {
@ -10601,9 +10628,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// for a deferred embed (stub vector `[]`; counted later at landing).
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0),
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
...(deferringEmbed
? []
: [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]),
// Gated on `vector.length > 0` — see the single-add() insert path's
// matching comment: an explicit `vector: []` has nothing to index
// either, deferred or not.
...(vector.length > 0
? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]
: []),
new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration)
)
plan.touchedNouns.push(id)
@ -16032,6 +16062,63 @@ export class Brainy<T = any> implements BrainyInterface<T> {
return !this.pluginRegistry.hasProvider('embeddings')
}
/**
* SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's
* persisted vector from a real (non-empty) vector to the "unvectored"
* empty-array shape: the vector record is rewritten to `[]`, the row is
* removed from the vector index (if present), and the vectored-noun
* ledger (`getCanonicalCounts().vectors.all`) is decremented through the
* sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the
* coverage ledger never silently drifts.
*
* Exists SOLELY for the VFS root zero-norm migration (see
* `VirtualFileSystem.doInitializeRoot()`, which detects a persisted root
* whose vector is the legacy all-zero placeholder and calls this once per
* store). This is NOT a general-purpose "clear my vector" API — ordinary
* application data has no sanctioned path from vectored back to
* unvectored (`update()` refuses an empty vector as a dimension mismatch,
* by design). Never call this outside the VFS root migration.
*
* Idempotent: a noun already unvectored (`vector.length === 0`) or absent
* is a no-op — safe to call on every `init()`.
*
* @param id - The canonical noun id to migrate.
* @returns `true` if a migration write happened, `false` if the noun was
* already unvectored (or absent) — a no-op.
*/
async unvectorNounForRootMigration(id: string): Promise<boolean> {
const noun = await this.storage.getNoun(id)
if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false
await this.persistSingleOp({ nouns: [id] }, async (tx) => {
// Rewrite the vector leg to the unvectored shape. Placeholder adjacency
// (mirrors update()'s own SaveNounOperation staging) — the op preserves
// stored graph state when `connections.size === 0`.
tx.addOperation(
new SaveNounOperation(this.storage, {
id,
vector: [],
connections: new Map(),
level: 0
})
)
// Remove from the vector index — safe even if the row was never
// actually indexed (RemoveFromVectorIndexOperation's removeItem is a
// no-op when the id is absent).
tx.addOperation(
new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration)
)
})
// Vectored-noun ledger: this migration carries a vector write with no
// accompanying metadata operation (metadata is untouched), so the
// saveNounMetadata(..., hasVector) seam never fires for it — mirrors the
// deferred-embed LANDING path's use of the narrow storage hook, in
// reverse.
await this.storage.noteVectorUnlanded?.(id)
return true
}
/**
* Setup embedder
*/

21
src/coreTypes.ts
View file

@ -872,6 +872,27 @@ export interface StorageAdapter {
*/
noteVectorLanded?(id: string): Promise<void>
/**
* OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}:
* record that a canonical noun's vector was just REMOVED — rewritten from
* a real (non-empty) vector to the "unvectored" empty-array shape. Exists
* for the ONE sanctioned reverse migration this engine supports: the VFS
* root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and
* `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix
* store's all-zero placeholder root vector to `[]` and must decrement
* `vectors.all` through this hook so the coverage ledger never drifts.
* NOT a general-purpose "I removed a vector" callback — ordinary
* application data has no sanctioned path from vectored back to
* unvectored (`update()` refuses an empty vector as a dimension
* mismatch by design). Callers MUST call this only when the noun held a
* REAL vector immediately before this write (the caller already holds
* that fact for free, from its own pre-write read — never an added read).
* A backend without vectored-noun tracking is a no-op via this method's
* absence (feature-detected).
* @param id - The noun whose vector was just removed.
*/
noteVectorUnlanded?(id: string): Promise<void>
/**
* Get noun with metadata combined
* @returns Combined HNSWNounWithMetadata or null

18
src/storage/adapters/baseStorageAdapter.ts
View file

@ -1152,6 +1152,24 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
})
}
/**
* OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}):
* the mirror of {@link noteVectorLanded} — record a noun's vector was just
* REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a
* caller that (incorrectly) fires this for a noun already unvectored would
* otherwise drive the ledger negative — clamped defensively, matching the
* delete path's `if (this.totalVectoredNounCount > 0)` guard.
* @param id - The noun whose vector was just removed (retained for a
* future narration seam; the count itself needs no id-keyed state).
*/
async noteVectorUnlanded(id: string): Promise<void> {
void id
if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount--
this.scheduleCountPersist().catch(() => {
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
})
}
/**
* Increment count for entity type - O(1) operation.
* Concurrency is handled by the process-global mutex

55
src/transaction/operations/IndexOperations.ts
View file

@ -13,6 +13,8 @@ import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js'
import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
import type { GraphVerb } from '../../coreTypes.js'
import type { Operation, RollbackAction } from '../types.js'
import { isZeroNormVector } from '../../utils/distance.js'
import { prodLog } from '../../utils/logger.js'
/**
* Backend identity stamped into an operation's emitted `name` string (e.g.
@ -88,6 +90,30 @@ export class AddToVectorIndexOperation implements Operation {
}
async execute(): Promise<RollbackAction> {
// THE ZERO-NORM LAW (the live provider-write seam's belt): a zero-norm
// vector is not a vector — it never crosses an engine boundary. This
// engine's own cosine distance treats an all-zero vector safely (a
// zero-norm operand always scores MAXIMUM distance, see
// {@link isZeroNormVector}'s JSDoc), but a downstream engine serving
// squared-euclidean distance cannot tell it apart from a legitimate
// origin point — a false attractor that silently darkens real results.
// The canonical write already landed (SaveNoun/SaveNounMetadata
// operations are staged ahead of this one in every caller) — only the
// INDEX INSERT is refused here, loudly, never a throw. A length-0
// vector is the unrelated "unvectored" shape and is skipped silently
// (the same contract callers already rely on for deferred embeds).
if (this.vector.length === 0) {
return async () => {}
}
if (isZeroNormVector(this.vector)) {
prodLog.warn(
`[vector-index] refusing to index a zero-norm vector for entity ${this.id} — ` +
`a zero-norm vector is not a vector and never crosses an engine boundary ` +
`(the canonical write is unaffected; only the vector-index insert is skipped)`
)
return async () => {}
}
// Check if item already exists (for rollback decision)
const existed = await this.itemExists(this.id)
@ -263,6 +289,35 @@ export class ReplaceInVectorIndexOperation implements Operation {
// One commit generation for the whole replace (both branches + rollback).
const generation = this.generationFn?.()
// THE ZERO-NORM LAW (see AddToVectorIndexOperation's matching JSDoc): a
// real all-zero replacement vector must never land in the index — refuse
// loudly, canonical write unaffected. The row must not be left stale
// either: if it was genuinely indexed under `oldVector`, remove it
// rather than pretend the old vector still describes the row. A
// length-0 `newVector` (the unrelated "unvectored" shape) is handled the
// same way, silently — no caller today reaches this with an empty
// replacement (update() rejects a dimension-mismatched empty vector),
// but the seam stays consistent in case one ever legitimately does.
if (isZeroNormVector(this.newVector) || this.newVector.length === 0) {
const wasIndexed = this.oldVector.length > 0 && !isZeroNormVector(this.oldVector)
if (isZeroNormVector(this.newVector)) {
prodLog.warn(
`[vector-index] refusing to replace with a zero-norm vector for entity ${this.id} — ` +
`a zero-norm vector is not a vector and never crosses an engine boundary ` +
`(the canonical write is unaffected; the row is removed from the vector index instead)`
)
}
if (wasIndexed) {
await this.index.removeItem(this.id, generation)
}
return async () => {
// Restore the declared before-state.
if (wasIndexed) {
await this.index.addItem({ id: this.id, vector: this.oldVector }, generation)
}
}
}
if (typeof index.updateItem === 'function') {
// Atomic path: one in-place call, the row never leaves the index.
await index.updateItem({ id: this.id, vector: this.newVector }, generation)

23
src/utils/distance.ts
View file

@ -65,6 +65,29 @@ export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number =
return 1 - similarity
}
/**
* True when `vector` is a REAL (non-empty) all-zero vector — the "false
* attractor" shape this engine's own cosine distance treats safely (a
* zero-norm operand always scores the MAXIMUM distance, see
* {@link cosineDistance}) but a downstream engine serving squared-euclidean
* distance cannot distinguish from a legitimate origin point. THE LAW: a
* zero-norm vector is not a vector — it never crosses an engine boundary
* (never handed to a vector-index provider as a searchable item).
*
* A length-0 vector is the UNRELATED "unvectored, not yet embedded" shape
* (the deferred-embed stub, a permanently-vectorless system row) and is
* deliberately NOT zero-norm here — callers checking for "nothing to index"
* should test `vector.length === 0` separately; this only flags the
* dangerous non-empty all-zero case.
*/
export function isZeroNormVector(vector: readonly number[]): boolean {
if (vector.length === 0) return false
for (let i = 0; i < vector.length; i++) {
if (vector[i] !== 0) return false
}
return true
}
/**
* Calculates the Manhattan (L1) distance between two vectors.
* Lower values indicate higher similarity.

10
src/utils/paramValidation.ts
View file

@ -588,8 +588,14 @@ export function validateAddParams(params: AddParams): void {
)
}
// Validate vector dimensions if provided
if (params.vector) {
// Validate vector dimensions if provided. A length-0 vector is the
// "unvectored" shape — an explicit `vector: []` (e.g. the VFS root's
// permanently-vectorless creation, see
// VirtualFileSystem.doInitializeRoot()'s zero-norm fix) carries no
// dimension information, exactly like an absent vector or a deferred
// embed's internal stub, so it is exempt from the dimension check rather
// than refused as a "0-dimensional vector".
if (params.vector && params.vector.length > 0) {
const config = ValidationConfig.getInstance()
if (params.vector.length !== config.maxVectorDimensions) {
throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`)

100
src/vfs/VirtualFileSystem.ts
View file

@ -11,6 +11,7 @@ import { v4 as uuidv4 } from '../universal/uuid.js'
import { Brainy } from '../brainy.js'
import { Entity, AddParams, RelateParams, FindParams, Relation } from '../types/brainy.types.js'
import { NounType, VerbType } from '../types/graphTypes.js'
import { isZeroNormVector } from '../utils/distance.js'
import { PathResolver } from './PathResolver.js'
import { mimeDetector } from './MimeTypeDetector.js'
import {
@ -89,16 +90,6 @@ export class VirtualFileSystem implements IVirtualFileSystem {
// Uses deterministic UUID format for storage compatibility
private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000'
// OPEN-PATH FIX: the dimension of the placeholder vector given to the VFS
// root when it is first created (see `doInitializeRoot`). Mirrors the
// built-in WASM embedding engine's fixed, hardcoded output size
// (all-MiniLM-L6-v2 — see src/embeddings/candle-wasm/src/lib.rs
// HIDDEN_SIZE and src/embeddings/EmbeddingManager.ts). Deliberately NOT
// derived from `brain.dimensions` — this constant only applies to the
// default-embedder branch, where the true output dimension is this fixed
// value by construction, never a moving target.
private static readonly VFS_ROOT_VECTOR_DIMENSIONS = 384
/**
* Construct a VFS bound to a Brainy instance.
*
@ -241,9 +232,11 @@ export class VirtualFileSystem implements IVirtualFileSystem {
private async doInitializeRoot(): Promise<string> {
const rootId = VirtualFileSystem.VFS_ROOT_ID
// Try to get existing root by fixed ID (O(1) lookup, not query)
// Try to get existing root by fixed ID (O(1) lookup, not query).
// includeVectors: true — the zero-norm migration below (leg 2) needs to
// inspect the persisted vector to detect the legacy placeholder shape.
try {
const existingRoot = await this.brain.get(rootId)
const existingRoot = await this.brain.get(rootId, { includeVectors: true })
if (existingRoot) {
// Root exists - verify metadata is correct
@ -260,6 +253,34 @@ export class VirtualFileSystem implements IVirtualFileSystem {
})
}
// ZERO-NORM ROOT MIGRATION (one-time): a pre-fix store persisted the
// root with a REAL all-zero placeholder vector — lawful inside
// brainy (cosineDistance treats a zero-norm operand as MAXIMUM
// distance, see src/utils/distance.ts) but a "false attractor" for a
// downstream engine serving squared-euclidean distance, which cannot
// tell an all-zero vector apart from a legitimate origin point (a
// production incident silently darkened 150+ rows in a partner
// engine's index this way). THE LAW: a zero-norm vector is not a
// vector — it never crosses an engine boundary. Detect the legacy
// shape via NORM, not length or dimension (any real all-zero vector
// qualifies, not just the historical 384-dim one), and rewrite it to
// the "unvectored" `[]` shape through the sanctioned migration path
// (Brainy.unvectorNounForRootMigration — see its JSDoc), which keeps
// `getCanonicalCounts().vectors.all` honest and removes the row from
// the vector index. Idempotent: a store already on the new shape
// (vector.length === 0) takes the false branch below on every
// subsequent init() — a permanent no-op, not a one-time flag.
const existingVector = existingRoot.vector ?? []
if (existingVector.length > 0 && isZeroNormVector(existingVector)) {
const migrated = await this.brain.unvectorNounForRootMigration(rootId)
if (migrated) {
console.log(
'VFS: migrated root vector from the legacy all-zero placeholder to the ' +
'unvectored shape (zero-norm vectors never cross an engine boundary)'
)
}
}
return rootId
}
} catch (error) {
@ -277,31 +298,42 @@ export class VirtualFileSystem implements IVirtualFileSystem {
// meant every writer's FIRST-EVER open forced the process-global WASM
// engine to cold-compile its model (measured 90-140s on throttled
// CPUs) before the brain could even finish init(). This branch only
// runs once per store (the root already exists — with a real vector —
// in every previously-opened production store; reopening never re-adds
// it), so the fix applies only to brand-new stores.
// runs once per store (a previously-opened store already has a root —
// see the migration above for the pre-fix shape — reopening never
// re-adds it), so the fix applies only to brand-new stores.
//
// Chosen fix: an explicit all-zero vector, not `deferEmbedding: true`.
// `deferEmbedding` looked attractive (ack now, embed later) but its
// landing path (`kickEmbedWorker()`, called synchronously right after
// commit — see brainy.ts add()/update()) would still force the WASM
// engine to cold-compile within milliseconds of open, just off the
// awaited path instead of never paying it at all — worse than the
// explicit-vector path, which never touches the engine for this row.
// An all-zero vector is safe: `cosineDistance` (src/utils/distance.ts)
// explicitly returns the MAXIMUM distance whenever either operand's
// norm is zero, so the root can never rank ahead of real content in a
// similarity search, and HNSW indexes it like any other vector.
// ZERO-NORM LAW (current shape, superseding the historical all-zero
// placeholder): the root's vector is `[]` — the SAME "unvectored"
// empty-array shape used for a deferred embed's stub and any other
// not-yet-embedded row — never a real all-zero vector. A zero-norm
// vector is lawful inside brainy (`cosineDistance`, see
// src/utils/distance.ts, returns the MAXIMUM distance whenever either
// operand's norm is zero) but is a "false attractor" for a downstream
// engine serving squared-euclidean distance, which cannot tell a real
// all-zero vector apart from a legitimate origin point — it silently
// darkened 150+ rows in a partner engine's index in production. THE
// LAW: a zero-norm vector is not a vector — it never crosses an engine
// boundary. `vector: []` achieves the SAME cold-compile avoidance the
// original placeholder did (`add()`'s dimension-pin and HNSW-insert
// gates both key off `vector.length > 0`, so an empty vector never
// calls embed(), never pins `brain.dimensions`, and never reaches the
// vector index — see brainy.ts add()'s matching comments) while never
// persisting a searchable zero vector for a downstream engine to trip
// over. Deliberately NOT `deferEmbedding: true`: that flag's landing
// path (`kickEmbedWorker()`, called synchronously right after commit —
// see brainy.ts add()/update()) would still force the WASM engine to
// cold-compile within milliseconds of open (just off the awaited path
// instead of never paying it at all) AND would eventually embed the
// root's data for real, which this fix forbids — the root must NEVER
// be embedded, not merely "not yet".
//
// Only the default WASM engine gets this treatment — its output
// dimension (384) is fixed and hardcoded, so the placeholder can never
// mis-pin `brain.dimensions` for it. A plugin-registered native
// 'embeddings' provider has no cold-compile cost AND may use a
// different dimension, so it keeps embedding the root for real (same
// as before this fix) rather than risk pinning the wrong dimension
// ahead of the caller's own first real embed.
// Only the default WASM engine gets this treatment — a plugin-
// registered native 'embeddings' provider has no cold-compile cost and
// may use a different dimension, so it keeps embedding the root for
// real (same as before this fix) rather than leave Brainy's own
// plumbing permanently unvectored on a store where embedding is cheap.
const rootVector = this.brain.usesDefaultWasmEmbedder()
? new Array(VirtualFileSystem.VFS_ROOT_VECTOR_DIMENSIONS).fill(0)
? ([] as number[])
: undefined
await this.brain.add({