diff --git a/CHANGELOG.md b/CHANGELOG.md index ec925642..0b154af5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) + +- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) +- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) +- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1) +- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4) +- Merge branch 'next/enumeration-identity-rekey' (204d74c1) +- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16) +- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a) +- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab) +- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9) +- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245) +- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee) +- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb) +- chore(release): 10.4.1-rc.1 (7870dc40) + + ### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md index 18bf7010..96199f11 100644 --- a/docs/concepts/index-health.md +++ b/docs/concepts/index-health.md @@ -68,6 +68,19 @@ a maintenance window, a divergence `repairIndex()` will clean up on its own schedule. `serving: false` is not benign. It means this provider is refusing to answer, on its own word, right now. +**How a failure gets its grade — the serving law.** A provider grades `heal` by +one question only: *could an answer be wrong?* — never *how expensive is the +fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post +exactly what the ledger names, reads serving throughout); it can never withhold +serving just because healing it takes work. `serving` is withheld only by a +small, named set of rebuild-graded conditions — the index not initialized, its +durable state absent, a manifest naming files that are not resident, a replay +that did not complete cleanly — the states in which an answer could genuinely be +wrong. And a read is only ever refused by the family it actually consults: a +metadata filter is answered by the metadata index alone, vector search by the +vector index, traversal by the graph index — one family's refusal never blocks +another family's reads. + ## Reads refuse — they never rebuild A query that reaches a not-serving provider does not trigger a rebuild from inside diff --git a/package-lock.json b/package-lock.json index 70635835..889c67c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 4125b073..214feeac 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.1", + "version": "10.4.2-rc.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/buildEmbeddedPatterns.ts b/scripts/buildEmbeddedPatterns.ts index 73e51224..c046df45 100644 --- a/scripts/buildEmbeddedPatterns.ts +++ b/scripts/buildEmbeddedPatterns.ts @@ -10,6 +10,7 @@ import { TransformerEmbedding } from '../src/utils/embedding.js' import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' +import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -97,13 +98,22 @@ async function buildEmbeddedPatterns() { // Convert to base64 for embedding in TypeScript const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - + + // Deterministic stamp: derived from the git commit time of this + // generator's inputs, never from wall-clock time — two builds of the + // same source tree must produce byte-identical output. + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') + const generatedStamp = resolveDeterministicStamp( + [path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath], + outputPath + ) + // Generate TypeScript file with everything embedded const tsContent = `/** * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${new Date().toISOString()} + * Generated: ${generatedStamp} * Patterns: ${libraryData.patterns.length} * Coverage: 94-98% of all queries * @@ -197,7 +207,6 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length} ` // Write the TypeScript file - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/buildTypeEmbeddings.ts b/scripts/buildTypeEmbeddings.ts index 61bcf238..688d6ac1 100644 --- a/scripts/buildTypeEmbeddings.ts +++ b/scripts/buildTypeEmbeddings.ts @@ -11,6 +11,7 @@ import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' import { NounType, VerbType } from '../src/types/graphTypes.js' +import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -373,12 +374,24 @@ async function buildTypeEmbeddings() { const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') + // Deterministic stamp: derived from the git commit time of this + // generator's inputs, never from wall-clock time — two builds of the + // same source tree must produce byte-identical output. + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') + const generatedStamp = resolveDeterministicStamp( + [ + path.join(__dirname, 'buildTypeEmbeddings.ts'), + path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts') + ], + outputPath + ) + // Generate TypeScript file const tsContent = `/** * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${new Date().toISOString()} + * Generated: ${generatedStamp} * Noun Types: ${nounTypes.length} * Verb Types: ${verbTypes.length} * @@ -395,7 +408,7 @@ export const TYPE_METADATA = { verbTypes: ${verbTypes.length}, totalTypes: ${totalTypes}, embeddingDimensions: ${embeddingDim}, - generatedAt: "${new Date().toISOString()}", + generatedAt: "${generatedStamp}", sizeBytes: { embeddings: ${buffer.byteLength}, base64: ${base64.length} @@ -494,7 +507,6 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n ` // Write the TypeScript file - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/lib/deterministicStamp.ts b/scripts/lib/deterministicStamp.ts new file mode 100644 index 00000000..c2a66604 --- /dev/null +++ b/scripts/lib/deterministicStamp.ts @@ -0,0 +1,118 @@ +/** + * Deterministic generation-stamp resolution for Brainy's build-time code + * generators. + * + * Two builds of the same source tree must produce byte-identical output. + * A wall-clock stamp (`new Date()`) breaks that guarantee, so every + * generator that writes a "Generated:" header or a `generatedAt` field + * into its output must resolve the stamp through this module instead. + * + * Resolution order: + * 1. The newest git commit timestamp among the generator's input files + * (the generator script itself always counts as an input). + * 2. If git metadata is unavailable (for example, building from a + * published npm tarball with no `.git` directory), the stamp already + * recorded in the previously generated output file. + * 3. If neither is available, the fixed epoch string + * `1970-01-01T00:00:00.000Z`. + * + * Every fallback logs a line to stderr — deterministic degradation is + * loud, never a silent divergence. + */ + +import { execFileSync } from 'child_process' +import * as fs from 'fs' + +const EPOCH_STAMP = '1970-01-01T00:00:00.000Z' +const STAMP_PATTERN = /\*\s*Generated:\s*(\S+)/ + +/** + * Resolve the deterministic stamp for a generator run. + * + * @param inputPaths Absolute paths to every file whose content determines + * the generator's output, including the generator script itself. + * @param previousOutputPath Absolute path to the previously generated + * file, used for the existing-stamp fallback when git is unavailable. + * @returns An ISO-8601 timestamp string that is deterministic for a given + * source tree. + */ +export function resolveDeterministicStamp( + inputPaths: string[], + previousOutputPath: string +): string { + const gitStamp = newestGitCommitTimestamp(inputPaths) + if (gitStamp) { + return gitStamp + } + + const existingStamp = readExistingStamp(previousOutputPath) + if (existingStamp) { + process.stderr.write( + `[deterministic-stamp] no git commit history found for generator inputs; ` + + `reusing existing stamp from ${previousOutputPath}: ${existingStamp}\n` + ) + return existingStamp + } + + process.stderr.write( + `[deterministic-stamp] no git commit history and no previous output at ` + + `${previousOutputPath}; falling back to fixed epoch stamp ${EPOCH_STAMP}\n` + ) + return EPOCH_STAMP +} + +/** + * Find the newest git commit timestamp among the given input paths. + * Returns null if git is unavailable, the tree is not a git repository, + * or none of the inputs have any commit history yet. + */ +function newestGitCommitTimestamp(inputPaths: string[]): string | null { + let newest: string | null = null + + for (const inputPath of inputPaths) { + if (!fs.existsSync(inputPath)) { + continue + } + + let out: string + try { + out = execFileSync( + 'git', + ['log', '-1', '--format=%cI', '--', inputPath], + { stdio: ['ignore', 'pipe', 'ignore'] } + ) + .toString() + .trim() + } catch { + // git missing, not a repository, or no permissions — handled by the + // caller's fallback chain. + continue + } + + if (!out) { + // Path exists but has no commit history yet (e.g. newly created, + // uncommitted file). + continue + } + + if (!newest || new Date(out).getTime() > new Date(newest).getTime()) { + newest = out + } + } + + return newest +} + +/** + * Parse the `* Generated: ` header out of a previously + * generated file, if one exists. + */ +function readExistingStamp(outputPath: string): string | null { + if (!fs.existsSync(outputPath)) { + return null + } + + const content = fs.readFileSync(outputPath, 'utf-8') + const match = content.match(STAMP_PATTERN) + return match ? match[1] : null +} diff --git a/src/brainy.ts b/src/brainy.ts index 6289e8f2..ed958a67 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -25,6 +25,7 @@ import { } from './storage/brainFormat.js' import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' +import { isZeroNormVector } from './utils/distance.js' import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, @@ -395,6 +396,15 @@ interface PlannedTransact { * marker outlives its write. */ markerRecords: FactMarkerRecord[] + /** + * Ids the batch's `{ op: 'update' }` unvector door (`vector: []`) needs to + * decrement on the vectored-noun ledger — consumed by `transact()` with a + * proper `await this.storage.noteVectorUnlanded?.(id)` per id, AFTER + * `commitTransaction` resolves (never for a rejected batch). Kept separate + * from `postCommit` (`Array<() => void>`, called synchronously, fire-and- + * forget) because the ledger hook is async and must be awaited. + */ + vectorUnlands: string[] } /** @@ -1503,6 +1513,15 @@ export class Brainy implements BrainyInterface { }).backfillBlobHistoryRefCountsIfNeeded() } + // LEG C (zero-norm/unvector-door law): migrate a legacy zero-norm VFS + // root BEFORE the vector-leg open gate below ever compares the + // canonical vectored-noun count against the vector index's size — see + // migrateLegacyZeroNormVfsRootIfNeeded's JSDoc for why this is a safe + // O(1) exception to "nothing at open may scale with brain size", and + // why it must run here rather than waiting on VirtualFileSystem's own + // (VFS-instance-gated) lazy migration. + await this.migrateLegacyZeroNormVfsRootIfNeeded() + // Rebuild indexes if needed for existing data. Runs to completion before // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true @@ -1517,8 +1536,9 @@ export class Brainy implements BrainyInterface { // cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph // index construction, the eager cold-load, id-resolver + connections- // codec wiring, crash-recovery index rebuild, the replay-gap check, - // legacy VFS blob adoption, blob-history backfill, and the - // rebuildIndexesIfNeeded() gate + migration check. + // legacy VFS blob adoption, blob-history backfill, the legacy + // zero-norm VFS root migration, and the rebuildIndexesIfNeeded() gate + // + migration check. markPhase('index-init-gate') // Register shutdown hooks for graceful count flushing (once globally) @@ -1814,7 +1834,11 @@ export class Brainy implements BrainyInterface { if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { throw error } - throw new Error(`Failed to initialize Brainy: ${error}`) + // Wrap with the original as `cause` so the originating frame (a plugin's + // own file:line, e.g. a provider boot failure) survives to the caller's + // log — a plain string interpolation discards both stack and cause. + const message = error instanceof Error ? error.message : String(error) + throw new Error(`Failed to initialize Brainy: ${message}`, { cause: error }) } } @@ -2907,13 +2931,43 @@ export class Brainy implements BrainyInterface { // vector shape, is structurally impossible). The background worker // embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed + let vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) + // THE ZERO-NORM LAW (canonical write side): a zero-norm vector is not a + // vector — it never crosses an engine boundary (the engine pair's seam + // law). This engine's own cosine distance treats an all-zero vector + // safely (a zero-norm operand always scores MAXIMUM distance — see + // isZeroNormVector's JSDoc), but a downstream engine serving squared- + // euclidean distance cannot tell it apart from a legitimate origin + // point — a false attractor that silently darkened 150+ rows in a + // production deployment. The index belt (AddToVectorIndexOperation) + // already refuses to INDEX a zero-norm vector, but until now the + // CANONICAL write still persisted it and the vectored-noun ledger + // counted it — so a near-empty store whose only vectored row was + // zero-norm read "canonical vectored > 0, index size 0" and threw a + // not-ready error at open. Normalize HERE, before the dimension pin, + // the vectored-ledger flag (`SaveNounMetadataOperation`'s `hasVector`), + // and the index ops below ever see it, so it persists as the sanctioned + // "unvectored" `[]` shape instead — the canonical write still succeeds. + if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { + prodLog.warn( + `[Brainy] add(): entity ${id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + vector = [] + } + // Ensure dimensions are set (a deferred-embed stub carries no dimension // information — the worker's real vector goes through the same guard). - if (!deferringEmbed) { + // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose + // vector is the "unvectored" empty-array shape carries no dimension + // information, deferred or not — an explicit `vector: []` (e.g. the VFS + // root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must + // never pin `this.dimensions` to 0, which would poison every subsequent + // real embed's dimension check for the life of the store. + if (!deferringEmbed && vector.length > 0) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -3029,10 +3083,15 @@ export class Brainy implements BrainyInterface { }, true) ) - // Operation 3: Add to HNSW index (after entity saved). A deferred - // embed has nothing to index yet — the worker's atomic update - // inserts the real vector. - if (!deferringEmbed) { + // Operation 3: Add to HNSW index (after entity saved). Gated on + // `vector.length > 0`, not `!deferringEmbed`: a deferred embed has + // nothing to index yet (the worker's atomic update inserts the real + // vector later), and an explicit `vector: []` insert (the VFS root's + // zero-norm fix — permanently unvectored plumbing, never embedded) + // is exactly the same "nothing to index yet" shape. The zero-norm + // BELT (a real all-zero vector, non-empty) is enforced inside + // AddToVectorIndexOperation itself — see its JSDoc. + if (vector.length > 0) { tx.addOperation( new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration) ) @@ -3607,25 +3666,53 @@ export class Brainy implements BrainyInterface { // often the host writes. const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged + + // THE ZERO-NORM LAW (canonical write side) — see add()'s matching + // comment: an explicit REAL all-zero vector is not a vector. Normalize + // to the sanctioned "unvectored" `[]` shape BEFORE the dimension + // check, the unvector-door decision below, and the index ops ever see + // it — a local copy; `params.vector` itself is never mutated. + let explicitVector = params.vector + if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { + prodLog.warn( + `[Brainy] update(): entity ${params.id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + explicitVector = [] + } + + // THE SANCTIONED UNVECTOR DOOR: `explicitVector` at length 0 (an + // explicit `vector: []`, or a real all-zero vector just normalized + // above) is an instruction to remove the vector NOW — never "please + // embed". `validateUpdateParams` already refuses combining it with + // `deferEmbedding: true` (an empty array is truthy, so that guard + // fires unconditionally on any explicit `vector`). Idempotent on an + // already-unvectored row: the ledger decrement near the end of this + // method is gated on the PRIOR vector actually having been real. + const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 + // MT5 deferred re-embedding: the OLD vector keeps serving semantic // search — stale-but-present, never absent (the flicker law) — until // the background worker embeds the new data and swaps it atomically. const deferringEmbed = - params.deferEmbedding === true && hasNewData && !params.vector - if (params.vector) { - if (this.dimensions && params.vector.length !== this.dimensions) { + params.deferEmbedding === true && hasNewData && !explicitVector + if (explicitVector) { + // A length-0 explicit vector (the unvector door) carries no + // dimension information — exempt from the check, mirroring add()'s + // own `vector.length > 0` gate on the dimension pin. + if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` ) } - vector = params.vector + vector = explicitVector } else if (hasNewData && !deferringEmbed) { vector = await this.embed(params.data) } // A deferred data change does NOT reindex now (the vector is unchanged; // the worker's atomic swap carries the real reindex later). const needsReindexing = Boolean( - (hasNewData && !deferringEmbed) || params.type || params.vector + (hasNewData && !deferringEmbed) || params.type || explicitVector ) // Always update the noun with new metadata @@ -3719,6 +3806,22 @@ export class Brainy implements BrainyInterface { ? [this.enqueuePendingEmbed(params.id)] : undefined + // Leg D — the unvector door clears a PENDING deferred-embed marker: + // without this, the worker would later embed this row's current data + // and silently re-vector it, defeating the caller's explicit "remove + // the vector now" instruction. The clear rides THIS SAME commit fact + // (an `embed.landed` record with an empty vector — the recovery fold + // disarms a pending marker on ANY `embed.landed` for the id, + // regardless of the vector it carries), so a crash between the write + // and the in-memory clear below still recovers disarmed. Mutually + // exclusive with `embedMarkers` above: `deferringEmbed` requires an + // ABSENT `explicitVector`, so the two branches never both apply. + const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) + const commitRecords: FactMarkerRecord[] | undefined = + embedMarkers ?? (clearsPendingEmbed + ? [{ type: 'embed.landed', id: params.id, vector: [] }] + : undefined) + // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { @@ -3807,7 +3910,33 @@ export class Brainy implements BrainyInterface { } } ] - : undefined, embedMarkers) + : undefined, commitRecords) + + // Leg D continued — the in-memory pending-embed clear runs only AFTER + // the commit above actually succeeded (an aborted update must not + // disarm a marker whose durable `embed.landed` twin was never + // written). + if (clearsPendingEmbed) { + this.clearPendingEmbed(params.id) + prodLog.warn( + `[Brainy] update(): entity ${params.id} had a pending deferred embed — ` + + `the unvector door cleared it ('vector: []' is an explicit instruction, ` + + `never "please embed").` + ) + } + + // Leg D — vectored-ledger decrement for the sanctioned unvector door. + // update()'s own metadata write goes through UpdateNounMetadataOperation + // (isNew=false), so the saveNounMetadata(..., hasVector) seam never + // fires here — noteVectorUnlanded is the ONLY seam, the same + // sanctioned hook unvectorNounForRootMigration() uses. Gated on the + // PRIOR vector having actually been real (non-empty, non-zero-norm): + // an already-unvectored row's second call is a true no-op — no + // decrement, matching the ledger-exactness law (never double-count, + // never drift negative). + if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { + await this.storage.noteVectorUnlanded?.(params.id) + } // Aggregation hook (outside transaction — derived data). `existing` is // the full get() view — every reserved field top-level — and must be @@ -9163,6 +9292,15 @@ export class Brainy implements BrainyInterface { hook() } + // Leg D — vectored-ledger decrements for this batch's unvector-door + // updates (see planTxUpdate's matching comment), applied after the + // commit point and properly awaited (unlike `postCommit`'s synchronous + // fire-and-forget hooks) — each is the same sanctioned hook + // unvectorNounForRootMigration() uses. + for (const id of plan.vectorUnlands) { + await this.storage.noteVectorUnlanded?.(id) + } + // Change feed: the batch's events share its single committed generation. // A rejected batch throws at commitTransaction and never reaches here. this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) @@ -10245,6 +10383,18 @@ export class Brainy implements BrainyInterface { for (const id of nounIds) { const noun = await snapshotStorage.getNoun(id) if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) { + // THE ZERO-NORM LAW: a direct provider-write seam (this materializer + // inserts one-by-one, bypassing AddToVectorIndexOperation's own + // belt) — apply the same refusal here rather than handing a false + // attractor to the ephemeral reader's index. + if (isZeroNormVector(noun.vector)) { + prodLog.warn( + `[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` + + `entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` + + `engine boundary (the materialized record is unaffected)` + ) + continue + } await reader.index.addItem({ id: noun.id, vector: noun.vector }) } } @@ -10394,7 +10544,8 @@ export class Brainy implements BrainyInterface { casUpdates: [], createdNouns: new Set(), changeEvents: [], - markerRecords: [] + markerRecords: [], + vectorUnlands: [] } for (const op of ops) { @@ -10516,10 +10667,26 @@ export class Brainy implements BrainyInterface { // marker-less committed row would be a silently missing vector, which is // the disallowed direction). The background worker embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - const vector = deferringEmbed + let vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) - if (!deferringEmbed) { + + // THE ZERO-NORM LAW — see the single-add() insert path's matching + // comment (a zero-norm vector is not a vector; never crosses an engine + // boundary). Normalized here BEFORE the dimension pin and the + // vectored-ledger `hasVector` flag below ever see it. + if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { + prodLog.warn( + `[Brainy] transact add: entity ${id} was given an explicit all-zero vector — ` + + `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + vector = [] + } + + // Gated on `vector.length > 0` — see the single-add() insert path's + // matching comment: an explicit `vector: []` carries no dimension + // information either, deferred or not. + if (!deferringEmbed && vector.length > 0) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -10601,9 +10768,12 @@ export class Brainy implements BrainyInterface { // for a deferred embed (stub vector `[]`; counted later at landing). new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), - ...(deferringEmbed - ? [] - : [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]), + // Gated on `vector.length > 0` — see the single-add() insert path's + // matching comment: an explicit `vector: []` has nothing to index + // either, deferred or not. + ...(vector.length > 0 + ? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)] + : []), new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration) ) plan.touchedNouns.push(id) @@ -10683,17 +10853,62 @@ export class Brainy implements BrainyInterface { const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged let vector = existing.vector - if (params.vector) { - if (this.dimensions && params.vector.length !== this.dimensions) { + + // THE ZERO-NORM LAW + THE SANCTIONED UNVECTOR DOOR — transact() mirror + // of update()'s matching block: an explicit REAL all-zero vector + // normalizes to `[]` (never crosses an engine boundary), and an + // explicit `vector: []` (post-normalization) is the sanctioned unvector + // instruction, exempt from the dimension check. `validateUpdateParams` + // already refuses combining it with `deferEmbedding: true`. + let explicitVector = params.vector + if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { + prodLog.warn( + `[Brainy] transact update: entity ${params.id} was given an explicit all-zero ` + + `vector — a zero-norm vector is not a vector; persisted unvectored ([]) instead.` + ) + explicitVector = [] + } + const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 + + if (explicitVector) { + if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` ) } - vector = params.vector + vector = explicitVector } else if (hasNewData) { vector = await this.embed(params.data) } - const needsReindexing = Boolean(hasNewData || params.type || params.vector) + const needsReindexing = Boolean(hasNewData || params.type || explicitVector) + + // Leg D — the unvector door clears a PENDING deferred-embed marker (see + // update()'s matching comment for the full rationale): the durable + // clear (an `embed.landed` record, empty vector) rides the batch's ONE + // commit fact via `plan.markerRecords`; the in-memory clear is deferred + // to `plan.postCommit` so an aborted batch never disarms a marker whose + // durable twin was never written. + const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) + if (clearsPendingEmbed) { + plan.markerRecords.push({ type: 'embed.landed', id: params.id, vector: [] }) + plan.postCommit.push(() => { + this.clearPendingEmbed(params.id) + prodLog.warn( + `[Brainy] transact update: entity ${params.id} had a pending deferred embed — ` + + `the unvector door cleared it ('vector: []' is an explicit instruction, ` + + `never "please embed").` + ) + }) + } + + // Leg D — vectored-ledger decrement for the sanctioned unvector door, + // deferred to `plan.vectorUnlands` (consumed with a proper `await` in + // `transact()`, AFTER the commit succeeds — see its matching comment). + // Gated on the PRIOR vector having actually been real (non-empty, + // non-zero-norm): idempotent on an already-unvectored row. + if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { + plan.vectorUnlands.push(params.id) + } const newMetadata = params.merge !== false @@ -12397,21 +12612,49 @@ export class Brainy implements BrainyInterface { const metadataStats = await this.metadataIndex.getStats() const graphSize = await this.graphIndex.size() - // 1. Index size parity. HNSW must hold at least one node per indexed entity. - if (hnswSize === metadataStats.totalEntries) { + // 1. Index size parity. HNSW must hold one node per VECTORED noun — the + // vectored-noun ledger (`getCanonicalCounts().vectors.all`), NOT the raw + // metadata-entry count: every store's VFS root is PERMANENTLY unvectored + // (`vector: []` by design — a zero-norm/empty vector never crosses into + // the index, see AddToVectorIndexOperation/JsHnswVectorIndex.rebuild()'s + // matching belts), and a not-yet-landed deferred embed is unvectored + // too. Comparing against total entries counted the always-unvectored + // root as a permanent 1-node "drift" on every VFS-having store — a false + // warn on an otherwise perfectly healthy handoff. `vectors.all` is + // already the documented coverage denominator for exactly this + // comparison (see `CanonicalCounts.vectors`'s JSDoc). Falls back to the + // metadata-entry count when the ledger is unavailable or suspect (a + // storage adapter without the optional hook, or an unrecounted store) — + // never worse than the prior behavior in that case. + const vectorLedgerForParity = await this.storage.getCanonicalCounts?.() + const vectorParityTarget = + vectorLedgerForParity && !vectorLedgerForParity.suspect + ? vectorLedgerForParity.vectors.all + : metadataStats.totalEntries + if (hnswSize === vectorParityTarget) { checks.push({ name: 'index-parity', status: 'pass', - message: `HNSW (${hnswSize}) and metadata index (${metadataStats.totalEntries}) agree.`, - details: { hnswSize, metadataEntries: metadataStats.totalEntries, graphRelationships: graphSize } + message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) agree.`, + details: { + hnswSize, + vectoredNouns: vectorParityTarget, + metadataEntries: metadataStats.totalEntries, + graphRelationships: graphSize + } }) } else { - const drift = Math.abs(hnswSize - metadataStats.totalEntries) + const drift = Math.abs(hnswSize - vectorParityTarget) checks.push({ name: 'index-parity', - status: drift > Math.max(10, metadataStats.totalEntries * 0.01) ? 'fail' : 'warn', - message: `HNSW (${hnswSize}) and metadata (${metadataStats.totalEntries}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, - details: { hnswSize, metadataEntries: metadataStats.totalEntries, drift } + status: drift > Math.max(10, vectorParityTarget * 0.01) ? 'fail' : 'warn', + message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, + details: { + hnswSize, + vectoredNouns: vectorParityTarget, + metadataEntries: metadataStats.totalEntries, + drift + } }) } @@ -16032,6 +16275,141 @@ export class Brainy implements BrainyInterface { return !this.pluginRegistry.hasProvider('embeddings') } + /** + * @description LEG C of the zero-norm/unvector-door law — migrate a + * legacy zero-norm VFS root BEFORE the vector-leg open gate + * ({@link rebuildIndexesIfNeeded}'s `vectorCoverageGap` check) ever + * compares the canonical vectored-noun count against the vector index's + * size. A pre-fix store may have persisted the VFS root (the fixed + * all-zeros UUID) with a REAL all-zero placeholder vector — lawful inside + * brainy (`cosineDistance` treats a zero-norm operand as MAXIMUM distance, + * see {@link isZeroNormVector}'s JSDoc) but never indexed (the index belt + * refuses to insert a zero-norm vector) and never meant to cross an + * engine boundary. Left unmigrated, the canonical ledger still counts it + * as vectored while the vector index correctly holds nothing for it — a + * near-empty store whose ONLY vectored row is this zero-norm root reads + * "canonical vectored 1, index size 0" and throws + * `VectorIndexNotReadyError` at open, going DARK instead of serving. + * + * THE LIFECYCLE LAW: nothing at open may scale with brain size. This step + * is safe under that law BECAUSE the VFS root lives at a FIXED, + * well-known id (`00000000-0000-0000-0000-000000000000` — mirrors + * `VirtualFileSystem.VFS_ROOT_ID`; kept as a literal here, the same + * convention as the other reserved-root literals in this file and in + * `db/factLog.ts`/`db/portableGraph.ts` — `brainy.ts` cannot import + * `VirtualFileSystem.ts`, which itself imports `Brainy`) — this is ONE + * direct canonical read by id (`storage.getNoun`, the same O(1) + * fixed-path lookup {@link unvectorNounForRootMigration} itself uses + * internally), NEVER a listing or a walk over `entities/nouns/**`. An + * absent root (a store that has never used the VFS) is a no-op, no error. + * + * Runs UNCONDITIONALLY at every open, independent of whether a + * `VirtualFileSystem` is ever constructed this session — the vector-leg + * gate this fixes runs during Brainy's OWN init, before any + * `VirtualFileSystem` instance exists to run its own lazy migration at + * `doInitializeRoot()` (kept in place as the second line of defense for a + * VFS actually opened this session — belt AND suspenders, never either + * alone). + */ + private async migrateLegacyZeroNormVfsRootIfNeeded(): Promise { + const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + // TORN-TOLERANT: a torn root record is a recovery-walk healer's job + // (see tests/integration/recovery-walk-tolerance.test.ts — an init-time + // walk that meets a torn record narrates+counts, via the adapter's own + // loud floor at the read site, and heals PAST it; the open itself must + // still succeed), not this O(1) migration check's. Skip this open's + // migration attempt rather than aborting init(): this leg is a + // defensive EXTRA (the index belt + VirtualFileSystem's own + // doInitializeRoot() migration still stand as the other lines of + // defense), and it retries harmlessly at a later open once the root + // heals. + let root: HNSWNounWithMetadata | null + try { + root = await this.storage.getNoun(VFS_ROOT_ID) + } catch (err) { + if ((err as { code?: string }).code !== 'TORN_RECORD') throw err + prodLog.warn( + `[Brainy] open(): the VFS root's record is TORN — skipping the zero-norm root ` + + `migration check this open (the recovery walk is the healer; this migration ` + + `retries harmlessly once the root heals).` + ) + return + } + if (!root || !Array.isArray(root.vector) || root.vector.length === 0) return + if (!isZeroNormVector(root.vector)) return + const migrated = await this.unvectorNounForRootMigration(VFS_ROOT_ID) + if (migrated) { + prodLog.warn( + `[Brainy] open(): migrated the VFS root's legacy all-zero placeholder vector to ` + + `the unvectored shape (zero-norm vectors never cross an engine boundary) — run ` + + `before the vector-leg open gate compares canonical-vectored-count against the ` + + `vector index, so a near-empty store never reads a false coverage gap.` + ) + } + } + + /** + * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's + * persisted vector from a real (non-empty) vector to the "unvectored" + * empty-array shape: the vector record is rewritten to `[]`, the row is + * removed from the vector index (if present), and the vectored-noun + * ledger (`getCanonicalCounts().vectors.all`) is decremented through the + * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the + * coverage ledger never silently drifts. + * + * Exists SOLELY for the VFS root zero-norm migration, called from two + * sites that detect the same legacy shape (a persisted root whose vector + * is the legacy all-zero placeholder): {@link migrateLegacyZeroNormVfsRootIfNeeded} + * (this brain's own init sequence, BEFORE the vector-leg open gate — Leg + * C of the zero-norm/unvector-door law) and + * `VirtualFileSystem.doInitializeRoot()` (the second line of defense, for + * a VFS actually constructed this session). This is NOT the general- + * purpose unvector API — ordinary application data uses the sanctioned + * unvector DOOR instead (`update({ id, vector: [] })` / the same op inside + * `transact()`), which decrements the ledger and clears any pending + * deferred-embed marker inline; it does not call this method. Never call + * this outside a VFS root migration. + * + * Idempotent: a noun already unvectored (`vector.length === 0`) or absent + * is a no-op — safe to call on every `init()`. + * + * @param id - The canonical noun id to migrate. + * @returns `true` if a migration write happened, `false` if the noun was + * already unvectored (or absent) — a no-op. + */ + async unvectorNounForRootMigration(id: string): Promise { + const noun = await this.storage.getNoun(id) + if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false + + await this.persistSingleOp({ nouns: [id] }, async (tx) => { + // Rewrite the vector leg to the unvectored shape. Placeholder adjacency + // (mirrors update()'s own SaveNounOperation staging) — the op preserves + // stored graph state when `connections.size === 0`. + tx.addOperation( + new SaveNounOperation(this.storage, { + id, + vector: [], + connections: new Map(), + level: 0 + }) + ) + // Remove from the vector index — safe even if the row was never + // actually indexed (RemoveFromVectorIndexOperation's removeItem is a + // no-op when the id is absent). + tx.addOperation( + new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration) + ) + }) + + // Vectored-noun ledger: this migration carries a vector write with no + // accompanying metadata operation (metadata is untouched), so the + // saveNounMetadata(..., hasVector) seam never fires for it — mirrors the + // deferred-embed LANDING path's use of the narrow storage hook, in + // reverse. + await this.storage.noteVectorUnlanded?.(id) + return true + } + /** * Setup embedder */ diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 8112afd2..4b018e94 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -872,6 +872,27 @@ export interface StorageAdapter { */ noteVectorLanded?(id: string): Promise + /** + * OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}: + * record that a canonical noun's vector was just REMOVED — rewritten from + * a real (non-empty) vector to the "unvectored" empty-array shape. Exists + * for the ONE sanctioned reverse migration this engine supports: the VFS + * root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and + * `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix + * store's all-zero placeholder root vector to `[]` and must decrement + * `vectors.all` through this hook so the coverage ledger never drifts. + * NOT a general-purpose "I removed a vector" callback — ordinary + * application data has no sanctioned path from vectored back to + * unvectored (`update()` refuses an empty vector as a dimension + * mismatch by design). Callers MUST call this only when the noun held a + * REAL vector immediately before this write (the caller already holds + * that fact for free, from its own pre-write read — never an added read). + * A backend without vectored-noun tracking is a no-op via this method's + * absence (feature-detected). + * @param id - The noun whose vector was just removed. + */ + noteVectorUnlanded?(id: string): Promise + /** * Get noun with metadata combined * @returns Combined HNSWNounWithMetadata or null diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index bfb68959..81bed338 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -654,10 +654,43 @@ export class GenerationStore { let replayed = 0 const replayFact = async (fact: CommitFact): Promise => { for (const op of fact.ops) { - const image = - op.record === null - ? { metadata: null, vector: null } - : { metadata: op.record.metadata, vector: op.record.vector } + let image: { metadata: unknown | null; vector: unknown | null } + if (op.record === null) { + // A genuine tombstone (both legs absent) — the fold removes + // both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw + // exact-restore contract. + image = { metadata: null, vector: null } + } else if ( + op.record.metadata !== null && + (op.record.vector === null || op.record.vector === undefined) + ) { + // PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's + // half): a metadata-only after-image must never DELETE an + // existing vector leg through the fold. `writeNounRaw`/ + // `writeVerbRaw` are exact-restore primitives — a `vector: + // null` there means "delete", which is exactly right for + // `rollBackUncommittedGeneration`'s before-image restore (a + // transaction abort legitimately un-writes a vector the failed + // transaction added). It is NOT right here: this fold replays + // AFTER-IMAGES, and re-applying an already-intact record must + // be byte-safe (this module's own invariant, see the log-authority + // comment above) — silently erasing a landed vector because one + // replayed fact's vector leg came back null is the exact defect + // that left metadata-counted, never-enumerated rows in a + // production store (confirmed root cause: the enumeration walk + // used to key on the vector leg, so a preserved-but-then-deleted + // vector made the row invisible while the ledger still counted + // it by metadata). A genuine "unvector" has its own sanctioned, + // ledger-correct path (`Brainy.unvectorNounForRootMigration`) — + // never this raw primitive, and never the fold. + const current = + op.kind === 'verb' + ? await this.storage.readVerbRaw(op.id) + : await this.storage.readNounRaw(op.id) + image = { metadata: op.record.metadata, vector: current.vector ?? null } + } else { + image = { metadata: op.record.metadata, vector: op.record.vector } + } if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) else await this.storage.writeNounRaw(op.id, image) this.noteCheckpointDirty(op.kind, op.id) diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 77e4f84d..8b9badc1 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -10,7 +10,7 @@ import { Vector, VectorDocument } from '../coreTypes.js' -import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' +import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' @@ -64,6 +64,34 @@ export class HnswFlushError extends Error { } } +/** + * @description Thrown by {@link JsHnswVectorIndex.addItem} / {@link + * JsHnswVectorIndex.updateItem} when handed a length-0 vector. A length-0 + * vector is the sanctioned "unvectored" shape for a canonical noun record + * (class-J: a VFS-system row, a deferred embed not yet landed, or any other + * legitimately-vector-less row) — but it is NEVER a legal INDEX insert. The + * index itself has no concept of "unvectored"; deciding that a row is + * unvectored and therefore skippable is the FILL/REBUILD/LOAD consumer's job + * (see {@link JsHnswVectorIndex.rebuild}), done BEFORE ever calling addItem. + * A length-0 vector reaching this point is a caller bug: silently accepting + * it would pin `this.dimension = 0` on an empty index (poisoning every real + * insert thereafter with a dimension mismatch) or store a vector-less node + * that a distance calculation can never safely compare against. Loud errors, + * never quiet losses — this throws instead of either. + */ +export class EmptyVectorIndexError extends Error { + constructor(public readonly id: string, operation: 'addItem' | 'updateItem') { + super( + `${operation}(${id}): refusing to index a length-0 vector — a length-0 vector is the ` + + `sanctioned "unvectored" shape for a canonical row, but it is never a legal index ` + + `insert. Callers that fill/rebuild/load the index must skip vector.length === 0 rows ` + + `themselves (unvectored = nothing to index, not an error at that layer); reaching ` + + `here with one is a caller bug.` + ) + this.name = 'EmptyVectorIndexError' + } +} + /** * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native @@ -580,6 +608,15 @@ export class JsHnswVectorIndex implements VectorIndexProvider { throw new Error('Vector is undefined or null') } + // THE INDEX REFUSES A LENGTH-0 VECTOR (see EmptyVectorIndexError's JSDoc): + // an empty vector is the sanctioned "unvectored" shape at the canonical + // layer, never a legal index member. Refusing here — loudly, before the + // dimension pin below — means no future fill/rebuild/load path can ever + // poison `this.dimension` to 0 or park a vector-less node in the graph. + if (vector.length === 0) { + throw new EmptyVectorIndexError(id, 'addItem') + } + // Set dimension on first insert if (this.dimension === null) { this.dimension = vector.length @@ -954,6 +991,13 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return } + // Same refusal as addItem (see EmptyVectorIndexError's JSDoc) — an + // in-place relink must never rewrite an already-indexed node down to the + // unvectored shape or poison the pinned dimension. + if (vector.length === 0) { + throw new EmptyVectorIndexError(id, 'updateItem') + } + if (this.dimension === null) { this.dimension = vector.length } else if (vector.length !== this.dimension) { @@ -1555,7 +1599,15 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } const loaded = await this.storage.getNounVector(noun.id) - if (!loaded) { + // `loaded` is a length-0 array (not null/undefined) for a canonical row + // that is legitimately unvectored — `![]` is FALSE (an empty array is + // truthy), so the bare `!loaded` check below would silently accept it + // as "found" and hand a dimension-0 vector to a distance calculation. + // A node only reaches this lazy-load path because it is a MEMBER of + // the live index (rebuild() now refuses to admit unvectored rows — see + // its JSDoc), so an empty vector here is never legitimate: treat it + // exactly like "not found", loudly. + if (!loaded || loaded.length === 0) { throw new Error(`Vector not found for noun ${noun.id}`) } @@ -1765,9 +1817,56 @@ export class JsHnswVectorIndex implements VectorIndexProvider { totalCount = result.totalCount || result.items.length + // UNVECTORED ROWS ARE NOT AN INDEX MEMBER (the class-J law): a canonical + // noun whose vector leg is `[]` (a VFS-root-style system row, a + // deferred embed not yet landed, or a best-effort fallback for an + // unreadable vector leg) is a normal, enumerable, countable row — it + // is simply not indexed. `storage.getVectorIndexData()` derives its + // {level, connections} answer straight from the noun's OWN record, so + // it returns non-null for every existing noun regardless of whether + // that noun ever actually reached `addItem()` — it cannot be used to + // decide indexability. `nounData.vector.length === 0` is the one + // truthful signal (mirrors the `noun.vector.length > 0` guards in + // {@link getVectorSafe} / {@link getVectorSync}): skip here, counted + // once in a summary line, never per-row spam. + let skippedUnvectored = 0 + // Process all nouns at once for (const nounData of result.items) { try { + if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) { + skippedUnvectored++ + continue + } + // THE ZERO-NORM LAW — bulk-rebuild leg: a persisted zero-norm + // vector (a pre-10.4.2 row the canonical write has not yet + // normalized) must never enter the index either, mirroring the + // belt AddToVectorIndexOperation enforces on the live write path. + // Only the canonical vector is authoritative here — persisted + // HNSW graph metadata (level/connections) can outlive an unvector. + if (isZeroNormVector(nounData.vector)) { + prodLog.warn( + `[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` + + `zero-norm (a zero-norm vector is not a vector and never crosses an ` + + `engine boundary)` + ) + continue + } + + // Restore the pinned dimension from the first real vector this + // rebuild loads. `addItem`/`updateItem` only pin `this.dimension` + // on a LIVE insert — a fresh rebuild from storage never goes + // through either, so without this the pin stays `null` across a + // restart. A `null` pin means the very next insert (correct OR + // wrong length) silently BECOMES the new pin instead of being + // checked against the store's real dimension — the wrong-length + // case then fails much later and less clearly, inside a distance + // calculation against an already-loaded node, instead of here, + // immediately, with a named expected-vs-got mismatch. + if (this.dimension === null) { + this.dimension = nounData.vector.length + } + // Load HNSW graph data for this entity const hnswData = await this.storage.getVectorIndexData(nounData.id) @@ -1815,7 +1914,10 @@ export class JsHnswVectorIndex implements VectorIndexProvider { options.onProgress(loadedCount, totalCount) } - prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`) + prodLog.info( + `HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})` + + (skippedUnvectored > 0 ? ` — ${skippedUnvectored.toLocaleString()} unvectored row(s) skipped` : '') + ) } // Step 5: CRITICAL - Recover entry point if missing) diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index c15447e7..4f4339f4 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-07-02T21:43:26.976Z + * Generated: 2025-09-29T10:10:00-07:00 * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index b5f3546b..5b10116c 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-02-09T16:59:48.867Z + * Generated: 2026-06-29T10:04:19-07:00 * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-02-09T16:59:48.867Z", + generatedAt: "2026-06-29T10:04:19-07:00", sizeBytes: { embeddings: 259584, base64: 346112 diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 22bf1366..cabe2e30 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1066,6 +1066,18 @@ export abstract class BaseStorageAdapter implements StorageAdapter { protected allCountsSuspect = false /** One narration per session for the suspect transition (never per delete). */ private allCountsSuspectNarrated = false + /** + * Which rule produced the ALL scalars currently in memory. `'identity-record'` + * means one counted entity per metadata content leg — the honest rule: a + * bare id-directory (a ghost or scar left by a partial-delete defect, no + * content leg) counts zero. Set by the one-time derivation and by the + * sanctioned recount, alongside `allCountsSuspect = false`; left `undefined` + * when a loaded counts.json carries the ALL scalars but no stamp — the + * legacy container-rule derivation, which forces `allCountsSuspect = true` + * at load instead. A filesystem concern: `MemoryStorage` has no counts.json + * and never sets this. + */ + protected allCountsDerivedBy?: 'identity-record' protected entityCounts: Map = new Map() // type -> count protected verbCounts: Map = new Map() // verb type -> count protected countCache: Map = new Map() @@ -1152,6 +1164,24 @@ export abstract class BaseStorageAdapter implements StorageAdapter { }) } + /** + * OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}): + * the mirror of {@link noteVectorLanded} — record a noun's vector was just + * REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a + * caller that (incorrectly) fires this for a noun already unvectored would + * otherwise drive the ledger negative — clamped defensively, matching the + * delete path's `if (this.totalVectoredNounCount > 0)` guard. + * @param id - The noun whose vector was just removed (retained for a + * future narration seam; the count itself needs no id-keyed state). + */ + async noteVectorUnlanded(id: string): Promise { + void id + if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount-- + this.scheduleCountPersist().catch(() => { + // Ignore persist errors — the in-memory count is authoritative; a later op retries. + }) + } + /** * Increment count for entity type - O(1) operation. * Concurrency is handled by the process-global mutex diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index fd9dbb4c..4f2a43b0 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -18,6 +18,8 @@ import { } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' +import { prodLog } from '../../utils/logger.js' +import { isZeroNormVector } from '../../utils/distance.js' import { TornRecordError, isUnparseablePayloadError, @@ -602,6 +604,20 @@ export class FileSystemStorage extends BaseStorage { * automatically. Returns the pruned container ids so the caller can recompute * counts. */ + /** + * @description Whether an id directory's file legs include the metadata + * CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single + * test that decides whether an `entities////` container is + * a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete + * defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune + * and {@link scanCanonicalEntities} so the two agree by construction — one + * counted entity per identity record, never per bare container. + * @param legs - File names in one `entities////` directory. + */ + private hasMetadataContentLeg(legs: string[]): boolean { + return legs.some((f) => f.startsWith('metadata.json')) + } + public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { await this.ensureInitialized() const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } @@ -641,7 +657,7 @@ export class FileSystemStorage extends BaseStorage { } // A live entity has its metadata content leg. No content leg → a // vector-only ghost or an empty scar → prune the whole container. - if (legs.some((f) => f.startsWith('metadata.json'))) continue + if (this.hasMetadataContentLeg(legs)) continue await fs.promises.rm(idAbs, { recursive: true, force: true }) pruned[kind].push(entry.name) console.warn( @@ -2590,13 +2606,36 @@ export class FileSystemStorage extends BaseStorage { ) { this.totalNounCountAll = counts.totalNounCountAll this.totalVerbCountAll = counts.totalVerbCountAll - this.allCountsSuspect = counts.allCountsSuspect === true + if (counts.allCountsDerivedBy === 'identity-record') { + // Derived (or recounted) under the honest rule — one counted + // entity per metadata content leg. Trust the persisted suspect + // flag as-is; an unprovable delete since may still have set it. + this.allCountsDerivedBy = 'identity-record' + this.allCountsSuspect = counts.allCountsSuspect === true + } else { + // The ALL scalars exist but predate the identity-record stamp — + // they were derived under the legacy rule that counted one + // entity per id DIRECTORY, so orphaned ghost/scar containers (a + // pre-8.3.1 partial-delete defect — see pruneOrphanedEntities()) + // were counted as entities too. O(1) field read, NEVER a walk + // here: force suspect and name it loudly. A sanctioned recount + // (repairIndex) restores exact denominators and clears this. + this.allCountsDerivedBy = undefined + this.allCountsSuspect = true + needsPersist = true + prodLog.warn( + '[FileSystemStorage] canonical count ledger was derived under the legacy ' + + 'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' + + 'exact denominators' + ) + } } else { const nouns = await this.scanCanonicalEntities('nouns') const verbs = await this.scanCanonicalEntities('verbs') this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false + this.allCountsDerivedBy = 'identity-record' console.warn( `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` + `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + @@ -2667,6 +2706,7 @@ export class FileSystemStorage extends BaseStorage { this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false + this.allCountsDerivedBy = 'identity-record' // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT // (a deferred-embed noun's file exists but holds an empty vector until // its embed lands), so this is a full O(nouns) content scan — see @@ -2704,10 +2744,19 @@ export class FileSystemStorage extends BaseStorage { /** * Walk the canonical `entities//<2-hex-shard>//` tree, counting - * one entity per id directory (the layout `getNounVectorPath`/`getNouns` - * use). Returns up to 100 sampled entity directories (absolute paths) — - * nouns feed the type-distribution estimate above. An absent tree (fresh - * store) counts zero. + * one entity per id directory that holds the metadata CONTENT leg + * (`metadata.json` or its `.json.gz` variant — see + * {@link hasMetadataContentLeg}). A bare container — a ghost (a stale + * `vectors.json` left with no metadata leg) or a scar (an empty directory), + * both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the + * identity record IS the population (ADR-008 G1), never the directory. + * This is the ONE-TIME legacy derivation walk (see callers); a prior + * version of this scan counted every id directory regardless of content, + * over-counting any store carrying orphaned containers — see + * `allCountsDerivedBy` for how a counts.json derived under that old rule is + * marked suspect on load. Returns up to 100 sampled *counted* entity + * directories (absolute paths) — nouns feed the type-distribution estimate + * above. An absent tree (fresh store) counts zero. */ private async scanCanonicalEntities( kind: 'nouns' | 'verbs' @@ -2724,9 +2773,21 @@ export class FileSystemStorage extends BaseStorage { const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) for (const entry of ids) { if (!entry.isDirectory()) continue + const idAbs = path.join(shardPath, entry.name) + let legs: string[] + try { + legs = await fs.promises.readdir(idAbs) + } catch (error: any) { + if (error?.code === 'ENOENT') continue + throw error + } + // No metadata content leg → a ghost or scar container → not an + // entity. Same test pruneOrphanedEntities() uses, so the two agree + // by construction. + if (!this.hasMetadataContentLeg(legs)) continue count++ if (sampleDirs.length < SAMPLE_MAX) { - sampleDirs.push(path.join(shardPath, entry.name)) + sampleDirs.push(idAbs) } } } @@ -2781,14 +2842,20 @@ export class FileSystemStorage extends BaseStorage { } /** - * Count canonical nouns holding a REAL (non-empty) vector — the vectored- - * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence - * cannot be decided from the id-directory listing alone: a deferred-embed - * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`) - * until its embed LANDS, so this walk reads every noun's `vectors.json` - * CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time - * legacy-counts.json derivation or a lost/corrupted counts.json recovery; - * the result is persisted so this scan never repeats. + * Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector — + * the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, + * presence cannot be decided from the id-directory listing alone: a + * deferred-embed noun's `vectors.json` EXISTS (written at `add()` time + * with `vector: []`) until its embed LANDS, so this walk reads every + * noun's `vectors.json` CONTENT — O(nouns) reads, not O(ids) listing. + * ZERO-NORM LAW: a real all-zero vector is not a vector — it never counts + * here either (Brainy's write paths normalize an explicit zero-norm + * vector to `[]` at write time, but a store created before that fix may + * still carry legacy all-zero rows on disk; this derivation must agree + * with the live ledger's definition of "vectored" regardless of when the + * row was written). Used ONLY for a one-time legacy-counts.json derivation + * or a lost/corrupted counts.json recovery; the result is persisted so + * this scan never repeats. */ private async scanVectoredNounCount(): Promise { const base = path.join(this.rootDir, 'entities', 'nouns') @@ -2802,7 +2869,12 @@ export class FileSystemStorage extends BaseStorage { for (const entry of ids) { if (!entry.isDirectory()) continue const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) - if (record && Array.isArray(record.vector) && record.vector.length > 0) { + if ( + record && + Array.isArray(record.vector) && + record.vector.length > 0 && + !isZeroNormVector(record.vector) + ) { vectored++ } } @@ -2834,6 +2906,13 @@ export class FileSystemStorage extends BaseStorage { // scanVectoredNounCount()'s JSDoc). totalVectoredNounCount: this.totalVectoredNounCount, allCountsSuspect: this.allCountsSuspect, + // Derivation-rule stamp for the ALL scalars above — 'identity-record' + // when they were counted one-per-metadata-content-leg (the honest + // rule); omitted (JSON.stringify drops `undefined`) when the current + // in-memory scalars came from a legacy container-rule counts.json + // that hasn't been through a sanctioned recount yet, so a future load + // keeps naming them suspect rather than trusting an unproven value. + allCountsDerivedBy: this.allCountsDerivedBy, lastUpdated: new Date().toISOString() } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index d6ccc5fa..5510f93b 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -203,6 +203,40 @@ function idFromVectorPath(path: string): string { return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix } +/** + * @description Extract the entity id embedded in a metadata path + * (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) — the IDENTITY-RECORD + * mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their + * population on this file (ADR-008 G1: the metadata record IS the population; + * the vector leg is optional), so walk ordering and cursor resume derive the + * id from THIS path, never the vector path — a row with metadata and no + * vector file must still be listed, ordered, and resumable. + * @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`). + * @returns The entity id (the path segment immediately before `/metadata.json`). + */ +function idFromMetadataPath(path: string): string { + const withoutSuffix = path.replace(/\/metadata\.json$/, '') + const lastSlash = withoutSuffix.lastIndexOf('/') + return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix +} + +/** + * @description The sanctioned UNVECTORED shape for a noun hydrated during + * enumeration when its identity record (metadata.json) exists but its vector + * leg (vectors.json) does not — a fold-born metadata-only after-image, or any + * row genuinely without a vector yet. Mirrors the shape + * `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned + * unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a + * walk-yielded unvectored row is byte-shape-identical to one produced by that + * migration. Callers already handle `vector: []` as first-class + * (validateAddParams exempts it; index gates key on `length > 0`). + * @param id - The noun id. + * @returns A structurally-valid, vector-empty `HNSWNoun`. + */ +function unvectoredNoun(id: string): HNSWNoun { + return { id, vector: [], connections: new Map>(), level: 0 } +} + /** * Get ID-first path for verb metadata * No type parameter needed - direct O(1) lookup by ID @@ -1453,6 +1487,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { * rollups are derived state with their own rebuild paths * (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`). * + * EXACT-RESTORE PRIMITIVE — `vector: null` DELETES the vector leg, on + * purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on + * this to legitimately un-write a vector a failed transaction added. This + * is deliberately NOT "preserve if absent" — a caller replaying an + * AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must + * apply preserve-if-absent itself BEFORE calling this, by reading the + * current vector and carrying it forward when the after-image's own + * vector leg is null/undefined but its metadata is not (see `replayFact` + * for the implementation and full rationale). A caller that genuinely + * wants to unvector a row uses the sanctioned, ledger-correct path + * (`Brainy.unvectorNounForRootMigration`) — never this primitive. + * * @param id - The entity id. * @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}. */ @@ -1489,7 +1535,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Restore a relationship's raw stored objects byte-for-byte (verb-side - * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats). + * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats, + * same EXACT-RESTORE contract — `vector: null` deletes, on purpose; the + * fold's preserve-if-absent logic lives at its call site, not here). * * @param id - The relationship id. * @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}. @@ -2183,9 +2231,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Stable within-shard order (by noun id) so offset windows and cursor resume // are deterministic; ids come from the path so skipped nouns are never read. + // + // IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record + // (not the vector) IS the population — a noun with metadata and no vector + // file (a fold-born after-image, see writeNounRaw's preserve-if-absent + // contract) must still enumerate. Keying on metadata.json here means the + // ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed) + // and this walk agree on population by construction. Ordering is + // unaffected for a healthy store: every vectored noun has both legs, so + // the id set and sort order are identical to the old vectors.json keying. const entries = nounFiles - .filter((p) => p.includes('/vectors.json')) - .map((p) => ({ path: p, id: idFromVectorPath(p) })) + .filter((p) => p.includes('/metadata.json')) + .map((p) => ({ path: p, id: idFromMetadataPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor @@ -2210,13 +2267,24 @@ export abstract class BaseStorage extends BaseStorageAdapter { ) { const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) const hydrated = await Promise.all( - batch.map(async ({ path: nounPath }) => { + batch.map(async ({ path: metadataPath, id }) => { try { - const noun = await this.readCanonicalObject(nounPath) - if (!noun) return null - const deserialized = this.deserializeNoun(noun) - const metadata = await this.getNounMetadata(deserialized.id) + const metadata = await this.readCanonicalObject(metadataPath) if (!metadata) return null + // The vector leg is OPTIONAL (population law): a metadata-only + // row hydrates with the sanctioned unvectored shape rather than + // being dropped from the walk. A fault reading the vector leg + // is treated the same as absence — best-effort, matching the + // canonical recount's tolerance for an unreadable vectors.json + // (rebuildTypeCounts) — a vector-leg problem never hides an + // otherwise-good identity record. + let deserialized: HNSWNoun + try { + const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id)) + deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id) + } catch { + deserialized = unvectoredNoun(id) + } return { deserialized, metadata } } catch (error) { // A TORN record must surface typed — a paginated read that @@ -2226,7 +2294,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { // walk's job is to HEAL PAST it — skip the victim, serve the rest. // Identity point-reads (get-by-id) still throw typed upstream. if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } - // Skip nouns that fail to load + // Skip nouns whose IDENTITY record fails to load (the metadata + // read above) — that is the one leg this walk cannot proceed + // without. return null } }) @@ -2347,9 +2417,14 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/nouns/${shardHex}` try { const nounFiles = await this.listCanonicalObjects(shardDir) + // IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching + // comment in getNounsWithPagination: metadata.json is the population; + // the vector leg is optional, so a metadata-only row must still be + // listed (and here, for the unfiltered case, needs ZERO reads either + // way — the id comes straight from the path). const entries = nounFiles - .filter((p) => p.includes('/vectors.json')) - .map((p) => idFromVectorPath(p)) + .filter((p) => p.includes('/metadata.json')) + .map((p) => idFromMetadataPath(p)) .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) const toWalk = cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries @@ -2560,23 +2635,79 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Stable within-shard order (by verb id) so offset windows and cursor resume // are deterministic and consistent across calls. Ids come from the path, so // verbs skipped by the cursor are never read. + // + // IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of + // this comment in getNounsWithPagination applies here too: metadata.json + // is the population; keying on it here means this walk and the ledger + // recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed) + // agree on population by construction. Unchanged for a healthy store — + // `relate()` always writes both legs of a verb in the same commit, so + // the id set and order match the old vectors.json keying exactly; this + // only additionally surfaces a fold-born metadata-only row (see + // writeVerbRaw's preserve-if-absent contract). const entries = verbFiles - .filter((p) => p.includes('/vectors.json')) - .map((p) => ({ path: p, id: idFromVectorPath(p) })) + .filter((p) => p.includes('/metadata.json')) + .map((p) => ({ path: p, id: idFromMetadataPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) - for (const { path: verbPath, id: verbId } of entries) { + for (const { path: metadataPath, id: verbId } of entries) { if (collected.length >= peekCount) break // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id // (later shards are processed in full). No read for skipped verbs. if (cursor && shard === cursor.shard && verbId <= cursor.id) continue try { - const rawVerb = await this.readCanonicalObject(verbPath) - if (!rawVerb) continue + // Identity leg first — required. A verb this walk cannot read + // metadata for cannot be hydrated at all (same as before). + const metadata = await this.readCanonicalObject(metadataPath) + if (!metadata) continue - // Deserialize connections Map from JSON storage format - const verb = this.deserializeVerb(rawVerb) + // The vector leg is the verb's STRUCTURAL core (verb/sourceId/ + // targetId live there — see coreTypes.ts HNSWVerb), unlike a + // noun's vector, which is pure embedding data. `relate()` always + // writes both legs atomically and verbs have no deferred-embed + // path, so a healthy store's verbs always have both. A vector-leg + // absence here can only be a fold-born after-image (see + // writeVerbRaw's preserve-if-absent contract) — and unlike a + // noun, this walk cannot safely FABRICATE sourceId/targetId to + // synthesize a structurally-valid verb (an empty-string endpoint + // would silently create a phantom edge — worse than omission). + // If the metadata record happens to carry its own sourceId/ + // targetId (never true for current production writes, but not + // disallowed — e.g. a future schema or a repair tool could + // populate them), reconstruct from those; otherwise this row is + // loudly skipped — counted by the ledger, but not returned as an + // item, until a repair can supply the missing endpoints. + const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId)) + let verb: HNSWVerb + if (rawVerb) { + verb = this.deserializeVerb(rawVerb) + } else { + const metaSourceId = (metadata as Record).sourceId + const metaTargetId = (metadata as Record).targetId + const metaVerbType = (metadata as Record).verb + if ( + typeof metaSourceId === 'string' && metaSourceId.length > 0 && + typeof metaTargetId === 'string' && metaTargetId.length > 0 && + typeof metaVerbType === 'string' && metaVerbType.length > 0 + ) { + verb = { + id: verbId, + vector: [], + connections: new Map>(), + verb: metaVerbType as VerbType, + sourceId: metaSourceId, + targetId: metaTargetId + } + } else { + prodLog.error( + `[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` + + `record but no vector leg and no recoverable sourceId/targetId — ` + + `skipping (counted by the ledger, not yielded; needs repair).` + ) + continue + } + } // Apply type filter if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) { @@ -2593,9 +2724,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { continue } - // Load metadata - const metadata = await this.getVerbMetadata(verb.id) - // Apply subtype filter (requires metadata — checked AFTER load) if (filterSubtypes) { const subtype = metadata?.subtype as string | undefined @@ -4692,6 +4820,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.totalVerbCountAll = allVerbs this.totalVectoredNounCount = allVectoredNouns this.allCountsSuspect = false + // This walk counts one entity per metadata.json record (never per bare + // container) — the identity-record rule. Stamp it so a future load + // trusts these scalars instead of naming them suspect at open. + this.allCountsDerivedBy = 'identity-record' this.countCache.clear() await this.persistCounts() diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 139c67fe..1bbbca88 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -13,6 +13,8 @@ import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' +import { isZeroNormVector } from '../../utils/distance.js' +import { prodLog } from '../../utils/logger.js' /** * Backend identity stamped into an operation's emitted `name` string (e.g. @@ -88,6 +90,30 @@ export class AddToVectorIndexOperation implements Operation { } async execute(): Promise { + // THE ZERO-NORM LAW (the live provider-write seam's belt): a zero-norm + // vector is not a vector — it never crosses an engine boundary. This + // engine's own cosine distance treats an all-zero vector safely (a + // zero-norm operand always scores MAXIMUM distance, see + // {@link isZeroNormVector}'s JSDoc), but a downstream engine serving + // squared-euclidean distance cannot tell it apart from a legitimate + // origin point — a false attractor that silently darkens real results. + // The canonical write already landed (SaveNoun/SaveNounMetadata + // operations are staged ahead of this one in every caller) — only the + // INDEX INSERT is refused here, loudly, never a throw. A length-0 + // vector is the unrelated "unvectored" shape and is skipped silently + // (the same contract callers already rely on for deferred embeds). + if (this.vector.length === 0) { + return async () => {} + } + if (isZeroNormVector(this.vector)) { + prodLog.warn( + `[vector-index] refusing to index a zero-norm vector for entity ${this.id} — ` + + `a zero-norm vector is not a vector and never crosses an engine boundary ` + + `(the canonical write is unaffected; only the vector-index insert is skipped)` + ) + return async () => {} + } + // Check if item already exists (for rollback decision) const existed = await this.itemExists(this.id) @@ -263,14 +289,52 @@ export class ReplaceInVectorIndexOperation implements Operation { // One commit generation for the whole replace (both branches + rollback). const generation = this.generationFn?.() + // THE ZERO-NORM LAW (see AddToVectorIndexOperation's matching JSDoc): a + // real all-zero replacement vector must never land in the index — refuse + // loudly, canonical write unaffected. The row must not be left stale + // either: if it was genuinely indexed under `oldVector`, remove it + // rather than pretend the old vector still describes the row. A + // length-0 `newVector` (the unrelated "unvectored" shape) is handled the + // same way, silently — no caller today reaches this with an empty + // replacement (update() rejects a dimension-mismatched empty vector), + // but the seam stays consistent in case one ever legitimately does. + if (isZeroNormVector(this.newVector) || this.newVector.length === 0) { + const wasIndexed = this.oldVector.length > 0 && !isZeroNormVector(this.oldVector) + if (isZeroNormVector(this.newVector)) { + prodLog.warn( + `[vector-index] refusing to replace with a zero-norm vector for entity ${this.id} — ` + + `a zero-norm vector is not a vector and never crosses an engine boundary ` + + `(the canonical write is unaffected; the row is removed from the vector index instead)` + ) + } + if (wasIndexed) { + await this.index.removeItem(this.id, generation) + } + return async () => { + // Restore the declared before-state. + if (wasIndexed) { + await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + } + } + } + if (typeof index.updateItem === 'function') { // Atomic path: one in-place call, the row never leaves the index. await index.updateItem({ id: this.id, vector: this.newVector }, generation) return async () => { // Restore the declared before-state in place (see class JSDoc for - // the item-did-not-exist posture). - await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) + // the item-did-not-exist posture). A length-0 oldVector means the row + // was never actually indexed before this op ran (a length-0 vector is + // never a legal index member — see EmptyVectorIndexError) — there is + // no in-place "restore to empty" for the provider to perform, so + // rollback removes the row instead, leaving the same "not indexed" + // state the row was in before execute(). + if (this.oldVector.length > 0) { + await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) + } else { + await this.index.removeItem(this.id, generation) + } } } @@ -281,9 +345,14 @@ export class ReplaceInVectorIndexOperation implements Operation { return async () => { // updateItem-style restore via the same adjacent pair, back to the - // declared before-state. + // declared before-state. Same length-0 carve-out as the updateItem + // path above: an empty oldVector was never a legal index member, so + // rollback just leaves the row removed rather than attempting an + // illegal empty re-add. await this.index.removeItem(this.id, generation) - await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + if (this.oldVector.length > 0) { + await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) + } } } } diff --git a/src/utils/distance.ts b/src/utils/distance.ts index 36e9e8e5..d61bc12e 100644 --- a/src/utils/distance.ts +++ b/src/utils/distance.ts @@ -65,6 +65,29 @@ export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number = return 1 - similarity } +/** + * True when `vector` is a REAL (non-empty) all-zero vector — the "false + * attractor" shape this engine's own cosine distance treats safely (a + * zero-norm operand always scores the MAXIMUM distance, see + * {@link cosineDistance}) but a downstream engine serving squared-euclidean + * distance cannot distinguish from a legitimate origin point. THE LAW: a + * zero-norm vector is not a vector — it never crosses an engine boundary + * (never handed to a vector-index provider as a searchable item). + * + * A length-0 vector is the UNRELATED "unvectored, not yet embedded" shape + * (the deferred-embed stub, a permanently-vectorless system row) and is + * deliberately NOT zero-norm here — callers checking for "nothing to index" + * should test `vector.length === 0` separately; this only flags the + * dangerous non-empty all-zero case. + */ +export function isZeroNormVector(vector: readonly number[]): boolean { + if (vector.length === 0) return false + for (let i = 0; i < vector.length; i++) { + if (vector[i] !== 0) return false + } + return true +} + /** * Calculates the Manhattan (L1) distance between two vectors. * Lower values indicate higher similarity. diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index d43559dc..00790a4a 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -588,8 +588,14 @@ export function validateAddParams(params: AddParams): void { ) } - // Validate vector dimensions if provided - if (params.vector) { + // Validate vector dimensions if provided. A length-0 vector is the + // "unvectored" shape — an explicit `vector: []` (e.g. the VFS root's + // permanently-vectorless creation, see + // VirtualFileSystem.doInitializeRoot()'s zero-norm fix) carries no + // dimension information, exactly like an absent vector or a deferred + // embed's internal stub, so it is exempt from the dimension check rather + // than refused as a "0-dimensional vector". + if (params.vector && params.vector.length > 0) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) @@ -607,6 +613,17 @@ export function validateUpdateParams(params: UpdateParams): void { // null/undefined means "no new data was given". const hasData = params.data !== undefined && params.data !== null if ((params as UpdateParams & { deferEmbedding?: boolean }).deferEmbedding === true) { + if (params.vector && params.vector.length === 0) { + // The nonsensical combination Leg D of the zero-norm/unvector-door law + // refuses: `vector: []` is the SANCTIONED UNVECTOR DOOR — an explicit + // instruction to remove the vector NOW, never "please embed" — so it + // cannot be paired with a request to defer an embed. + throw new Error( + `update(): 'vector: []' (the unvector door) cannot be combined with ` + + `'deferEmbedding: true' — an unvector is an explicit instruction to remove ` + + `the vector now, not a request to defer an embed. Drop one of the two.` + ) + } if (params.vector) { throw new Error( `update(): deferEmbedding cannot be combined with an explicit 'vector' — ` + @@ -643,8 +660,16 @@ export function validateUpdateParams(params: UpdateParams): void { throw new Error(`invalid NounType: ${params.type}`) } - // Validate vector dimensions if provided - if (params.vector) { + // Validate vector dimensions if provided. A length-0 vector is the + // SANCTIONED UNVECTOR DOOR (see brainy.ts update()'s matching comment): an + // explicit `vector: []` — or a real all-zero vector, normalized to `[]` + // upstream by the zero-norm law — carries no dimension information, + // exactly like validateAddParams's identical exemption, so it is exempt + // from the dimension check rather than refused as a "0-dimensional + // vector". (The `deferEmbedding` combination above already refuses + // `vector: []` paired with `deferEmbedding: true` — an empty array is + // truthy, so that guard fires unconditionally on any explicit `vector`.) + if (params.vector && params.vector.length > 0) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 90018863..59a16be4 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -11,6 +11,7 @@ import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' import { Entity, AddParams, RelateParams, FindParams, Relation } from '../types/brainy.types.js' import { NounType, VerbType } from '../types/graphTypes.js' +import { isZeroNormVector } from '../utils/distance.js' import { PathResolver } from './PathResolver.js' import { mimeDetector } from './MimeTypeDetector.js' import { @@ -89,16 +90,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Uses deterministic UUID format for storage compatibility private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' - // OPEN-PATH FIX: the dimension of the placeholder vector given to the VFS - // root when it is first created (see `doInitializeRoot`). Mirrors the - // built-in WASM embedding engine's fixed, hardcoded output size - // (all-MiniLM-L6-v2 — see src/embeddings/candle-wasm/src/lib.rs - // HIDDEN_SIZE and src/embeddings/EmbeddingManager.ts). Deliberately NOT - // derived from `brain.dimensions` — this constant only applies to the - // default-embedder branch, where the true output dimension is this fixed - // value by construction, never a moving target. - private static readonly VFS_ROOT_VECTOR_DIMENSIONS = 384 - /** * Construct a VFS bound to a Brainy instance. * @@ -241,9 +232,11 @@ export class VirtualFileSystem implements IVirtualFileSystem { private async doInitializeRoot(): Promise { const rootId = VirtualFileSystem.VFS_ROOT_ID - // Try to get existing root by fixed ID (O(1) lookup, not query) + // Try to get existing root by fixed ID (O(1) lookup, not query). + // includeVectors: true — the zero-norm migration below (leg 2) needs to + // inspect the persisted vector to detect the legacy placeholder shape. try { - const existingRoot = await this.brain.get(rootId) + const existingRoot = await this.brain.get(rootId, { includeVectors: true }) if (existingRoot) { // Root exists - verify metadata is correct @@ -260,6 +253,34 @@ export class VirtualFileSystem implements IVirtualFileSystem { }) } + // ZERO-NORM ROOT MIGRATION (one-time): a pre-fix store persisted the + // root with a REAL all-zero placeholder vector — lawful inside + // brainy (cosineDistance treats a zero-norm operand as MAXIMUM + // distance, see src/utils/distance.ts) but a "false attractor" for a + // downstream engine serving squared-euclidean distance, which cannot + // tell an all-zero vector apart from a legitimate origin point (a + // production incident silently darkened 150+ rows in a partner + // engine's index this way). THE LAW: a zero-norm vector is not a + // vector — it never crosses an engine boundary. Detect the legacy + // shape via NORM, not length or dimension (any real all-zero vector + // qualifies, not just the historical 384-dim one), and rewrite it to + // the "unvectored" `[]` shape through the sanctioned migration path + // (Brainy.unvectorNounForRootMigration — see its JSDoc), which keeps + // `getCanonicalCounts().vectors.all` honest and removes the row from + // the vector index. Idempotent: a store already on the new shape + // (vector.length === 0) takes the false branch below on every + // subsequent init() — a permanent no-op, not a one-time flag. + const existingVector = existingRoot.vector ?? [] + if (existingVector.length > 0 && isZeroNormVector(existingVector)) { + const migrated = await this.brain.unvectorNounForRootMigration(rootId) + if (migrated) { + console.log( + 'VFS: migrated root vector from the legacy all-zero placeholder to the ' + + 'unvectored shape (zero-norm vectors never cross an engine boundary)' + ) + } + } + return rootId } } catch (error) { @@ -277,31 +298,42 @@ export class VirtualFileSystem implements IVirtualFileSystem { // meant every writer's FIRST-EVER open forced the process-global WASM // engine to cold-compile its model (measured 90-140s on throttled // CPUs) before the brain could even finish init(). This branch only - // runs once per store (the root already exists — with a real vector — - // in every previously-opened production store; reopening never re-adds - // it), so the fix applies only to brand-new stores. + // runs once per store (a previously-opened store already has a root — + // see the migration above for the pre-fix shape — reopening never + // re-adds it), so the fix applies only to brand-new stores. // - // Chosen fix: an explicit all-zero vector, not `deferEmbedding: true`. - // `deferEmbedding` looked attractive (ack now, embed later) but its - // landing path (`kickEmbedWorker()`, called synchronously right after - // commit — see brainy.ts add()/update()) would still force the WASM - // engine to cold-compile within milliseconds of open, just off the - // awaited path instead of never paying it at all — worse than the - // explicit-vector path, which never touches the engine for this row. - // An all-zero vector is safe: `cosineDistance` (src/utils/distance.ts) - // explicitly returns the MAXIMUM distance whenever either operand's - // norm is zero, so the root can never rank ahead of real content in a - // similarity search, and HNSW indexes it like any other vector. + // ZERO-NORM LAW (current shape, superseding the historical all-zero + // placeholder): the root's vector is `[]` — the SAME "unvectored" + // empty-array shape used for a deferred embed's stub and any other + // not-yet-embedded row — never a real all-zero vector. A zero-norm + // vector is lawful inside brainy (`cosineDistance`, see + // src/utils/distance.ts, returns the MAXIMUM distance whenever either + // operand's norm is zero) but is a "false attractor" for a downstream + // engine serving squared-euclidean distance, which cannot tell a real + // all-zero vector apart from a legitimate origin point — it silently + // darkened 150+ rows in a partner engine's index in production. THE + // LAW: a zero-norm vector is not a vector — it never crosses an engine + // boundary. `vector: []` achieves the SAME cold-compile avoidance the + // original placeholder did (`add()`'s dimension-pin and HNSW-insert + // gates both key off `vector.length > 0`, so an empty vector never + // calls embed(), never pins `brain.dimensions`, and never reaches the + // vector index — see brainy.ts add()'s matching comments) while never + // persisting a searchable zero vector for a downstream engine to trip + // over. Deliberately NOT `deferEmbedding: true`: that flag's landing + // path (`kickEmbedWorker()`, called synchronously right after commit — + // see brainy.ts add()/update()) would still force the WASM engine to + // cold-compile within milliseconds of open (just off the awaited path + // instead of never paying it at all) AND would eventually embed the + // root's data for real, which this fix forbids — the root must NEVER + // be embedded, not merely "not yet". // - // Only the default WASM engine gets this treatment — its output - // dimension (384) is fixed and hardcoded, so the placeholder can never - // mis-pin `brain.dimensions` for it. A plugin-registered native - // 'embeddings' provider has no cold-compile cost AND may use a - // different dimension, so it keeps embedding the root for real (same - // as before this fix) rather than risk pinning the wrong dimension - // ahead of the caller's own first real embed. + // Only the default WASM engine gets this treatment — a plugin- + // registered native 'embeddings' provider has no cold-compile cost and + // may use a different dimension, so it keeps embedding the root for + // real (same as before this fix) rather than leave Brainy's own + // plumbing permanently unvectored on a store where embedding is cheap. const rootVector = this.brain.usesDefaultWasmEmbedder() - ? new Array(VirtualFileSystem.VFS_ROOT_VECTOR_DIMENSIONS).fill(0) + ? ([] as number[]) : undefined await this.brain.add({ diff --git a/tests/integration/canonical-count-ledger.test.ts b/tests/integration/canonical-count-ledger.test.ts index f3c8ce20..3d292e97 100644 --- a/tests/integration/canonical-count-ledger.test.ts +++ b/tests/integration/canonical-count-ledger.test.ts @@ -225,9 +225,12 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s } /** Baseline vectored count right after a fresh open() — init() creates a - * hidden system VFS-root noun that itself carries a real vector, so a - * brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off - * this baseline rather than hardcoding it away. */ + * hidden system VFS-root noun, but (the zero-norm root cure) it is + * deliberately UNVECTORED (`vector: []`, never a real all-zero + * placeholder — a zero-norm vector never crosses an engine boundary), so + * a brand-new store's `vectors.all` is 0. Tests still assert DELTAS off + * this baseline rather than hardcoding it away, in case that ever + * changes again. */ let baseline: number beforeEach(async () => { @@ -235,6 +238,7 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-')) brain = await open() baseline = (await brain.storage.getCanonicalCounts()).vectors.all + expect(baseline).toBe(0) // the unvectored VFS root contributes nothing }) afterEach(async () => { vi.restoreAllMocks() @@ -348,7 +352,7 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s brain = await open() const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun + expect(ledger.vectors.all).toBe(baseline + 1) // just the one non-deferred noun — the root is unvectored expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) expect(persisted.totalVectoredNounCount).toBe(baseline + 1) diff --git a/tests/integration/enumeration-population-law.test.ts b/tests/integration/enumeration-population-law.test.ts new file mode 100644 index 00000000..eaab7432 --- /dev/null +++ b/tests/integration/enumeration-population-law.test.ts @@ -0,0 +1,333 @@ +/** + * @module tests/integration/enumeration-population-law + * @description THE POPULATION LAW (ADR-008 G1): the unfiltered noun/verb walk + * and the canonical ALL scalar must agree on the population — a row's + * IDENTITY RECORD (metadata.json) is what defines membership; the vector leg + * is optional data, never a gate on visibility. Before this fix, the walk + * (getNounsWithPagination / getNounIdsWithPagination / getVerbsWithPagination) + * enumerated by keying on the VECTOR leg (`vectors.json`), so a row with + * metadata and no vector file was counted by the ledger (already + * metadata.json-keyed — see `rebuildTypeCounts`) but never yielded by the + * walk: a permanent "counted but invisible" phantom for any downstream + * consumer (a health-coverage row, an index-fill walk) that iterates the walk + * to account for the ledger's total. + * + * Two legs are pinned here: + * (a)/(b) LEG 1 — the walk re-keys on metadata.json. A fold-born + * metadata-only row (the exact shape `GenerationStore.replayFact` can + * leave behind, and the exact shape `writeNounRaw`/`writeVerbRaw` accept) + * must be YIELDED, hydrated with the sanctioned unvectored shape + * (`vector: []`) — not merely counted. + * + * For VERBS this closes only PARTIALLY: `sourceId`/`targetId` are + * HNSWVerb's structural core and live ONLY in the vector leg (never in + * metadata — see `RESERVED_RELATION_FIELDS` in reservedFields.ts, which + * does not include them). A metadata-only verb row therefore cannot be + * safely reconstructed without FABRICATING an edge's endpoints — which + * would silently create a phantom relationship, strictly worse than the + * original defect. The walk recovers the row when its metadata happens + * to carry `sourceId`/`targetId` (a defensive, forward-compatible + * fallback — never true for a CURRENT production write, but not + * disallowed either); otherwise it counts the row (ledger, unchanged) + * but loudly skips yielding it, logging the gap instead of hiding it. + * Closing this fully requires persisting `sourceId`/`targetId` in verb + * metadata — a schema change out of this task's scope; see the session + * report for the explicit call-out. + * + * (c)/(d) LEG 2 — the recovery fold's preserve-if-absent contract, exercised + * directly against `GenerationStore`/`FactLog` (below the `Brainy` API): + * a metadata-only after-image replayed over an already-vectored row must + * PRESERVE the existing vector leg (never delete it); a genuine tombstone + * (both legs absent) still removes both legs. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { randomUUID } from 'node:crypto' +import { Brainy } from '../../src/index.js' +import { GenerationStore } from '../../src/db/generationStore.js' +import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' +import { LOG_AUTHORITY_PATH } from '../../src/db/logAuthority.js' +import type { CommitFact } from '../../src/db/factLog.js' + +describe('enumeration population law — LEG 1 (identity-keyed walk)', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-population-law-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('(a) nouns.all equals the unfiltered walk-yield count with a fold-born metadata-only row present', async () => { + // Ordinary, fully-vectored background population. + await brain.add({ data: 'one', type: 'document' }) + await brain.add({ data: 'two', type: 'document' }) + await brain.flush() + + // THE EXACT PRE-FIX SHAPE: a metadata-only row against a FRESH id — no + // vector ever existed for it. Written through the raw primitive directly, + // exactly as `GenerationStore.replayFact` (the recovery fold) applies a + // replayed after-image whose vector leg came back null. + const freshId = randomUUID() + await brain.storage.writeNounRaw(freshId, { + metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 }, + vector: null + }) + + // writeNounRaw bypasses count bookkeeping on purpose (its own JSDoc) — the + // sanctioned recount brings the ledger scalar to ground truth. This walk + // was ALREADY metadata.json-keyed before this fix (rebuildTypeCounts), so + // the recount's answer does not depend on today's change. + await brain.repairIndex() + + const ledger = await brain.storage.getCanonicalCounts() + const walk = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } }) + + expect(walk.items.length).toBe(ledger.nouns.all) + expect(walk.totalCount).toBe(ledger.nouns.all) + + const yielded = walk.items.find((n: any) => n.id === freshId) + expect(yielded, 'the metadata-only row must be YIELDED, not merely counted').toBeDefined() + expect(yielded.vector).toEqual([]) + }) + + it('(a-ids) getNounIdsWithPagination (the zero-read unfiltered enumerator) also yields the metadata-only row', async () => { + await brain.add({ data: 'one', type: 'document' }) + await brain.flush() + + const freshId = randomUUID() + await brain.storage.writeNounRaw(freshId, { + metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 }, + vector: null + }) + await brain.repairIndex() + + const ledger = await brain.storage.getCanonicalCounts() + const page = await brain.storage.getNounIdsWithPagination({ limit: 1000, offset: 0 }) + expect(page.ids.length).toBe(ledger.nouns.all) + expect(page.ids).toContain(freshId) + }) + + it('(b) verbs.all counts a fold-born metadata-only row; the walk yields it when endpoints are recoverable from metadata, and loudly skips (never fabricates) when they are not', async () => { + const a = await brain.add({ data: 'a', type: 'document' }) + const b = await brain.add({ data: 'b', type: 'document' }) + await brain.relate({ from: a, to: b, type: 'relatedTo' }) + await brain.flush() + + // Case 1 — the REALISTIC production shape: metadata carries the verb + // type (a reserved field, kept for backward compat) but never + // sourceId/targetId — those are HNSWVerb's structural core and live + // ONLY in the vector leg. The walk cannot safely fabricate them (an + // empty-string endpoint would silently create a phantom edge), so this + // row is counted by the ledger but not yielded — a documented, + // loudly-logged gap, not a silent one. + const gapId = randomUUID() + await brain.storage.writeVerbRaw(gapId, { + metadata: { verb: 'relatedTo', createdAt: Date.now(), updatedAt: Date.now(), weight: 1 }, + vector: null + }) + + // Case 2 — endpoints ARE recoverable from metadata (never true for a + // current production write; modeled here as what a repair tool or a + // future schema could supply): the walk reconstructs and yields it. + const recoveredId = randomUUID() + await brain.storage.writeVerbRaw(recoveredId, { + metadata: { + verb: 'relatedTo', + sourceId: a, + targetId: b, + createdAt: Date.now(), + updatedAt: Date.now(), + weight: 1 + }, + vector: null + }) + + await brain.repairIndex() + const ledger = await brain.storage.getCanonicalCounts() + const walk = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } }) + + // The ledger counts every identity record — the real edge plus both + // synthetic metadata-only rows — unaffected by whether the walk can + // safely hydrate them. + expect(ledger.verbs.all).toBe(3) + + const recovered = walk.items.find((v: any) => v.id === recoveredId) + expect(recovered, 'endpoints recoverable from metadata must be yielded').toBeDefined() + expect(recovered.sourceId).toBe(a) + expect(recovered.targetId).toBe(b) + expect(recovered.vector).toEqual([]) + + // The documented gap: counted, not yielded — this is the one corner of + // the population law this task does NOT close (see the session report). + const gapped = walk.items.find((v: any) => v.id === gapId) + expect(gapped).toBeUndefined() + expect(walk.items.length).toBeLessThan(ledger.verbs.all) + }) +}) + +describe('enumeration population law — LEG 2 (fold preserve-if-absent, below the Brainy API)', () => { + /** A GenerationStore whose brain has already flipped to log authority — the + * precondition for `replayFact` (the recovery fold) to run at open(). */ + async function openLogAuthorityStore(): Promise<{ storage: MemoryStorage; store: GenerationStore }> { + const storage = new MemoryStorage() + await storage.init() + await storage.writeRawObject(LOG_AUTHORITY_PATH, { authority: 'log' }) + const store = new GenerationStore(storage) + await store.open() + return { storage, store } + } + + it('(c) nouns: a metadata-only after-image replayed over a vectored row PRESERVES the vector; it stays readable and the vectored ledger is untouched either way', async () => { + const { storage, store } = await openLogAuthorityStore() + const id = randomUUID() + const vectorRecord = { id, vector: [0.1, 0.2, 0.3], connections: {}, level: 0 } + + // Generation 1 — a real, honest commit: both legs land together. + await store.commitTransaction({ + touched: { nouns: [id], verbs: [] }, + execute: async () => { + await storage.writeNounRaw(id, { + metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 }, + vector: vectorRecord + }) + } + }) + const beforeVectoredCount = (await storage.getCanonicalCounts()).vectors.all + + // THE ANOMALOUS FACT, crafted directly (bypassing commitTransaction, + // whose honest read-after-write could never produce this on its own): + // metadata changed, vector leg null, while the row is STILL vectored on + // disk. This is exactly the shape the recovery fold must tolerate — + // modeling the confirmed production defect at the replay boundary. + const factLog = store.getFactLog()! + const anomalousFact: CommitFact = { + generation: 2, + timestamp: Date.now(), + ops: [ + { + kind: 'noun', + id, + record: { + metadata: { noun: 'document', createdAt: 1000, updatedAt: 2000, _rev: 2 }, + vector: null + } + } + ] + } + await factLog.append(anomalousFact) + await factLog.sync() + + // Reopen — a fresh GenerationStore over the SAME storage. Generation 2's + // fact sits above the (still generation-1) manifest, so it replays + // through the recovery fold — `replayFact`'s own call site. + const store2 = new GenerationStore(storage) + await store2.open() + + const after = await storage.readNounRaw(id) + expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull() + expect((after.vector as { vector: number[] }).vector).toEqual([0.1, 0.2, 0.3]) + expect((after.metadata as { updatedAt: number }).updatedAt).toBe(2000) // the new metadata DID apply + + // writeNounRaw bypasses ledger bookkeeping either way (by design — see + // its JSDoc), so this scalar is unaffected by the replay regardless of + // outcome; asserted for completeness against the task's exact wording. + const afterVectoredCount = (await storage.getCanonicalCounts()).vectors.all + expect(afterVectoredCount).toBe(beforeVectoredCount) + }) + + it('(c-verb) verbs: a metadata-only after-image replayed over a vectored edge PRESERVES the vector leg (sourceId/targetId/verb intact)', async () => { + const { storage, store } = await openLogAuthorityStore() + const id = randomUUID() + const sourceId = randomUUID() + const targetId = randomUUID() + const vectorRecord = { id, vector: [0.7, 0.8], connections: {}, verb: 'relatedTo', sourceId, targetId } + + await store.commitTransaction({ + touched: { nouns: [], verbs: [id] }, + execute: async () => { + await storage.writeVerbRaw(id, { + metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 1000, weight: 1 }, + vector: vectorRecord + }) + } + }) + + const factLog = store.getFactLog()! + const anomalousFact: CommitFact = { + generation: 2, + timestamp: Date.now(), + ops: [ + { + kind: 'verb', + id, + record: { + metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 2000, weight: 2 }, + vector: null + } + } + ] + } + await factLog.append(anomalousFact) + await factLog.sync() + + const store2 = new GenerationStore(storage) + await store2.open() + + const after = await storage.readVerbRaw(id) + expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull() + expect((after.vector as { sourceId: string }).sourceId).toBe(sourceId) + expect((after.vector as { targetId: string }).targetId).toBe(targetId) + expect((after.metadata as { weight: number }).weight).toBe(2) + }) + + it('(d) a genuine tombstone replay removes BOTH legs (never preserved)', async () => { + const { storage, store } = await openLogAuthorityStore() + const id = randomUUID() + const vectorRecord = { id, vector: [0.4, 0.5, 0.6], connections: {}, level: 0 } + + await store.commitTransaction({ + touched: { nouns: [id], verbs: [] }, + execute: async () => { + await storage.writeNounRaw(id, { + metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 }, + vector: vectorRecord + }) + } + }) + expect((await storage.readNounRaw(id)).vector).not.toBeNull() // sanity: it landed + + const factLog = store.getFactLog()! + await factLog.append({ + generation: 2, + timestamp: Date.now(), + ops: [{ kind: 'noun', id, record: null }] // a genuine tombstone — both legs absent + }) + await factLog.sync() + + const store2 = new GenerationStore(storage) + await store2.open() + + const after = await storage.readNounRaw(id) + expect(after.metadata, 'a genuine delete removes the metadata leg').toBeNull() + expect(after.vector, 'a genuine delete removes the vector leg too — preserve-if-absent never applies to a tombstone').toBeNull() + }) +}) diff --git a/tests/integration/index-skips-unvectored.test.ts b/tests/integration/index-skips-unvectored.test.ts new file mode 100644 index 00000000..c65aaa01 --- /dev/null +++ b/tests/integration/index-skips-unvectored.test.ts @@ -0,0 +1,253 @@ +/** + * @module tests/integration/index-skips-unvectored + * @description THE UNVECTORED-ROW CURE — two integration tests + * (`tests/lifecycle/biography.test.ts`'s Ch4/5/6 chapter and + * `tests/integration/clear-persistence.test.ts`'s multi-cycle test) started + * failing after a canonical-storage change made a vector-less row (the + * class-J shape: `vector: []`, e.g. the VFS root, a deferred embed not yet + * landed, or any other legitimately-unvectored canonical record) VISIBLE to + * the enumeration walk `getNounsWithPagination()` for the first time — before + * that change such rows were simply invisible to the walk. `hnswIndex.ts`'s + * `rebuild()` never guarded against that shape: it inserted every row the + * walk yielded into the live in-memory index, including ones with a length-0 + * vector, because `storage.getVectorIndexData()` derives its {level, + * connections} answer straight from the noun's OWN record — it returns + * non-null for ANY existing noun, whether or not that noun was ever actually + * indexed via `addItem()`. A vector-less node admitted into the graph could + * become the entry point (or occupy any graph position), and the very next + * real-vectored `addItem()` then ran a distance calculation against it — + * `cosineDistance` throws "Vectors must have the same dimensions" the moment + * one operand is a length-0 array. + * + * THE FIX, at two layers (`src/hnsw/hnswIndex.ts`): + * (1) FILL/REBUILD/LOAD consumers treat `vector.length === 0` as "unvectored — + * nothing to index" and skip the row (normal, not an error; one summary + * count line, never per-row spam) — `rebuild()`'s loop now checks this + * BEFORE ever creating a graph node, so an unvectored row can never + * become an index member, entry point, or dimension-setter. + * (2) THE INDEX ITSELF refuses a length-0 vector in `addItem()` / + * `updateItem()` with a typed `EmptyVectorIndexError`, loudly, instead of + * ever pinning `dimension = 0` or storing a vector-less node — so no + * future fill/rebuild/load path can silently poison the index even if it + * forgets law (1). + * + * Four legs pinned here: + * (a) `rebuild()` over a store mixing real-vectored rows and `vector: []` + * rows indexes ONLY the vectored ones — size === vectored count, + * dimension pinned to the real (non-zero) length. + * (b) `clear()` then real adds afterward never trip a dimension mismatch — + * the exact `clear-persistence.test.ts` regression shape, reproduced + * directly against the index/storage seam this module owns. + * (c) `index.addItem({ id, vector: [] })` throws `EmptyVectorIndexError` + * (and `updateItem` does too, for an existing node). + * (d) crash -> repair: the crashed generation's entities survive, the ledger + * recounts honestly, and a fresh real-vectored add afterward never trips + * a dimension mismatch against a leftover vector-less phantom. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { EmptyVectorIndexError } from '../../src/hnsw/hnswIndex.js' +import { abandonAsCrashed, openBrain as openKillMatrixBrain, uid, vec } from '../helpers/durabilityKillMatrix.js' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + for (const d of tmpDirs.splice(0)) { + try { + fs.rmSync(d, { recursive: true, force: true }) + } catch { + /* best-effort cleanup */ + } + } +}) + +/** A filesystem-backed brain with explicit vectors (no embedder needed) and + * manual persistence — mirrors `durabilityKillMatrix.ts`'s `openBrain` so + * every write in this module is explicit and provably durable. */ +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + persistence: { policy: 'manual' } + }) +} + +describe('HNSW index skips unvectored rows', () => { + it('(a) rebuild() indexes only vectored rows: size === vectored count, dimension pinned to the real length', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() + + // 5 real-vectored rows. + const vectoredIds: string[] = [] + for (let i = 0; i < 5; i++) { + const id = uid(`vectored-${i}`) + await brain.add({ id, data: `real entity ${i}`, type: NounType.Document, vector: vec(i) }) + vectoredIds.push(id) + } + // 3 explicit unvectored rows — the class-J "vector: []" shape, a normal, + // enumerable, countable canonical row that must never reach the index. + const unvectoredIds: string[] = [] + for (let i = 0; i < 3; i++) { + const id = uid(`unvectored-${i}`) + await brain.add({ id, data: `unvectored entity ${i}`, type: NounType.Document, vector: [] }) + unvectoredIds.push(id) + } + await brain.flush() + + // The canonical ledger already agrees before any rebuild: nouns.all + // counts every row (8 + the VFS root); vectors.all counts only the real + // ones (5) — the VFS root and the 3 explicit unvectored rows are excluded. + const ledgerBeforeReopen = await brain.storage.getCanonicalCounts() + expect(ledgerBeforeReopen.vectors.all).toBe(5) + expect(ledgerBeforeReopen.nouns.all).toBe(9) // 5 vectored + 3 unvectored + 1 VFS root + + await brain.close() + + // Reopen: open()'s index build IS hnswIndex.rebuild() run fresh from + // storage — this is the exact path that used to admit unvectored rows. + brain = openBrain(dir) + await brain.init() + + const status = await brain.getIndexStatus() + expect(status.hnswIndex.size, 'the rebuilt index must contain ONLY the 5 real-vectored rows').toBe(5) + + // Dimension is pinned to the REAL embedded length (384 via `vec()`), not + // 0 — adding a wrong-length vector must be refused naming that real + // dimension, proving no vector-less row ever set it. + const realDimension = vec(0).length + let mismatchMessage: string | undefined + try { + await brain.index.addItem({ id: uid('dimension-probe'), vector: vec(0).slice(0, realDimension - 1) }) + expect.fail('expected a dimension mismatch error') + } catch (err) { + mismatchMessage = (err as Error).message + } + expect(mismatchMessage).toContain(`expected ${realDimension}`) + + // Every unvectored row is still a normal, enumerable, readable canonical + // record — class-J semantics survive the rebuild fix untouched. + for (const id of unvectoredIds) { + const entity = await brain.get(id, { includeVectors: true }) + expect(entity, `unvectored entity ${id} must remain readable`).not.toBeNull() + expect(entity.vector).toEqual([]) + } + // A correct-dimension add succeeds cleanly against the pinned dimension. + const freshId = uid('post-reopen-fresh') + await expect(brain.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(50) })).resolves.toBe( + freshId + ) + + await brain.close() + }) + + it('(b) clear() then real adds afterward never trip a dimension mismatch (the clear-persistence regression shape)', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() // the VFS root (vector: []) is the store's only row + + await brain.clear() + await brain.close() + + // Reopen over a store whose only surviving row is the recreated, + // unvectored VFS root — this is exactly the shape that used to poison + // the entry point / dimension in `clear-persistence.test.ts`. + brain = openBrain(dir) + await brain.init() + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(0) + + const id1 = uid('after-clear-1') + await expect(brain.add({ id: id1, data: 'after clear 1', type: NounType.Document, vector: vec(1) })).resolves.toBe( + id1 + ) + const id2 = uid('after-clear-2') + await expect(brain.add({ id: id2, data: 'after clear 2', type: NounType.Document, vector: vec(2) })).resolves.toBe( + id2 + ) + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(2) + + await brain.close() + }) + + it('(c) index.addItem/updateItem refuse a length-0 vector with EmptyVectorIndexError', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + await expect(brain.index.addItem({ id: uid('empty-add'), vector: [] })).rejects.toThrow(EmptyVectorIndexError) + + // updateItem on an EXISTING (real-vectored) node must refuse the same way. + const existingId = uid('existing-for-update') + await brain.add({ id: existingId, data: 'existing', type: NounType.Document, vector: vec(9) }) + await expect(brain.index.updateItem({ id: existingId, vector: [] })).rejects.toThrow(EmptyVectorIndexError) + + // The index was never disturbed by either refused call. + expect((await brain.getIndexStatus()).hnswIndex.size).toBe(1) + + await brain.close() + }) + + it('(d) crash -> repair: the crashed generation survives, the ledger recounts honestly, and a fresh add afterward never trips a dimension mismatch', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-crash-')) + try { + let brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) + + // Baseline: real-vectored entities, durably flushed. + const baselineIds: string[] = [] + for (let i = 0; i < 5; i++) { + const id = uid(`baseline-${i}`) + await brain.add({ id, data: `baseline entity ${i}`, type: NounType.Document, vector: vec(i) }) + baselineIds.push(id) + } + await brain.flush() + + // Crash window: at-ack writes that are never flushed before the crash. + const crashedIds: string[] = [] + for (let i = 0; i < 4; i++) { + const id = uid(`crashed-${i}`) + await brain.add({ id, data: `crash-window entity ${i}`, type: NounType.Document, vector: vec(100 + i) }) + crashedIds.push(id) + } + await abandonAsCrashed(brain) + + // Reopen — logAuthority: 'adopt' replays the at-ack log for the crash window. + brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) + for (const id of [...baselineIds, ...crashedIds]) { + expect(await brain.get(id), `entity ${id} must survive the crash`).not.toBeNull() + } + + // Repair — must not disturb any entity, and must recount the ledger honestly. + const report = await brain.repairIndex() + expect(report.families.length).toBeGreaterThan(0) + for (const id of [...baselineIds, ...crashedIds]) { + expect(await brain.get(id), `entity ${id} must survive repair`).not.toBeNull() + } + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.vectors.all).toBe(baselineIds.length + crashedIds.length) + + // Second life: a fresh real-vectored add must never trip a dimension + // mismatch against a vector-less phantom left in the index — the exact + // mechanism `clear-persistence.test.ts` and the biography lane hit. + const secondLifeId = uid('second-life') + await expect( + brain.add({ id: secondLifeId, data: 'second life entity', type: NounType.Document, vector: vec(200) }) + ).resolves.toBe(secondLifeId) + expect(await brain.get(secondLifeId)).not.toBeNull() + + await brain.close() + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/integration/ledger-derivation-identity.test.ts b/tests/integration/ledger-derivation-identity.test.ts new file mode 100644 index 00000000..cb19af4a --- /dev/null +++ b/tests/integration/ledger-derivation-identity.test.ts @@ -0,0 +1,195 @@ +/** + * @module tests/integration/ledger-derivation-identity + * @description The ALL-visibility ledger scalars are an IDENTITY-RECORD + * count, never a container count. A pre-8.3.1 partial-delete defect can + * leave a "ghost" container (a stale `vectors.json` with no metadata content + * leg) or a "scar" container (an empty `entities////` + * directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need + * the metadata content leg — yet the legacy derivation counted one entity + * per id DIRECTORY, so orphaned containers inflated the ALL scalars forever + * (they were never clamped and never re-derived). Laws under test: + * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per + * metadata content leg (`metadata.json` or `.json.gz`), the same test + * `pruneOrphanedEntities()` uses, so the two agree by construction. + * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that + * carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'` + * stamp predates this fix; loading it marks `suspect = true` from a + * single field read alone, never a directory walk, and warns exactly + * once naming the cause. + * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned + * containers, recounts from the canonical metadata.json walk, and + * re-stamps — suspect clears and the ALL scalar is exact again. + * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store + * with no counts.json stamps as it writes, so a brand-new store never + * carries the legacy signature. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, FileSystemStorage } from '../../src/index.js' +import { prodLog } from '../../src/utils/logger.js' + +const countsPath = (root: string) => path.join(root, '_system', 'counts.json') + +/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */ +function plantGhost(root: string, shard: string, id: string): void { + const idDir = path.join(root, 'entities', 'nouns', shard, id) + fs.mkdirSync(idDir, { recursive: true }) + fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] })) +} + +/** Plant a scar container: an empty id directory, no legs at all. */ +function plantScar(root: string, shard: string, id: string): void { + fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true }) +} + +describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => { + let dir: string + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(() => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-')) + }) + afterEach(() => { + vi.restoreAllMocks() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => { + let brain = await open() + const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone + for (let i = 0; i < 3; i++) { + await brain.add({ data: `real ${i}`, type: 'document' }) + } + await brain.flush() + const realTotal = baseline + 3 + await brain.close() + + // 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) — + // neither is a live entity. + for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) + for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) + + // Remove counts.json so open() re-derives from scratch (the one-time + // legacy/lost-file derivation path). + fs.rmSync(countsPath(dir), { force: true }) + + brain = await open() + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing + expect(ledger.suspect).toBe(false) + + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.totalNounCountAll).toBe(realTotal) + expect(raw.allCountsDerivedBy).toBe('identity-record') + + await brain.close() + }) + + it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => { + let brain = await open() + await brain.add({ data: 'one', type: 'document' }) + await brain.add({ data: 'two', type: 'document' }) + await brain.flush() + await brain.close() + + // Confirm a normal close under the fix DOES stamp — then strip the stamp + // to simulate a counts.json produced before this fix existed. + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.allCountsDerivedBy).toBe('identity-record') + expect(typeof raw.totalNounCountAll).toBe('number') + expect(typeof raw.totalVerbCountAll).toBe('number') + expect(typeof raw.totalVectoredNounCount).toBe('number') + delete raw.allCountsDerivedBy + fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) + + const warnSpy = vi.spyOn(prodLog, 'warn') + // The two derivation walks live on FileSystemStorage's prototype — + // spying here (rather than on fs.promises.readdir globally) isolates + // THIS code path's behavior from unrelated walks elsewhere in the open + // sequence (a separate, pre-existing engine's own O(store) cost — not + // this fix's concern, and not something this pin should be sensitive + // to). Neither derivation method may run: the stamp check is a field + // read on the already-parsed counts.json, nothing more. + const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities') + const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount') + + brain = await open() + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(true) + + const stampWarnings = warnSpy.mock.calls.filter( + ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule') + ) + expect(stampWarnings.length).toBe(1) // exactly one, loud + + expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk + expect(scanVectoredSpy).not.toHaveBeenCalled() + + await brain.close() + }) + + it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => { + let brain = await open() + const baseline = (await brain.storage.getCanonicalCounts()).nouns.all + for (let i = 0; i < 3; i++) { + await brain.add({ data: `real ${i}`, type: 'document' }) + } + await brain.flush() + const realTotal = baseline + 3 + await brain.close() + + for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) + for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) + + // Force the legacy (unstamped, container-rule-inflated) shape directly — + // the shape a pre-existing production store actually carries. + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars + delete raw.allCountsDerivedBy + fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) + + brain = await open() + expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load + + await brain.repairIndex() + + let ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again + + const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(persisted.allCountsDerivedBy).toBe('identity-record') + expect(persisted.allCountsSuspect).toBe(false) + expect(persisted.totalNounCountAll).toBe(realTotal) + + await brain.close() + brain = await open() + ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.nouns.all).toBe(realTotal) + await brain.close() + }) + + it('(d) a fresh store derives with the stamp and is never suspect', async () => { + const brain = await open() + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.allCountsDerivedBy).toBe('identity-record') + await brain.close() + }) +}) diff --git a/tests/integration/vector-leg-open-build.test.ts b/tests/integration/vector-leg-open-build.test.ts index a1ccaefa..ea841545 100644 --- a/tests/integration/vector-leg-open-build.test.ts +++ b/tests/integration/vector-leg-open-build.test.ts @@ -176,21 +176,22 @@ describe('vector-leg open-build (two-engine gate, last red)', () => { }) it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => { - // ARCHITECTURAL NOTE (found while building this pin): every brainy store - // carries ONE permanent, always-vectored noun beyond user data — the VFS - // root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, - // src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero - // (but non-empty, length-384) vector on EVERY store's first open — never - // deferred (a deliberate WASM-cold-compile-avoidance fix, see that - // file's comment) — and VFS init unconditionally re-creates it if - // missing, before the rebuild gate ever runs. A literal "0 vectored - // nouns" store is therefore unreachable through the public API; a - // brand-new store's `vectors.all` floor is 1, not 0. This pin verifies - // the law the task names in the ACHIEVABLE form: nouns whose embed is - // still deferred/unlanded contribute NOTHING to the vectored-noun ledger - // — the coverage-gap comparison sees exactly the root (1), never - // root+deferred — and semantic search over deferred-only user content - // honestly returns `[]` (no error, no false "coverage restored" claim). + // ARCHITECTURAL NOTE (updated by the zero-norm root cure): every brainy + // store carries ONE permanent VFS root noun beyond user data + // (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, + // src/vfs/VirtualFileSystem.ts), created (or, on a pre-fix store, + // migrated) on every open — but it is deliberately UNVECTORED (vector + // `[]`), never a real all-zero placeholder: a zero-norm vector is not a + // vector and never crosses an engine boundary (see that file's + // doInitializeRoot() comment). It therefore contributes NOTHING to the + // vectored-noun ledger — a brand-new store's `vectors.all` floor is 0, + // not 1. This pin verifies the law the task names in the ACHIEVABLE + // form: nouns whose embed is still deferred/unlanded contribute NOTHING + // to the vectored-noun ledger either — the coverage-gap comparison sees + // exactly the baseline (the root, contributing 0), never + // baseline+deferred — and semantic search over deferred-only user + // content honestly returns `[]` (no error, no false "coverage restored" + // claim). const dir = mkTmp() const build: any = new Brainy({ @@ -202,6 +203,8 @@ describe('vector-leg open-build (two-engine gate, last red)', () => { }) await build.init() const rootOnlyLedger = await build.storage.getCanonicalCounts() + // THE NEW LAW: the root is unvectored — a brand-new store's floor is 0. + expect(rootOnlyLedger.vectors.all).toBe(0) // Block the embedder permanently so every add below stays deferred and // unlanded for the rest of this test (a fast deterministic embedder // could otherwise land it before we ever observe the "still 0 extra" diff --git a/tests/integration/vfs-root-zero-norm.test.ts b/tests/integration/vfs-root-zero-norm.test.ts new file mode 100644 index 00000000..577ae7ee --- /dev/null +++ b/tests/integration/vfs-root-zero-norm.test.ts @@ -0,0 +1,216 @@ +/** + * @module tests/integration/vfs-root-zero-norm + * @description THE ZERO-NORM ROOT CURE — a production incident traced 150+ + * darkened rows in a downstream engine's index to the VFS root's persisted + * ALL-ZERO placeholder vector: lawful inside brainy (`cosineDistance` + * treats a zero-norm operand as MAXIMUM distance, src/utils/distance.ts) + * but a "false attractor" for an engine serving squared-euclidean distance, + * which cannot tell a real all-zero vector apart from a legitimate origin + * point. THE LAW: a zero-norm vector is not a vector — it never crosses an + * engine boundary. + * + * Three legs pinned here: + * (a) the root persists NO zeros — a brand-new store creates it with + * vector `[]` (the "unvectored" shape), absent from the HNSW index, and + * the canonical vectored-noun ledger does not count it. + * (b) a ONE-TIME migration heals an existing (pre-fix) store: an old-shape + * root (a REAL all-zero vector, genuinely indexed and ledgered — the + * harness reproduces exactly what a pre-fix store looked like on disk) + * is rewritten to `[]` on the next `init()`, the ledger is decremented + * through the sanctioned path, and a second `init()` is a no-op. + * (c) THE CANONICAL-WRITE NORMALIZATION (Leg A of the follow-up + * zero-norm/unvector-door fix): an entity added with an EXPLICIT + * all-zero vector (any dimension) is normalized to the "unvectored" + * `[]` shape BEFORE the canonical write, the ledger flag, and the index + * ops ever see it — the canonical write still succeeds, loudly, and the + * vector-index insert never happens (nothing to index). Supersedes the + * original "canonical keeps the zero vector, only the index refuses" + * shape: a downstream engine's health-report gate reads the canonical + * ledger directly, so leaving a zero-norm vector on the canonical side + * re-opened the exact false-attractor risk this whole fix closes. + * (d) the migrated root never surfaces in `find()` results (it was already + * hidden behind `visibility: 'system'` — this pin holds regardless). + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { prodLog } from '../../src/utils/logger.js' + +const ROOT_ID = '00000000-0000-0000-0000-000000000000' + +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vfs-root-zero-norm-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + vi.restoreAllMocks() + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) +} + +describe('VFS root zero-norm cure', () => { + it('(a) a brand-new store persists the root with vector [], absent from the HNSW index, and the canonical ledger counts it unvectored', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const root = await brain.get(ROOT_ID, { includeVectors: true }) + expect(root).not.toBeNull() + expect(root.vector).toEqual([]) + + const status = await brain.getIndexStatus() + expect(status.hnswIndex.size).toBe(0) + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.vectors.all).toBe(0) + + await brain.close() + }) + + it('(b) an old-shape store (a real all-zero placeholder root) migrates to [] exactly once on init; the ledger is decremented through the sanctioned path; a second init is a no-op', async () => { + const dir = mkTmp() + + // SESSION 1 — build the store, then hand-rewrite the root to the LEGACY + // shape: a REAL all-zero 384-dim vector, genuinely inserted into the + // vector index and genuinely counted by the vectored-noun ledger — + // reproducing exactly what a pre-fix store's root looked like on disk + // (the pre-fix add() always indexed + counted it). `index.addItem` is + // called directly (bypassing AddToVectorIndexOperation's own zero-norm + // belt, added by this same fix) precisely because the pre-fix code path + // had no such belt — this harness must match history, not the cure. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.flush() + + const ledgerBeforeMigration = await brain.storage.getCanonicalCounts() + expect(ledgerBeforeMigration.vectors.all).toBe(1) + await brain.close() + + // SESSION 2 — reopen: VFS init must detect the legacy shape and migrate. + // Spy on the sanctioned migration method itself (not console output — + // `silent: true` monkey-patches `console.log` to a no-op INSIDE init(), + // which would silently swallow any pre-installed console spy too). + brain = openBrain(dir) + const migrateSpy = vi.spyOn(brain, 'unvectorNounForRootMigration') + await brain.init() + + expect(migrateSpy).toHaveBeenCalledTimes(1) + expect(migrateSpy).toHaveBeenCalledWith(ROOT_ID) + await expect(migrateSpy.mock.results[0].value).resolves.toBe(true) + + const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) + expect(migratedRoot.vector).toEqual([]) + + const ledgerAfterMigration = await brain.storage.getCanonicalCounts() + expect(ledgerAfterMigration.vectors.all).toBe(0) + + const statusAfterMigration = await brain.getIndexStatus() + expect(statusAfterMigration.hnswIndex.size).toBe(0) + + await brain.flush() + await brain.close() + + // SESSION 3 — reopen again: the migration is a permanent no-op, not a + // one-time flag that silently re-drifts or re-fires. The zero-norm + // detection at the VFS init site never even calls the migration method + // again — the root's vector is already `[]`. + brain = openBrain(dir) + const migrateSpy2 = vi.spyOn(brain, 'unvectorNounForRootMigration') + await brain.init() + + expect(migrateSpy2).not.toHaveBeenCalled() + + const rootAfterSecondInit = await brain.get(ROOT_ID, { includeVectors: true }) + expect(rootAfterSecondInit.vector).toEqual([]) + + const ledgerAfterSecondInit = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecondInit.vectors.all).toBe(0) + + await brain.close() + }) + + it('(c) canonical-write normalization: an entity added with an explicit all-zero vector persists UNVECTORED ([]), loudly, and never reaches the vector index', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const warnSpy = vi.spyOn(prodLog, 'warn') + + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const ledgerBefore = await brain.storage.getCanonicalCounts() + const zeroVector = new Array(384).fill(0) + const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) + + // The canonical write succeeded — but the zero-norm vector was + // normalized to the "unvectored" `[]` shape BEFORE it was persisted + // (Leg A: a zero-norm vector is not a vector — it never crosses an + // engine boundary, canonical side included). + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + // Nothing to index — the vector-index size never moved, and the + // vectored-noun ledger never counted this row. + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore) + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + // The normalization was LOUD and named the entity. + const loudCall = warnSpy.mock.calls.find( + (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') + ) + expect(loudCall).toBeDefined() + + await brain.close() + }) + + it('(d) find() over a store whose root has been migrated never returns the root (already hidden behind visibility: system — pinned anyway)', async () => { + const dir = mkTmp() + + // Build an old-shape store (same harness as pin (b)) and let it migrate. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.add({ data: 'a document about technology', type: NounType.Document }) + await brain.flush() + await brain.close() + + brain = openBrain(dir) // migrates on init() + await brain.init() + + const results = await brain.find({ query: 'technology', limit: 10 }) + expect(results.some((r: any) => r.id === ROOT_ID)).toBe(false) + + // Even asking explicitly for system-tier entities must never surface the + // root as a semantic-search HIT (it carries no vector to match against). + const resultsIncludingSystem = await brain.find({ query: 'technology', limit: 10, includeSystem: true }) + expect(resultsIncludingSystem.some((r: any) => r.id === ROOT_ID)).toBe(false) + + await brain.close() + }) +}) diff --git a/tests/integration/zero-norm-unvector-door.test.ts b/tests/integration/zero-norm-unvector-door.test.ts new file mode 100644 index 00000000..d3290010 --- /dev/null +++ b/tests/integration/zero-norm-unvector-door.test.ts @@ -0,0 +1,399 @@ +/** + * @module tests/integration/zero-norm-unvector-door + * @description THE SEAM LAW, GENERALIZED: "a zero-norm vector is not a + * vector — it never crosses an engine boundary." `tests/integration/ + * vfs-root-zero-norm.test.ts` pins the VFS-root-specific cure; this file + * pins the follow-up that generalizes it to every write path plus the + * sanctioned door for shedding a vector on purpose. + * + * Four legs pinned here: + * (A) THE CANONICAL WRITE NORMALIZES ZERO-NORM TO `[]` — `add()` (single and + * `transact()`) persists an explicit real all-zero vector as the + * "unvectored" `[]` shape, loudly, before the ledger flag/dimension + * pin/index ops ever see it. The canonical write still succeeds. + * (B) THE LEGACY DERIVATION IS ZERO-NORM-AWARE — a lost/corrupted + * `counts.json`'s one-time re-derivation walk excludes a persisted + * zero-norm row from the vectored-noun scalar, matching the live + * ledger's definition of "vectored". + * (C) THE LEGACY VFS ROOT MIGRATES AT OPEN, BEFORE THE GATE, IN O(1) — a + * store whose ONLY vectored row is a legacy all-zero VFS root opens + * clean (no `VectorIndexNotReadyError`), via one fixed-path read, never + * a listing. + * (D) THE UNVECTOR DOOR — `update({ id, vector: [] })` (and the same op + * inside `transact()`) is the sanctioned, idempotent way to shed a + * vector on purpose: ledger decrement exactly once, index removal, no + * re-embed, and a pending deferred-embed marker is cleared rather than + * left to re-vectorize the row later. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' +import { prodLog } from '../../src/utils/logger.js' +import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' +import { BaseStorage } from '../../src/storage/baseStorage.js' + +const ROOT_ID = '00000000-0000-0000-0000-000000000000' + +process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + +const tmpDirs: string[] = [] +function mkTmp(): string { + const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-zero-norm-unvector-')) + tmpDirs.push(d) + return d +} +afterEach(() => { + vi.restoreAllMocks() + for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) +}) + +function openBrain(dir: string): any { + return new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) +} + +const countsPath = (root: string) => path.join(root, '_system', 'counts.json') + +describe('zero-norm canonical write + the sanctioned unvector door', () => { + it('(A1) add() with an explicit all-zero vector persists [], warns loudly, never indexes, and the ledger is unchanged', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + const warnSpy = vi.spyOn(prodLog, 'warn') + + const zeroVector = new Array(384).fill(0) + const id = await brain.add({ data: 'zero-norm add', type: NounType.Document, vector: zeroVector }) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore) + + const loud = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') + ) + expect(loud).toBeDefined() + + await brain.close() + }) + + it('(A2) transact() add with an explicit all-zero vector — the same canonical normalization', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const warnSpy = vi.spyOn(prodLog, 'warn') + const zeroVector = new Array(384).fill(0) + const id = 'aaaaaaaa-0000-4000-8000-000000000001' + + await brain.transact([ + { op: 'add', id, type: NounType.Document, data: 'zero-norm transact add', vector: zeroVector } + ]) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity).not.toBeNull() + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) + + const loud = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') + ) + expect(loud).toBeDefined() + + await brain.close() + }) + + it('(B) the legacy counts.json derivation excludes a persisted zero-norm row from the vectored-noun scalar', async () => { + const dir = mkTmp() + let brain = openBrain(dir) + await brain.init() + + // The VFS root alone (unvectored — []) — the floor. + const baseline = (await brain.storage.getCanonicalCounts()).vectors.all + + const realId = await brain.add({ data: 'a real vectored document', type: NounType.Document }) + + // Plant the legacy all-zero shape BY HAND: a genuine identity record + // (via add(), so it has real metadata) whose vector leg is then + // overwritten directly through the raw storage primitive — bypassing + // Leg A's canonical-write normalization entirely (brain.storage.saveNoun + // is not Brainy.add()/update()'s normalized path) — reproducing exactly + // what a pre-fix store could have persisted on disk. + const zeroId = await brain.add({ data: 'a legacy zero-norm document', type: NounType.Document }) + const zeroVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: zeroId, vector: zeroVector, connections: new Map(), level: 0 }) + + await brain.flush() + await brain.close() + + // Remove counts.json so the next open re-derives from scratch (the + // one-time legacy/lost-file derivation path — Leg B). + fs.rmSync(countsPath(dir), { force: true }) + + brain = openBrain(dir) + await brain.init() + const ledger = await brain.storage.getCanonicalCounts() + // Only realId counts; zeroId's persisted all-zero vector does not. + expect(ledger.vectors.all).toBe(baseline + 1) + + await brain.close() + }) + + it('(C) a legacy all-zero VFS root as the ONLY vectored row: open succeeds with no not-ready error, via an O(1) fixed-path read (no entities-tree readdir), and the ledger is 0 after open', async () => { + const dir = mkTmp() + + // SESSION 1 — build the legacy shape: the root is a REAL all-zero + // 384-dim vector, genuinely indexed and genuinely ledgered — exactly + // what a pre-fix store's root looked like on disk (see + // vfs-root-zero-norm.test.ts pin (b) for the identical harness). + // `index.addItem` is called directly (bypassing the transactional + // zero-norm belt) because the pre-fix code path had no such belt — this + // harness must match history, not the cure. No other entity is added, + // so the root is the store's ONLY vectored row. + let brain = openBrain(dir) + await brain.init() + const oldVector = new Array(384).fill(0) + await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) + await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) + await brain.storage.noteVectorLanded(ROOT_ID) + await brain.storage.persistCounts() + await brain.flush() + expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(1) + await brain.close() + + // SESSION 2 — reopen with a FAKE native vector provider that claims + // `serving: true` at `size()===0` (the exact shape a downstream + // engine's own health report can legitimately carry — same technique as + // tests/integration/vector-leg-open-build.test.ts). This is the ONLY + // codepath where the vector-leg open gate's FAIL-TYPED throw + // (VectorIndexNotReadyError) can fire; the built-in JS engine alone + // never reaches it (the size-heuristic branch just rebuilds instead) — + // so this is the faithful reproduction of the incident Leg C closes. + const readdirCalls: string[] = [] + const originalReaddir = fs.promises.readdir.bind(fs.promises) + vi.spyOn(fs.promises, 'readdir').mockImplementation(((...args: any[]) => { + readdirCalls.push(String(args[0])) + return (originalReaddir as any)(...args) + }) as any) + + // Spy at the PROTOTYPE level (BaseStorage.getNoun) — the new brain's + // storage instance does not exist until init() runs, so an + // instance-level spy cannot be installed beforehand. Records the + // readdir-call delta across the FIRST call made with the root id — + // Leg C's own fixed-path read — proving it needs no directory listing. + let readdirDeltaDuringRootRead: number | null = null + const originalGetNoun = BaseStorage.prototype.getNoun + vi.spyOn(BaseStorage.prototype, 'getNoun').mockImplementation(async function ( + this: unknown, + id: string + ) { + const before = readdirCalls.length + const result = await originalGetNoun.call(this as BaseStorage, id) + if (id === ROOT_ID && readdirDeltaDuringRootRead === null) { + readdirDeltaDuringRootRead = readdirCalls.length - before + } + return result + }) + + brain = openBrain(dir) + brain.use({ + name: 'fake-native-vector-unledgered-coverage', + activate: async (ctx: any) => { + ctx.registerProvider('vector', (config: any, distance: any, options: any) => { + const real = new JsHnswVectorIndex(config, distance, options) + let rebuilt = false + const originalRebuild = real.rebuild.bind(real) + ;(real as any).rebuild = async (...args: any[]) => { + const r = await originalRebuild(...args) + rebuilt = true + return r + } + const originalSize = real.size.bind(real) + ;(real as any).size = () => (rebuilt ? originalSize() : 0) + ;(real as any).healthReport = () => ({ + provider: 'vector', + healthy: true, + serving: true, + invariants: [], + checkedAt: Date.now(), + durationMs: 0, + generation: 1, + unledgered: ['vector-coverage'] + }) + return real + }) + return true + } + }) + + // Must NOT throw VectorIndexNotReadyError (or anything else) — a + // near-empty store whose only vectored row is the zero-norm root must + // never go dark. + await brain.init() + + const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) + expect(migratedRoot.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(0) + + expect(readdirDeltaDuringRootRead).toBe(0) + + await brain.close() + }) + + describe('the sanctioned unvector door', () => { + it('(D1) update({ id, vector: [] }) unvectors a real vectored row — canonical [], removed from the index, ledger decremented by exactly 1, no embed call', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + await brain.flush() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + + const embedSpy = vi.spyOn(brain, 'embed') + await brain.update({ id, vector: [] }) + expect(embedSpy).not.toHaveBeenCalled() + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore - 1) + + await brain.close() + }) + + it('(D2) idempotent: a second update({ id, vector: [] }) on an already-unvectored row is a true no-op — no error, no further decrement', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + await brain.flush() + + await brain.update({ id, vector: [] }) + const ledgerAfterFirst = await brain.storage.getCanonicalCounts() + + await brain.update({ id, vector: [] }) + const ledgerAfterSecond = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfterFirst.vectors.all) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + await brain.close() + }) + + it('(D3) a PENDING deferred-embed row: the unvector door clears the marker; awaitPendingEmbeds() then leaves it unvectored', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + // Prevent the background worker from ever actually running — it is + // fire-and-forget from add(), and a real run would race this test's + // own assertions (see tests/integration/vector-leg-open-build.test.ts + // for the same concern). This isolates exactly the marker-clearing + // behavior under test. + vi.spyOn(brain as any, 'kickEmbedWorker').mockImplementation(() => {}) + + const id = await brain.add({ + data: 'deferred content, never embedded', + type: NounType.Document, + deferEmbedding: true + }) + expect(brain.pendingEmbedCount()).toBe(1) + + const warnSpy = vi.spyOn(prodLog, 'warn') + await brain.update({ id, vector: [] }) + + expect(brain.pendingEmbedCount()).toBe(0) + const clearedWarn = warnSpy.mock.calls.find( + (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('pending') + ) + expect(clearedWarn).toBeDefined() + + // The barrier must not hang and must not re-vectorize the row — the + // worker (still mocked to a no-op) never runs again. + await brain.awaitPendingEmbeds() + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + await brain.close() + }) + + it('(D4) update({ vector: [], deferEmbedding: true }) is a typed refusal — the unvector door cannot be paired with a deferred embed', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document', type: NounType.Document }) + const before = await brain.get(id, { includeVectors: true }) + + await expect( + brain.update({ id, vector: [], deferEmbedding: true }) + ).rejects.toThrow(/unvector door/i) + + // Refused before any write — the row is untouched. + const after = await brain.get(id, { includeVectors: true }) + expect(after.vector).toEqual(before.vector) + + await brain.close() + }) + + it('(D5) the transact() twin of the unvector door decrements the ledger exactly once, and is idempotent on a second call', async () => { + const dir = mkTmp() + const brain = openBrain(dir) + await brain.init() + + const id = await brain.add({ data: 'a real document for transact unvector', type: NounType.Document }) + await brain.flush() + + const ledgerBefore = await brain.storage.getCanonicalCounts() + const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size + + await brain.transact([{ op: 'update', id, vector: [] }]) + + const entity = await brain.get(id, { includeVectors: true }) + expect(entity.vector).toEqual([]) + + const ledgerAfter = await brain.storage.getCanonicalCounts() + expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) + + const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size + expect(sizeAfter).toBe(sizeBefore - 1) + + // Idempotent through transact() too. + await brain.transact([{ op: 'update', id, vector: [] }]) + const ledgerAfterSecond = await brain.storage.getCanonicalCounts() + expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfter.vectors.all) + + await brain.close() + }) + }) +}) diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts index 8b274fce..274f0ef0 100644 --- a/tests/lifecycle/biography.test.ts +++ b/tests/lifecycle/biography.test.ts @@ -382,9 +382,17 @@ describe.sequential('lifecycle — the working store', () => { }, // Every noun this biography ever adds carries an explicit/computed // vector (the harness never defers an embed), so the vectored-noun - // scalar tracks nouns.all exactly. + // scalar tracks nouns.all exactly EXCEPT for the VFS root counted + // in `vfsBaselineNouns`: the root is deliberately persisted with + // `vector: []` (the sanctioned "unvectored" shape — see + // VirtualFileSystem.doInitializeRoot()'s zero-norm-avoidance + // comment) so it never pays the WASM engine's cold-compile cost and + // never crosses an engine boundary as a false attractor. It is the + // ONE hidden-tier record `vfsBaselineNouns` represents (see + // biographyHarness's module header), so it is excluded here even + // though it counts toward `nouns.all`. vectors: { - all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns + all: aliveEntities.length + model.vfsFileNouns }, suspect: false })