diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b154af5..ec925642 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,23 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.4.2-rc.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.1...v10.4.2-rc.1) (2026-08-27) - -- Merge branch 'next/zero-norm-unvector-door' (9b84ef5b) -- fix(vectors): a zero-norm vector is not a vector, canonical side included, plus the sanctioned unvector door (0de76659) -- fix(hnsw): skip unvectored rows on rebuild; refuse empty vectors in the index (8fc553b1) -- fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load (fd6b4ce4) -- Merge branch 'next/enumeration-identity-rekey' (204d74c1) -- fix(storage): enumeration re-keys on the identity record, not the vector leg (f8d8ce16) -- fix(init): rethrow plugin activation failures with the original error as cause so the originating frame survives to the caller (2496e09a) -- Merge branch 'next/vfs-root-zero-norm' (4c7b0fab) -- fix(vfs): the VFS root never persists a zero-norm vector (c6cc0de9) -- build: derive generated-file stamps from git commit time, not wall clock (8a5c1245) -- Merge remote-tracking branch 'origin/release/10.4.1' (aad9e2ee) -- docs(concepts): the serving law — a failure is graded by whether an answer could be wrong, never by the cost of the fix; reads refuse per family (2914e0eb) -- chore(release): 10.4.1-rc.1 (7870dc40) - - ### [10.4.1](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0...v10.4.1) (2026-08-26) - fix(reads): the read gate is per-family; a write carrying unchanged data never re-embeds (c039411e) diff --git a/docs/concepts/index-health.md b/docs/concepts/index-health.md index 96199f11..18bf7010 100644 --- a/docs/concepts/index-health.md +++ b/docs/concepts/index-health.md @@ -68,19 +68,6 @@ a maintenance window, a divergence `repairIndex()` will clean up on its own schedule. `serving: false` is not benign. It means this provider is refusing to answer, on its own word, right now. -**How a failure gets its grade — the serving law.** A provider grades `heal` by -one question only: *could an answer be wrong?* — never *how expensive is the -fix?* A missing-postings shortfall, however large, is `heal: 'repair'` (re-post -exactly what the ledger names, reads serving throughout); it can never withhold -serving just because healing it takes work. `serving` is withheld only by a -small, named set of rebuild-graded conditions — the index not initialized, its -durable state absent, a manifest naming files that are not resident, a replay -that did not complete cleanly — the states in which an answer could genuinely be -wrong. And a read is only ever refused by the family it actually consults: a -metadata filter is answered by the metadata index alone, vector search by the -vector index, traversal by the graph index — one family's refusal never blocks -another family's reads. - ## Reads refuse — they never rebuild A query that reaches a not-serving provider does not trigger a rebuild from inside diff --git a/package-lock.json b/package-lock.json index 889c67c8..70635835 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.1", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index 214feeac..4125b073 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.2-rc.1", + "version": "10.4.1", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/scripts/buildEmbeddedPatterns.ts b/scripts/buildEmbeddedPatterns.ts index c046df45..73e51224 100644 --- a/scripts/buildEmbeddedPatterns.ts +++ b/scripts/buildEmbeddedPatterns.ts @@ -10,7 +10,6 @@ import { TransformerEmbedding } from '../src/utils/embedding.js' import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' -import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -98,22 +97,13 @@ async function buildEmbeddedPatterns() { // Convert to base64 for embedding in TypeScript const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - - // Deterministic stamp: derived from the git commit time of this - // generator's inputs, never from wall-clock time — two builds of the - // same source tree must produce byte-identical output. - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') - const generatedStamp = resolveDeterministicStamp( - [path.join(__dirname, 'buildEmbeddedPatterns.ts'), libraryPath], - outputPath - ) - + // Generate TypeScript file with everything embedded const tsContent = `/** * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${generatedStamp} + * Generated: ${new Date().toISOString()} * Patterns: ${libraryData.patterns.length} * Coverage: 94-98% of all queries * @@ -207,6 +197,7 @@ prodLog.info(\`🧠 Brainy Pattern Library loaded: \${EMBEDDED_PATTERNS.length} ` // Write the TypeScript file + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedPatterns.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/buildTypeEmbeddings.ts b/scripts/buildTypeEmbeddings.ts index 688d6ac1..61bcf238 100644 --- a/scripts/buildTypeEmbeddings.ts +++ b/scripts/buildTypeEmbeddings.ts @@ -11,7 +11,6 @@ import * as fs from 'fs/promises' import * as path from 'path' import { fileURLToPath } from 'url' import { NounType, VerbType } from '../src/types/graphTypes.js' -import { resolveDeterministicStamp } from './lib/deterministicStamp.js' const __dirname = path.dirname(fileURLToPath(import.meta.url)) @@ -374,24 +373,12 @@ async function buildTypeEmbeddings() { const uint8 = new Uint8Array(buffer) const base64 = Buffer.from(uint8).toString('base64') - // Deterministic stamp: derived from the git commit time of this - // generator's inputs, never from wall-clock time — two builds of the - // same source tree must produce byte-identical output. - const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') - const generatedStamp = resolveDeterministicStamp( - [ - path.join(__dirname, 'buildTypeEmbeddings.ts'), - path.join(__dirname, '..', 'src', 'types', 'graphTypes.ts') - ], - outputPath - ) - // Generate TypeScript file const tsContent = `/** * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: ${generatedStamp} + * Generated: ${new Date().toISOString()} * Noun Types: ${nounTypes.length} * Verb Types: ${verbTypes.length} * @@ -408,7 +395,7 @@ export const TYPE_METADATA = { verbTypes: ${verbTypes.length}, totalTypes: ${totalTypes}, embeddingDimensions: ${embeddingDim}, - generatedAt: "${generatedStamp}", + generatedAt: "${new Date().toISOString()}", sizeBytes: { embeddings: ${buffer.byteLength}, base64: ${base64.length} @@ -507,6 +494,7 @@ prodLog.info(\`🧠 Brainy Type Embeddings loaded: \${TYPE_METADATA.nounTypes} n ` // Write the TypeScript file + const outputPath = path.join(__dirname, '..', 'src', 'neural', 'embeddedTypeEmbeddings.ts') await fs.writeFile(outputPath, tsContent) // Report statistics diff --git a/scripts/lib/deterministicStamp.ts b/scripts/lib/deterministicStamp.ts deleted file mode 100644 index c2a66604..00000000 --- a/scripts/lib/deterministicStamp.ts +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Deterministic generation-stamp resolution for Brainy's build-time code - * generators. - * - * Two builds of the same source tree must produce byte-identical output. - * A wall-clock stamp (`new Date()`) breaks that guarantee, so every - * generator that writes a "Generated:" header or a `generatedAt` field - * into its output must resolve the stamp through this module instead. - * - * Resolution order: - * 1. The newest git commit timestamp among the generator's input files - * (the generator script itself always counts as an input). - * 2. If git metadata is unavailable (for example, building from a - * published npm tarball with no `.git` directory), the stamp already - * recorded in the previously generated output file. - * 3. If neither is available, the fixed epoch string - * `1970-01-01T00:00:00.000Z`. - * - * Every fallback logs a line to stderr — deterministic degradation is - * loud, never a silent divergence. - */ - -import { execFileSync } from 'child_process' -import * as fs from 'fs' - -const EPOCH_STAMP = '1970-01-01T00:00:00.000Z' -const STAMP_PATTERN = /\*\s*Generated:\s*(\S+)/ - -/** - * Resolve the deterministic stamp for a generator run. - * - * @param inputPaths Absolute paths to every file whose content determines - * the generator's output, including the generator script itself. - * @param previousOutputPath Absolute path to the previously generated - * file, used for the existing-stamp fallback when git is unavailable. - * @returns An ISO-8601 timestamp string that is deterministic for a given - * source tree. - */ -export function resolveDeterministicStamp( - inputPaths: string[], - previousOutputPath: string -): string { - const gitStamp = newestGitCommitTimestamp(inputPaths) - if (gitStamp) { - return gitStamp - } - - const existingStamp = readExistingStamp(previousOutputPath) - if (existingStamp) { - process.stderr.write( - `[deterministic-stamp] no git commit history found for generator inputs; ` + - `reusing existing stamp from ${previousOutputPath}: ${existingStamp}\n` - ) - return existingStamp - } - - process.stderr.write( - `[deterministic-stamp] no git commit history and no previous output at ` + - `${previousOutputPath}; falling back to fixed epoch stamp ${EPOCH_STAMP}\n` - ) - return EPOCH_STAMP -} - -/** - * Find the newest git commit timestamp among the given input paths. - * Returns null if git is unavailable, the tree is not a git repository, - * or none of the inputs have any commit history yet. - */ -function newestGitCommitTimestamp(inputPaths: string[]): string | null { - let newest: string | null = null - - for (const inputPath of inputPaths) { - if (!fs.existsSync(inputPath)) { - continue - } - - let out: string - try { - out = execFileSync( - 'git', - ['log', '-1', '--format=%cI', '--', inputPath], - { stdio: ['ignore', 'pipe', 'ignore'] } - ) - .toString() - .trim() - } catch { - // git missing, not a repository, or no permissions — handled by the - // caller's fallback chain. - continue - } - - if (!out) { - // Path exists but has no commit history yet (e.g. newly created, - // uncommitted file). - continue - } - - if (!newest || new Date(out).getTime() > new Date(newest).getTime()) { - newest = out - } - } - - return newest -} - -/** - * Parse the `* Generated: ` header out of a previously - * generated file, if one exists. - */ -function readExistingStamp(outputPath: string): string | null { - if (!fs.existsSync(outputPath)) { - return null - } - - const content = fs.readFileSync(outputPath, 'utf-8') - const match = content.match(STAMP_PATTERN) - return match ? match[1] : null -} diff --git a/src/brainy.ts b/src/brainy.ts index ed958a67..6289e8f2 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -25,7 +25,6 @@ import { } from './storage/brainFormat.js' import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' -import { isZeroNormVector } from './utils/distance.js' import type { HNSWNoun, HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, @@ -396,15 +395,6 @@ interface PlannedTransact { * marker outlives its write. */ markerRecords: FactMarkerRecord[] - /** - * Ids the batch's `{ op: 'update' }` unvector door (`vector: []`) needs to - * decrement on the vectored-noun ledger — consumed by `transact()` with a - * proper `await this.storage.noteVectorUnlanded?.(id)` per id, AFTER - * `commitTransaction` resolves (never for a rejected batch). Kept separate - * from `postCommit` (`Array<() => void>`, called synchronously, fire-and- - * forget) because the ledger hook is async and must be awaited. - */ - vectorUnlands: string[] } /** @@ -1513,15 +1503,6 @@ export class Brainy implements BrainyInterface { }).backfillBlobHistoryRefCountsIfNeeded() } - // LEG C (zero-norm/unvector-door law): migrate a legacy zero-norm VFS - // root BEFORE the vector-leg open gate below ever compares the - // canonical vectored-noun count against the vector index's size — see - // migrateLegacyZeroNormVfsRootIfNeeded's JSDoc for why this is a safe - // O(1) exception to "nothing at open may scale with brain size", and - // why it must run here rather than waiting on VirtualFileSystem's own - // (VFS-instance-gated) lazy migration. - await this.migrateLegacyZeroNormVfsRootIfNeeded() - // Rebuild indexes if needed for existing data. Runs to completion before // init() returns — there is no more first-query lazy path, so the flag // below (kept for getIndexStatus() API compatibility) simply flips true @@ -1536,9 +1517,8 @@ export class Brainy implements BrainyInterface { // cache, roaring, msgpack, sort:topK, distance), HNSW/metadata/graph // index construction, the eager cold-load, id-resolver + connections- // codec wiring, crash-recovery index rebuild, the replay-gap check, - // legacy VFS blob adoption, blob-history backfill, the legacy - // zero-norm VFS root migration, and the rebuildIndexesIfNeeded() gate - // + migration check. + // legacy VFS blob adoption, blob-history backfill, and the + // rebuildIndexesIfNeeded() gate + migration check. markPhase('index-init-gate') // Register shutdown hooks for graceful count flushing (once globally) @@ -1834,11 +1814,7 @@ export class Brainy implements BrainyInterface { if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { throw error } - // Wrap with the original as `cause` so the originating frame (a plugin's - // own file:line, e.g. a provider boot failure) survives to the caller's - // log — a plain string interpolation discards both stack and cause. - const message = error instanceof Error ? error.message : String(error) - throw new Error(`Failed to initialize Brainy: ${message}`, { cause: error }) + throw new Error(`Failed to initialize Brainy: ${error}`) } } @@ -2931,43 +2907,13 @@ export class Brainy implements BrainyInterface { // vector shape, is structurally impossible). The background worker // embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - let vector = deferringEmbed + const vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) - // THE ZERO-NORM LAW (canonical write side): a zero-norm vector is not a - // vector — it never crosses an engine boundary (the engine pair's seam - // law). This engine's own cosine distance treats an all-zero vector - // safely (a zero-norm operand always scores MAXIMUM distance — see - // isZeroNormVector's JSDoc), but a downstream engine serving squared- - // euclidean distance cannot tell it apart from a legitimate origin - // point — a false attractor that silently darkened 150+ rows in a - // production deployment. The index belt (AddToVectorIndexOperation) - // already refuses to INDEX a zero-norm vector, but until now the - // CANONICAL write still persisted it and the vectored-noun ledger - // counted it — so a near-empty store whose only vectored row was - // zero-norm read "canonical vectored > 0, index size 0" and threw a - // not-ready error at open. Normalize HERE, before the dimension pin, - // the vectored-ledger flag (`SaveNounMetadataOperation`'s `hasVector`), - // and the index ops below ever see it, so it persists as the sanctioned - // "unvectored" `[]` shape instead — the canonical write still succeeds. - if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { - prodLog.warn( - `[Brainy] add(): entity ${id} was given an explicit all-zero vector — ` + - `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` - ) - vector = [] - } - // Ensure dimensions are set (a deferred-embed stub carries no dimension // information — the worker's real vector goes through the same guard). - // Gated on `vector.length > 0`, not `!deferringEmbed`: ANY insert whose - // vector is the "unvectored" empty-array shape carries no dimension - // information, deferred or not — an explicit `vector: []` (e.g. the VFS - // root's zero-norm fix, see VirtualFileSystem.doInitializeRoot()) must - // never pin `this.dimensions` to 0, which would poison every subsequent - // real embed's dimension check for the life of the store. - if (!deferringEmbed && vector.length > 0) { + if (!deferringEmbed) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -3083,15 +3029,10 @@ export class Brainy implements BrainyInterface { }, true) ) - // Operation 3: Add to HNSW index (after entity saved). Gated on - // `vector.length > 0`, not `!deferringEmbed`: a deferred embed has - // nothing to index yet (the worker's atomic update inserts the real - // vector later), and an explicit `vector: []` insert (the VFS root's - // zero-norm fix — permanently unvectored plumbing, never embedded) - // is exactly the same "nothing to index yet" shape. The zero-norm - // BELT (a real all-zero vector, non-empty) is enforced inside - // AddToVectorIndexOperation itself — see its JSDoc. - if (vector.length > 0) { + // Operation 3: Add to HNSW index (after entity saved). A deferred + // embed has nothing to index yet — the worker's atomic update + // inserts the real vector. + if (!deferringEmbed) { tx.addOperation( new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration) ) @@ -3666,53 +3607,25 @@ export class Brainy implements BrainyInterface { // often the host writes. const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged - - // THE ZERO-NORM LAW (canonical write side) — see add()'s matching - // comment: an explicit REAL all-zero vector is not a vector. Normalize - // to the sanctioned "unvectored" `[]` shape BEFORE the dimension - // check, the unvector-door decision below, and the index ops ever see - // it — a local copy; `params.vector` itself is never mutated. - let explicitVector = params.vector - if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { - prodLog.warn( - `[Brainy] update(): entity ${params.id} was given an explicit all-zero vector — ` + - `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` - ) - explicitVector = [] - } - - // THE SANCTIONED UNVECTOR DOOR: `explicitVector` at length 0 (an - // explicit `vector: []`, or a real all-zero vector just normalized - // above) is an instruction to remove the vector NOW — never "please - // embed". `validateUpdateParams` already refuses combining it with - // `deferEmbedding: true` (an empty array is truthy, so that guard - // fires unconditionally on any explicit `vector`). Idempotent on an - // already-unvectored row: the ledger decrement near the end of this - // method is gated on the PRIOR vector actually having been real. - const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 - // MT5 deferred re-embedding: the OLD vector keeps serving semantic // search — stale-but-present, never absent (the flicker law) — until // the background worker embeds the new data and swaps it atomically. const deferringEmbed = - params.deferEmbedding === true && hasNewData && !explicitVector - if (explicitVector) { - // A length-0 explicit vector (the unvector door) carries no - // dimension information — exempt from the check, mirroring add()'s - // own `vector.length > 0` gate on the dimension pin. - if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { + params.deferEmbedding === true && hasNewData && !params.vector + if (params.vector) { + if (this.dimensions && params.vector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` ) } - vector = explicitVector + vector = params.vector } else if (hasNewData && !deferringEmbed) { vector = await this.embed(params.data) } // A deferred data change does NOT reindex now (the vector is unchanged; // the worker's atomic swap carries the real reindex later). const needsReindexing = Boolean( - (hasNewData && !deferringEmbed) || params.type || explicitVector + (hasNewData && !deferringEmbed) || params.type || params.vector ) // Always update the noun with new metadata @@ -3806,22 +3719,6 @@ export class Brainy implements BrainyInterface { ? [this.enqueuePendingEmbed(params.id)] : undefined - // Leg D — the unvector door clears a PENDING deferred-embed marker: - // without this, the worker would later embed this row's current data - // and silently re-vector it, defeating the caller's explicit "remove - // the vector now" instruction. The clear rides THIS SAME commit fact - // (an `embed.landed` record with an empty vector — the recovery fold - // disarms a pending marker on ANY `embed.landed` for the id, - // regardless of the vector it carries), so a crash between the write - // and the in-memory clear below still recovers disarmed. Mutually - // exclusive with `embedMarkers` above: `deferringEmbed` requires an - // ABSENT `explicitVector`, so the two branches never both apply. - const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) - const commitRecords: FactMarkerRecord[] | undefined = - embedMarkers ?? (clearsPendingEmbed - ? [{ type: 'embed.landed', id: params.id, vector: [] }] - : undefined) - // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { @@ -3910,33 +3807,7 @@ export class Brainy implements BrainyInterface { } } ] - : undefined, commitRecords) - - // Leg D continued — the in-memory pending-embed clear runs only AFTER - // the commit above actually succeeded (an aborted update must not - // disarm a marker whose durable `embed.landed` twin was never - // written). - if (clearsPendingEmbed) { - this.clearPendingEmbed(params.id) - prodLog.warn( - `[Brainy] update(): entity ${params.id} had a pending deferred embed — ` + - `the unvector door cleared it ('vector: []' is an explicit instruction, ` + - `never "please embed").` - ) - } - - // Leg D — vectored-ledger decrement for the sanctioned unvector door. - // update()'s own metadata write goes through UpdateNounMetadataOperation - // (isNew=false), so the saveNounMetadata(..., hasVector) seam never - // fires here — noteVectorUnlanded is the ONLY seam, the same - // sanctioned hook unvectorNounForRootMigration() uses. Gated on the - // PRIOR vector having actually been real (non-empty, non-zero-norm): - // an already-unvectored row's second call is a true no-op — no - // decrement, matching the ledger-exactness law (never double-count, - // never drift negative). - if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { - await this.storage.noteVectorUnlanded?.(params.id) - } + : undefined, embedMarkers) // Aggregation hook (outside transaction — derived data). `existing` is // the full get() view — every reserved field top-level — and must be @@ -9292,15 +9163,6 @@ export class Brainy implements BrainyInterface { hook() } - // Leg D — vectored-ledger decrements for this batch's unvector-door - // updates (see planTxUpdate's matching comment), applied after the - // commit point and properly awaited (unlike `postCommit`'s synchronous - // fire-and-forget hooks) — each is the same sanctioned hook - // unvectorNounForRootMigration() uses. - for (const id of plan.vectorUnlands) { - await this.storage.noteVectorUnlanded?.(id) - } - // Change feed: the batch's events share its single committed generation. // A rejected batch throws at commitTransaction and never reaches here. this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) @@ -10383,18 +10245,6 @@ export class Brainy implements BrainyInterface { for (const id of nounIds) { const noun = await snapshotStorage.getNoun(id) if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) { - // THE ZERO-NORM LAW: a direct provider-write seam (this materializer - // inserts one-by-one, bypassing AddToVectorIndexOperation's own - // belt) — apply the same refusal here rather than handing a false - // attractor to the ephemeral reader's index. - if (isZeroNormVector(noun.vector)) { - prodLog.warn( - `[Brainy] materializeAtGeneration: refusing to index a zero-norm vector for ` + - `entity ${noun.id} — a zero-norm vector is not a vector and never crosses an ` + - `engine boundary (the materialized record is unaffected)` - ) - continue - } await reader.index.addItem({ id: noun.id, vector: noun.vector }) } } @@ -10544,8 +10394,7 @@ export class Brainy implements BrainyInterface { casUpdates: [], createdNouns: new Set(), changeEvents: [], - markerRecords: [], - vectorUnlands: [] + markerRecords: [] } for (const op of ops) { @@ -10667,26 +10516,10 @@ export class Brainy implements BrainyInterface { // marker-less committed row would be a silently missing vector, which is // the disallowed direction). The background worker embeds + inserts. const deferringEmbed = params.deferEmbedding === true && !params.vector - let vector = deferringEmbed + const vector = deferringEmbed ? [] : params.vector || (await this.embed(params.data)) - - // THE ZERO-NORM LAW — see the single-add() insert path's matching - // comment (a zero-norm vector is not a vector; never crosses an engine - // boundary). Normalized here BEFORE the dimension pin and the - // vectored-ledger `hasVector` flag below ever see it. - if (!deferringEmbed && vector.length > 0 && isZeroNormVector(vector)) { - prodLog.warn( - `[Brainy] transact add: entity ${id} was given an explicit all-zero vector — ` + - `a zero-norm vector is not a vector; persisted unvectored ([]) instead.` - ) - vector = [] - } - - // Gated on `vector.length > 0` — see the single-add() insert path's - // matching comment: an explicit `vector: []` carries no dimension - // information either, deferred or not. - if (!deferringEmbed && vector.length > 0) { + if (!deferringEmbed) { if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { @@ -10768,12 +10601,9 @@ export class Brainy implements BrainyInterface { // for a deferred embed (stub vector `[]`; counted later at landing). new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), - // Gated on `vector.length > 0` — see the single-add() insert path's - // matching comment: an explicit `vector: []` has nothing to index - // either, deferred or not. - ...(vector.length > 0 - ? [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)] - : []), + ...(deferringEmbed + ? [] + : [new AddToVectorIndexOperation(this.index, id, vector, this.indexWriteGeneration)]), new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing, this.indexWriteGeneration) ) plan.touchedNouns.push(id) @@ -10853,62 +10683,17 @@ export class Brainy implements BrainyInterface { const dataUnchanged = rawHasNewData && Brainy.sameEntityData(params.data, existing.data) const hasNewData = rawHasNewData && !dataUnchanged let vector = existing.vector - - // THE ZERO-NORM LAW + THE SANCTIONED UNVECTOR DOOR — transact() mirror - // of update()'s matching block: an explicit REAL all-zero vector - // normalizes to `[]` (never crosses an engine boundary), and an - // explicit `vector: []` (post-normalization) is the sanctioned unvector - // instruction, exempt from the dimension check. `validateUpdateParams` - // already refuses combining it with `deferEmbedding: true`. - let explicitVector = params.vector - if (explicitVector && explicitVector.length > 0 && isZeroNormVector(explicitVector)) { - prodLog.warn( - `[Brainy] transact update: entity ${params.id} was given an explicit all-zero ` + - `vector — a zero-norm vector is not a vector; persisted unvectored ([]) instead.` - ) - explicitVector = [] - } - const isExplicitUnvector = explicitVector !== undefined && explicitVector.length === 0 - - if (explicitVector) { - if (explicitVector.length > 0 && this.dimensions && explicitVector.length !== this.dimensions) { + if (params.vector) { + if (this.dimensions && params.vector.length !== this.dimensions) { throw new Error( - `Vector dimension mismatch: expected ${this.dimensions}, got ${explicitVector.length}` + `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` ) } - vector = explicitVector + vector = params.vector } else if (hasNewData) { vector = await this.embed(params.data) } - const needsReindexing = Boolean(hasNewData || params.type || explicitVector) - - // Leg D — the unvector door clears a PENDING deferred-embed marker (see - // update()'s matching comment for the full rationale): the durable - // clear (an `embed.landed` record, empty vector) rides the batch's ONE - // commit fact via `plan.markerRecords`; the in-memory clear is deferred - // to `plan.postCommit` so an aborted batch never disarms a marker whose - // durable twin was never written. - const clearsPendingEmbed = isExplicitUnvector && this._pendingEmbedIds.has(params.id) - if (clearsPendingEmbed) { - plan.markerRecords.push({ type: 'embed.landed', id: params.id, vector: [] }) - plan.postCommit.push(() => { - this.clearPendingEmbed(params.id) - prodLog.warn( - `[Brainy] transact update: entity ${params.id} had a pending deferred embed — ` + - `the unvector door cleared it ('vector: []' is an explicit instruction, ` + - `never "please embed").` - ) - }) - } - - // Leg D — vectored-ledger decrement for the sanctioned unvector door, - // deferred to `plan.vectorUnlands` (consumed with a proper `await` in - // `transact()`, AFTER the commit succeeds — see its matching comment). - // Gated on the PRIOR vector having actually been real (non-empty, - // non-zero-norm): idempotent on an already-unvectored row. - if (isExplicitUnvector && existing.vector.length > 0 && !isZeroNormVector(existing.vector)) { - plan.vectorUnlands.push(params.id) - } + const needsReindexing = Boolean(hasNewData || params.type || params.vector) const newMetadata = params.merge !== false @@ -12612,49 +12397,21 @@ export class Brainy implements BrainyInterface { const metadataStats = await this.metadataIndex.getStats() const graphSize = await this.graphIndex.size() - // 1. Index size parity. HNSW must hold one node per VECTORED noun — the - // vectored-noun ledger (`getCanonicalCounts().vectors.all`), NOT the raw - // metadata-entry count: every store's VFS root is PERMANENTLY unvectored - // (`vector: []` by design — a zero-norm/empty vector never crosses into - // the index, see AddToVectorIndexOperation/JsHnswVectorIndex.rebuild()'s - // matching belts), and a not-yet-landed deferred embed is unvectored - // too. Comparing against total entries counted the always-unvectored - // root as a permanent 1-node "drift" on every VFS-having store — a false - // warn on an otherwise perfectly healthy handoff. `vectors.all` is - // already the documented coverage denominator for exactly this - // comparison (see `CanonicalCounts.vectors`'s JSDoc). Falls back to the - // metadata-entry count when the ledger is unavailable or suspect (a - // storage adapter without the optional hook, or an unrecounted store) — - // never worse than the prior behavior in that case. - const vectorLedgerForParity = await this.storage.getCanonicalCounts?.() - const vectorParityTarget = - vectorLedgerForParity && !vectorLedgerForParity.suspect - ? vectorLedgerForParity.vectors.all - : metadataStats.totalEntries - if (hnswSize === vectorParityTarget) { + // 1. Index size parity. HNSW must hold at least one node per indexed entity. + if (hnswSize === metadataStats.totalEntries) { checks.push({ name: 'index-parity', status: 'pass', - message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) agree.`, - details: { - hnswSize, - vectoredNouns: vectorParityTarget, - metadataEntries: metadataStats.totalEntries, - graphRelationships: graphSize - } + message: `HNSW (${hnswSize}) and metadata index (${metadataStats.totalEntries}) agree.`, + details: { hnswSize, metadataEntries: metadataStats.totalEntries, graphRelationships: graphSize } }) } else { - const drift = Math.abs(hnswSize - vectorParityTarget) + const drift = Math.abs(hnswSize - metadataStats.totalEntries) checks.push({ name: 'index-parity', - status: drift > Math.max(10, vectorParityTarget * 0.01) ? 'fail' : 'warn', - message: `HNSW (${hnswSize}) and the vectored-noun ledger (${vectorParityTarget}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, - details: { - hnswSize, - vectoredNouns: vectorParityTarget, - metadataEntries: metadataStats.totalEntries, - drift - } + status: drift > Math.max(10, metadataStats.totalEntries * 0.01) ? 'fail' : 'warn', + message: `HNSW (${hnswSize}) and metadata (${metadataStats.totalEntries}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, + details: { hnswSize, metadataEntries: metadataStats.totalEntries, drift } }) } @@ -16275,141 +16032,6 @@ export class Brainy implements BrainyInterface { return !this.pluginRegistry.hasProvider('embeddings') } - /** - * @description LEG C of the zero-norm/unvector-door law — migrate a - * legacy zero-norm VFS root BEFORE the vector-leg open gate - * ({@link rebuildIndexesIfNeeded}'s `vectorCoverageGap` check) ever - * compares the canonical vectored-noun count against the vector index's - * size. A pre-fix store may have persisted the VFS root (the fixed - * all-zeros UUID) with a REAL all-zero placeholder vector — lawful inside - * brainy (`cosineDistance` treats a zero-norm operand as MAXIMUM distance, - * see {@link isZeroNormVector}'s JSDoc) but never indexed (the index belt - * refuses to insert a zero-norm vector) and never meant to cross an - * engine boundary. Left unmigrated, the canonical ledger still counts it - * as vectored while the vector index correctly holds nothing for it — a - * near-empty store whose ONLY vectored row is this zero-norm root reads - * "canonical vectored 1, index size 0" and throws - * `VectorIndexNotReadyError` at open, going DARK instead of serving. - * - * THE LIFECYCLE LAW: nothing at open may scale with brain size. This step - * is safe under that law BECAUSE the VFS root lives at a FIXED, - * well-known id (`00000000-0000-0000-0000-000000000000` — mirrors - * `VirtualFileSystem.VFS_ROOT_ID`; kept as a literal here, the same - * convention as the other reserved-root literals in this file and in - * `db/factLog.ts`/`db/portableGraph.ts` — `brainy.ts` cannot import - * `VirtualFileSystem.ts`, which itself imports `Brainy`) — this is ONE - * direct canonical read by id (`storage.getNoun`, the same O(1) - * fixed-path lookup {@link unvectorNounForRootMigration} itself uses - * internally), NEVER a listing or a walk over `entities/nouns/**`. An - * absent root (a store that has never used the VFS) is a no-op, no error. - * - * Runs UNCONDITIONALLY at every open, independent of whether a - * `VirtualFileSystem` is ever constructed this session — the vector-leg - * gate this fixes runs during Brainy's OWN init, before any - * `VirtualFileSystem` instance exists to run its own lazy migration at - * `doInitializeRoot()` (kept in place as the second line of defense for a - * VFS actually opened this session — belt AND suspenders, never either - * alone). - */ - private async migrateLegacyZeroNormVfsRootIfNeeded(): Promise { - const VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' - // TORN-TOLERANT: a torn root record is a recovery-walk healer's job - // (see tests/integration/recovery-walk-tolerance.test.ts — an init-time - // walk that meets a torn record narrates+counts, via the adapter's own - // loud floor at the read site, and heals PAST it; the open itself must - // still succeed), not this O(1) migration check's. Skip this open's - // migration attempt rather than aborting init(): this leg is a - // defensive EXTRA (the index belt + VirtualFileSystem's own - // doInitializeRoot() migration still stand as the other lines of - // defense), and it retries harmlessly at a later open once the root - // heals. - let root: HNSWNounWithMetadata | null - try { - root = await this.storage.getNoun(VFS_ROOT_ID) - } catch (err) { - if ((err as { code?: string }).code !== 'TORN_RECORD') throw err - prodLog.warn( - `[Brainy] open(): the VFS root's record is TORN — skipping the zero-norm root ` + - `migration check this open (the recovery walk is the healer; this migration ` + - `retries harmlessly once the root heals).` - ) - return - } - if (!root || !Array.isArray(root.vector) || root.vector.length === 0) return - if (!isZeroNormVector(root.vector)) return - const migrated = await this.unvectorNounForRootMigration(VFS_ROOT_ID) - if (migrated) { - prodLog.warn( - `[Brainy] open(): migrated the VFS root's legacy all-zero placeholder vector to ` + - `the unvectored shape (zero-norm vectors never cross an engine boundary) — run ` + - `before the vector-leg open gate compares canonical-vectored-count against the ` + - `vector index, so a near-empty store never reads a false coverage gap.` - ) - } - } - - /** - * SANCTIONED, ONE-TIME MIGRATION HOOK — rewrite a canonical noun's - * persisted vector from a real (non-empty) vector to the "unvectored" - * empty-array shape: the vector record is rewritten to `[]`, the row is - * removed from the vector index (if present), and the vectored-noun - * ledger (`getCanonicalCounts().vectors.all`) is decremented through the - * sanctioned {@link StorageAdapter.noteVectorUnlanded} hook — so the - * coverage ledger never silently drifts. - * - * Exists SOLELY for the VFS root zero-norm migration, called from two - * sites that detect the same legacy shape (a persisted root whose vector - * is the legacy all-zero placeholder): {@link migrateLegacyZeroNormVfsRootIfNeeded} - * (this brain's own init sequence, BEFORE the vector-leg open gate — Leg - * C of the zero-norm/unvector-door law) and - * `VirtualFileSystem.doInitializeRoot()` (the second line of defense, for - * a VFS actually constructed this session). This is NOT the general- - * purpose unvector API — ordinary application data uses the sanctioned - * unvector DOOR instead (`update({ id, vector: [] })` / the same op inside - * `transact()`), which decrements the ledger and clears any pending - * deferred-embed marker inline; it does not call this method. Never call - * this outside a VFS root migration. - * - * Idempotent: a noun already unvectored (`vector.length === 0`) or absent - * is a no-op — safe to call on every `init()`. - * - * @param id - The canonical noun id to migrate. - * @returns `true` if a migration write happened, `false` if the noun was - * already unvectored (or absent) — a no-op. - */ - async unvectorNounForRootMigration(id: string): Promise { - const noun = await this.storage.getNoun(id) - if (!noun || !Array.isArray(noun.vector) || noun.vector.length === 0) return false - - await this.persistSingleOp({ nouns: [id] }, async (tx) => { - // Rewrite the vector leg to the unvectored shape. Placeholder adjacency - // (mirrors update()'s own SaveNounOperation staging) — the op preserves - // stored graph state when `connections.size === 0`. - tx.addOperation( - new SaveNounOperation(this.storage, { - id, - vector: [], - connections: new Map(), - level: 0 - }) - ) - // Remove from the vector index — safe even if the row was never - // actually indexed (RemoveFromVectorIndexOperation's removeItem is a - // no-op when the id is absent). - tx.addOperation( - new RemoveFromVectorIndexOperation(this.index, id, noun.vector, this.indexWriteGeneration) - ) - }) - - // Vectored-noun ledger: this migration carries a vector write with no - // accompanying metadata operation (metadata is untouched), so the - // saveNounMetadata(..., hasVector) seam never fires for it — mirrors the - // deferred-embed LANDING path's use of the narrow storage hook, in - // reverse. - await this.storage.noteVectorUnlanded?.(id) - return true - } - /** * Setup embedder */ diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 4b018e94..8112afd2 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -872,27 +872,6 @@ export interface StorageAdapter { */ noteVectorLanded?(id: string): Promise - /** - * OPTIONAL narrow ledger hook, the mirror of {@link noteVectorLanded}: - * record that a canonical noun's vector was just REMOVED — rewritten from - * a real (non-empty) vector to the "unvectored" empty-array shape. Exists - * for the ONE sanctioned reverse migration this engine supports: the VFS - * root's zero-norm fix (see `VirtualFileSystem.doInitializeRoot()` and - * `Brainy.unvectorNounForRootMigration()`), which rewrites a pre-fix - * store's all-zero placeholder root vector to `[]` and must decrement - * `vectors.all` through this hook so the coverage ledger never drifts. - * NOT a general-purpose "I removed a vector" callback — ordinary - * application data has no sanctioned path from vectored back to - * unvectored (`update()` refuses an empty vector as a dimension - * mismatch by design). Callers MUST call this only when the noun held a - * REAL vector immediately before this write (the caller already holds - * that fact for free, from its own pre-write read — never an added read). - * A backend without vectored-noun tracking is a no-op via this method's - * absence (feature-detected). - * @param id - The noun whose vector was just removed. - */ - noteVectorUnlanded?(id: string): Promise - /** * Get noun with metadata combined * @returns Combined HNSWNounWithMetadata or null diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 81bed338..bfb68959 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -654,43 +654,10 @@ export class GenerationStore { let replayed = 0 const replayFact = async (fact: CommitFact): Promise => { for (const op of fact.ops) { - let image: { metadata: unknown | null; vector: unknown | null } - if (op.record === null) { - // A genuine tombstone (both legs absent) — the fold removes - // both legs, exactly like `writeNounRaw`/`writeVerbRaw`'s raw - // exact-restore contract. - image = { metadata: null, vector: null } - } else if ( - op.record.metadata !== null && - (op.record.vector === null || op.record.vector === undefined) - ) { - // PRESERVE-IF-ABSENT (population law, ADR-008 G1 — the fold's - // half): a metadata-only after-image must never DELETE an - // existing vector leg through the fold. `writeNounRaw`/ - // `writeVerbRaw` are exact-restore primitives — a `vector: - // null` there means "delete", which is exactly right for - // `rollBackUncommittedGeneration`'s before-image restore (a - // transaction abort legitimately un-writes a vector the failed - // transaction added). It is NOT right here: this fold replays - // AFTER-IMAGES, and re-applying an already-intact record must - // be byte-safe (this module's own invariant, see the log-authority - // comment above) — silently erasing a landed vector because one - // replayed fact's vector leg came back null is the exact defect - // that left metadata-counted, never-enumerated rows in a - // production store (confirmed root cause: the enumeration walk - // used to key on the vector leg, so a preserved-but-then-deleted - // vector made the row invisible while the ledger still counted - // it by metadata). A genuine "unvector" has its own sanctioned, - // ledger-correct path (`Brainy.unvectorNounForRootMigration`) — - // never this raw primitive, and never the fold. - const current = - op.kind === 'verb' - ? await this.storage.readVerbRaw(op.id) - : await this.storage.readNounRaw(op.id) - image = { metadata: op.record.metadata, vector: current.vector ?? null } - } else { - image = { metadata: op.record.metadata, vector: op.record.vector } - } + const image = + op.record === null + ? { metadata: null, vector: null } + : { metadata: op.record.metadata, vector: op.record.vector } if (op.kind === 'verb') await this.storage.writeVerbRaw(op.id, image) else await this.storage.writeNounRaw(op.id, image) this.noteCheckpointDirty(op.kind, op.id) diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 8b9badc1..77e4f84d 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -10,7 +10,7 @@ import { Vector, VectorDocument } from '../coreTypes.js' -import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js' +import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js' import type { BaseStorage } from '../storage/baseStorage.js' import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' @@ -64,34 +64,6 @@ export class HnswFlushError extends Error { } } -/** - * @description Thrown by {@link JsHnswVectorIndex.addItem} / {@link - * JsHnswVectorIndex.updateItem} when handed a length-0 vector. A length-0 - * vector is the sanctioned "unvectored" shape for a canonical noun record - * (class-J: a VFS-system row, a deferred embed not yet landed, or any other - * legitimately-vector-less row) — but it is NEVER a legal INDEX insert. The - * index itself has no concept of "unvectored"; deciding that a row is - * unvectored and therefore skippable is the FILL/REBUILD/LOAD consumer's job - * (see {@link JsHnswVectorIndex.rebuild}), done BEFORE ever calling addItem. - * A length-0 vector reaching this point is a caller bug: silently accepting - * it would pin `this.dimension = 0` on an empty index (poisoning every real - * insert thereafter with a dimension mismatch) or store a vector-less node - * that a distance calculation can never safely compare against. Loud errors, - * never quiet losses — this throws instead of either. - */ -export class EmptyVectorIndexError extends Error { - constructor(public readonly id: string, operation: 'addItem' | 'updateItem') { - super( - `${operation}(${id}): refusing to index a length-0 vector — a length-0 vector is the ` + - `sanctioned "unvectored" shape for a canonical row, but it is never a legal index ` + - `insert. Callers that fill/rebuild/load the index must skip vector.length === 0 rows ` + - `themselves (unvectored = nothing to index, not an error at that layer); reaching ` + - `here with one is a caller bug.` - ) - this.name = 'EmptyVectorIndexError' - } -} - /** * Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls * on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native @@ -608,15 +580,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { throw new Error('Vector is undefined or null') } - // THE INDEX REFUSES A LENGTH-0 VECTOR (see EmptyVectorIndexError's JSDoc): - // an empty vector is the sanctioned "unvectored" shape at the canonical - // layer, never a legal index member. Refusing here — loudly, before the - // dimension pin below — means no future fill/rebuild/load path can ever - // poison `this.dimension` to 0 or park a vector-less node in the graph. - if (vector.length === 0) { - throw new EmptyVectorIndexError(id, 'addItem') - } - // Set dimension on first insert if (this.dimension === null) { this.dimension = vector.length @@ -991,13 +954,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider { return } - // Same refusal as addItem (see EmptyVectorIndexError's JSDoc) — an - // in-place relink must never rewrite an already-indexed node down to the - // unvectored shape or poison the pinned dimension. - if (vector.length === 0) { - throw new EmptyVectorIndexError(id, 'updateItem') - } - if (this.dimension === null) { this.dimension = vector.length } else if (vector.length !== this.dimension) { @@ -1599,15 +1555,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider { } const loaded = await this.storage.getNounVector(noun.id) - // `loaded` is a length-0 array (not null/undefined) for a canonical row - // that is legitimately unvectored — `![]` is FALSE (an empty array is - // truthy), so the bare `!loaded` check below would silently accept it - // as "found" and hand a dimension-0 vector to a distance calculation. - // A node only reaches this lazy-load path because it is a MEMBER of - // the live index (rebuild() now refuses to admit unvectored rows — see - // its JSDoc), so an empty vector here is never legitimate: treat it - // exactly like "not found", loudly. - if (!loaded || loaded.length === 0) { + if (!loaded) { throw new Error(`Vector not found for noun ${noun.id}`) } @@ -1817,56 +1765,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider { totalCount = result.totalCount || result.items.length - // UNVECTORED ROWS ARE NOT AN INDEX MEMBER (the class-J law): a canonical - // noun whose vector leg is `[]` (a VFS-root-style system row, a - // deferred embed not yet landed, or a best-effort fallback for an - // unreadable vector leg) is a normal, enumerable, countable row — it - // is simply not indexed. `storage.getVectorIndexData()` derives its - // {level, connections} answer straight from the noun's OWN record, so - // it returns non-null for every existing noun regardless of whether - // that noun ever actually reached `addItem()` — it cannot be used to - // decide indexability. `nounData.vector.length === 0` is the one - // truthful signal (mirrors the `noun.vector.length > 0` guards in - // {@link getVectorSafe} / {@link getVectorSync}): skip here, counted - // once in a summary line, never per-row spam. - let skippedUnvectored = 0 - // Process all nouns at once for (const nounData of result.items) { try { - if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) { - skippedUnvectored++ - continue - } - // THE ZERO-NORM LAW — bulk-rebuild leg: a persisted zero-norm - // vector (a pre-10.4.2 row the canonical write has not yet - // normalized) must never enter the index either, mirroring the - // belt AddToVectorIndexOperation enforces on the live write path. - // Only the canonical vector is authoritative here — persisted - // HNSW graph metadata (level/connections) can outlive an unvector. - if (isZeroNormVector(nounData.vector)) { - prodLog.warn( - `[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` + - `zero-norm (a zero-norm vector is not a vector and never crosses an ` + - `engine boundary)` - ) - continue - } - - // Restore the pinned dimension from the first real vector this - // rebuild loads. `addItem`/`updateItem` only pin `this.dimension` - // on a LIVE insert — a fresh rebuild from storage never goes - // through either, so without this the pin stays `null` across a - // restart. A `null` pin means the very next insert (correct OR - // wrong length) silently BECOMES the new pin instead of being - // checked against the store's real dimension — the wrong-length - // case then fails much later and less clearly, inside a distance - // calculation against an already-loaded node, instead of here, - // immediately, with a named expected-vs-got mismatch. - if (this.dimension === null) { - this.dimension = nounData.vector.length - } - // Load HNSW graph data for this entity const hnswData = await this.storage.getVectorIndexData(nounData.id) @@ -1914,10 +1815,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider { options.onProgress(loadedCount, totalCount) } - prodLog.info( - `HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})` + - (skippedUnvectored > 0 ? ` — ${skippedUnvectored.toLocaleString()} unvectored row(s) skipped` : '') - ) + prodLog.info(`HNSW: Loaded ${loadedCount.toLocaleString()} nodes (${storageType})`) } // Step 5: CRITICAL - Recover entry point if missing) diff --git a/src/neural/embeddedPatterns.ts b/src/neural/embeddedPatterns.ts index 4f4339f4..c15447e7 100644 --- a/src/neural/embeddedPatterns.ts +++ b/src/neural/embeddedPatterns.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED PATTERNS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2025-09-29T10:10:00-07:00 + * Generated: 2026-07-02T21:43:26.976Z * Patterns: 220 * Coverage: 94-98% of all queries * diff --git a/src/neural/embeddedTypeEmbeddings.ts b/src/neural/embeddedTypeEmbeddings.ts index 5b10116c..b5f3546b 100644 --- a/src/neural/embeddedTypeEmbeddings.ts +++ b/src/neural/embeddedTypeEmbeddings.ts @@ -2,7 +2,7 @@ * 🧠 BRAINY EMBEDDED TYPE EMBEDDINGS * * AUTO-GENERATED - DO NOT EDIT - * Generated: 2026-06-29T10:04:19-07:00 + * Generated: 2026-02-09T16:59:48.867Z * Noun Types: 42 * Verb Types: 127 * @@ -19,7 +19,7 @@ export const TYPE_METADATA = { verbTypes: 127, totalTypes: 169, embeddingDimensions: 384, - generatedAt: "2026-06-29T10:04:19-07:00", + generatedAt: "2026-02-09T16:59:48.867Z", sizeBytes: { embeddings: 259584, base64: 346112 diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index cabe2e30..22bf1366 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1066,18 +1066,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { protected allCountsSuspect = false /** One narration per session for the suspect transition (never per delete). */ private allCountsSuspectNarrated = false - /** - * Which rule produced the ALL scalars currently in memory. `'identity-record'` - * means one counted entity per metadata content leg — the honest rule: a - * bare id-directory (a ghost or scar left by a partial-delete defect, no - * content leg) counts zero. Set by the one-time derivation and by the - * sanctioned recount, alongside `allCountsSuspect = false`; left `undefined` - * when a loaded counts.json carries the ALL scalars but no stamp — the - * legacy container-rule derivation, which forces `allCountsSuspect = true` - * at load instead. A filesystem concern: `MemoryStorage` has no counts.json - * and never sets this. - */ - protected allCountsDerivedBy?: 'identity-record' protected entityCounts: Map = new Map() // type -> count protected verbCounts: Map = new Map() // verb type -> count protected countCache: Map = new Map() @@ -1164,24 +1152,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { }) } - /** - * OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorUnlanded}): - * the mirror of {@link noteVectorLanded} — record a noun's vector was just - * REMOVED (rewritten to the unvectored `[]` shape). Never below zero: a - * caller that (incorrectly) fires this for a noun already unvectored would - * otherwise drive the ledger negative — clamped defensively, matching the - * delete path's `if (this.totalVectoredNounCount > 0)` guard. - * @param id - The noun whose vector was just removed (retained for a - * future narration seam; the count itself needs no id-keyed state). - */ - async noteVectorUnlanded(id: string): Promise { - void id - if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount-- - this.scheduleCountPersist().catch(() => { - // Ignore persist errors — the in-memory count is authoritative; a later op retries. - }) - } - /** * Increment count for entity type - O(1) operation. * Concurrency is handled by the process-global mutex diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 4f2a43b0..fd9dbb4c 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -18,8 +18,6 @@ import { } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' -import { prodLog } from '../../utils/logger.js' -import { isZeroNormVector } from '../../utils/distance.js' import { TornRecordError, isUnparseablePayloadError, @@ -604,20 +602,6 @@ export class FileSystemStorage extends BaseStorage { * automatically. Returns the pruned container ids so the caller can recompute * counts. */ - /** - * @description Whether an id directory's file legs include the metadata - * CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single - * test that decides whether an `entities////` container is - * a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete - * defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune - * and {@link scanCanonicalEntities} so the two agree by construction — one - * counted entity per identity record, never per bare container. - * @param legs - File names in one `entities////` directory. - */ - private hasMetadataContentLeg(legs: string[]): boolean { - return legs.some((f) => f.startsWith('metadata.json')) - } - public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { await this.ensureInitialized() const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } @@ -657,7 +641,7 @@ export class FileSystemStorage extends BaseStorage { } // A live entity has its metadata content leg. No content leg → a // vector-only ghost or an empty scar → prune the whole container. - if (this.hasMetadataContentLeg(legs)) continue + if (legs.some((f) => f.startsWith('metadata.json'))) continue await fs.promises.rm(idAbs, { recursive: true, force: true }) pruned[kind].push(entry.name) console.warn( @@ -2606,36 +2590,13 @@ export class FileSystemStorage extends BaseStorage { ) { this.totalNounCountAll = counts.totalNounCountAll this.totalVerbCountAll = counts.totalVerbCountAll - if (counts.allCountsDerivedBy === 'identity-record') { - // Derived (or recounted) under the honest rule — one counted - // entity per metadata content leg. Trust the persisted suspect - // flag as-is; an unprovable delete since may still have set it. - this.allCountsDerivedBy = 'identity-record' - this.allCountsSuspect = counts.allCountsSuspect === true - } else { - // The ALL scalars exist but predate the identity-record stamp — - // they were derived under the legacy rule that counted one - // entity per id DIRECTORY, so orphaned ghost/scar containers (a - // pre-8.3.1 partial-delete defect — see pruneOrphanedEntities()) - // were counted as entities too. O(1) field read, NEVER a walk - // here: force suspect and name it loudly. A sanctioned recount - // (repairIndex) restores exact denominators and clears this. - this.allCountsDerivedBy = undefined - this.allCountsSuspect = true - needsPersist = true - prodLog.warn( - '[FileSystemStorage] canonical count ledger was derived under the legacy ' + - 'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' + - 'exact denominators' - ) - } + this.allCountsSuspect = counts.allCountsSuspect === true } else { const nouns = await this.scanCanonicalEntities('nouns') const verbs = await this.scanCanonicalEntities('verbs') this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false - this.allCountsDerivedBy = 'identity-record' console.warn( `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` + `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + @@ -2706,7 +2667,6 @@ export class FileSystemStorage extends BaseStorage { this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false - this.allCountsDerivedBy = 'identity-record' // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT // (a deferred-embed noun's file exists but holds an empty vector until // its embed lands), so this is a full O(nouns) content scan — see @@ -2744,19 +2704,10 @@ export class FileSystemStorage extends BaseStorage { /** * Walk the canonical `entities//<2-hex-shard>//` tree, counting - * one entity per id directory that holds the metadata CONTENT leg - * (`metadata.json` or its `.json.gz` variant — see - * {@link hasMetadataContentLeg}). A bare container — a ghost (a stale - * `vectors.json` left with no metadata leg) or a scar (an empty directory), - * both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the - * identity record IS the population (ADR-008 G1), never the directory. - * This is the ONE-TIME legacy derivation walk (see callers); a prior - * version of this scan counted every id directory regardless of content, - * over-counting any store carrying orphaned containers — see - * `allCountsDerivedBy` for how a counts.json derived under that old rule is - * marked suspect on load. Returns up to 100 sampled *counted* entity - * directories (absolute paths) — nouns feed the type-distribution estimate - * above. An absent tree (fresh store) counts zero. + * one entity per id directory (the layout `getNounVectorPath`/`getNouns` + * use). Returns up to 100 sampled entity directories (absolute paths) — + * nouns feed the type-distribution estimate above. An absent tree (fresh + * store) counts zero. */ private async scanCanonicalEntities( kind: 'nouns' | 'verbs' @@ -2773,21 +2724,9 @@ export class FileSystemStorage extends BaseStorage { const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) for (const entry of ids) { if (!entry.isDirectory()) continue - const idAbs = path.join(shardPath, entry.name) - let legs: string[] - try { - legs = await fs.promises.readdir(idAbs) - } catch (error: any) { - if (error?.code === 'ENOENT') continue - throw error - } - // No metadata content leg → a ghost or scar container → not an - // entity. Same test pruneOrphanedEntities() uses, so the two agree - // by construction. - if (!this.hasMetadataContentLeg(legs)) continue count++ if (sampleDirs.length < SAMPLE_MAX) { - sampleDirs.push(idAbs) + sampleDirs.push(path.join(shardPath, entry.name)) } } } @@ -2842,20 +2781,14 @@ export class FileSystemStorage extends BaseStorage { } /** - * Count canonical nouns holding a REAL (non-empty, non-zero-norm) vector — - * the vectored-noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, - * presence cannot be decided from the id-directory listing alone: a - * deferred-embed noun's `vectors.json` EXISTS (written at `add()` time - * with `vector: []`) until its embed LANDS, so this walk reads every - * noun's `vectors.json` CONTENT — O(nouns) reads, not O(ids) listing. - * ZERO-NORM LAW: a real all-zero vector is not a vector — it never counts - * here either (Brainy's write paths normalize an explicit zero-norm - * vector to `[]` at write time, but a store created before that fix may - * still carry legacy all-zero rows on disk; this derivation must agree - * with the live ledger's definition of "vectored" regardless of when the - * row was written). Used ONLY for a one-time legacy-counts.json derivation - * or a lost/corrupted counts.json recovery; the result is persisted so - * this scan never repeats. + * Count canonical nouns holding a REAL (non-empty) vector — the vectored- + * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence + * cannot be decided from the id-directory listing alone: a deferred-embed + * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`) + * until its embed LANDS, so this walk reads every noun's `vectors.json` + * CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time + * legacy-counts.json derivation or a lost/corrupted counts.json recovery; + * the result is persisted so this scan never repeats. */ private async scanVectoredNounCount(): Promise { const base = path.join(this.rootDir, 'entities', 'nouns') @@ -2869,12 +2802,7 @@ export class FileSystemStorage extends BaseStorage { for (const entry of ids) { if (!entry.isDirectory()) continue const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) - if ( - record && - Array.isArray(record.vector) && - record.vector.length > 0 && - !isZeroNormVector(record.vector) - ) { + if (record && Array.isArray(record.vector) && record.vector.length > 0) { vectored++ } } @@ -2906,13 +2834,6 @@ export class FileSystemStorage extends BaseStorage { // scanVectoredNounCount()'s JSDoc). totalVectoredNounCount: this.totalVectoredNounCount, allCountsSuspect: this.allCountsSuspect, - // Derivation-rule stamp for the ALL scalars above — 'identity-record' - // when they were counted one-per-metadata-content-leg (the honest - // rule); omitted (JSON.stringify drops `undefined`) when the current - // in-memory scalars came from a legacy container-rule counts.json - // that hasn't been through a sanctioned recount yet, so a future load - // keeps naming them suspect rather than trusting an unproven value. - allCountsDerivedBy: this.allCountsDerivedBy, lastUpdated: new Date().toISOString() } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 5510f93b..d6ccc5fa 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -203,40 +203,6 @@ function idFromVectorPath(path: string): string { return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix } -/** - * @description Extract the entity id embedded in a metadata path - * (`entities/{nouns|verbs}/{shard}/{id}/metadata.json`) — the IDENTITY-RECORD - * mirror of {@link idFromVectorPath}. The cursored noun/verb walks key their - * population on this file (ADR-008 G1: the metadata record IS the population; - * the vector leg is optional), so walk ordering and cursor resume derive the - * id from THIS path, never the vector path — a row with metadata and no - * vector file must still be listed, ordered, and resumable. - * @param path - A metadata path (full or prefix-relative; must end with `/metadata.json`). - * @returns The entity id (the path segment immediately before `/metadata.json`). - */ -function idFromMetadataPath(path: string): string { - const withoutSuffix = path.replace(/\/metadata\.json$/, '') - const lastSlash = withoutSuffix.lastIndexOf('/') - return lastSlash >= 0 ? withoutSuffix.slice(lastSlash + 1) : withoutSuffix -} - -/** - * @description The sanctioned UNVECTORED shape for a noun hydrated during - * enumeration when its identity record (metadata.json) exists but its vector - * leg (vectors.json) does not — a fold-born metadata-only after-image, or any - * row genuinely without a vector yet. Mirrors the shape - * `unvectorNounForRootMigration` (src/brainy.ts) writes for the sanctioned - * unvector path (`{ vector: [], connections: new Map(), level: 0 }`), so a - * walk-yielded unvectored row is byte-shape-identical to one produced by that - * migration. Callers already handle `vector: []` as first-class - * (validateAddParams exempts it; index gates key on `length > 0`). - * @param id - The noun id. - * @returns A structurally-valid, vector-empty `HNSWNoun`. - */ -function unvectoredNoun(id: string): HNSWNoun { - return { id, vector: [], connections: new Map>(), level: 0 } -} - /** * Get ID-first path for verb metadata * No type parameter needed - direct O(1) lookup by ID @@ -1487,18 +1453,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { * rollups are derived state with their own rebuild paths * (`rebuildTypeCounts()` / `rebuildSubtypeCounts()`). * - * EXACT-RESTORE PRIMITIVE — `vector: null` DELETES the vector leg, on - * purpose: `GenerationStore.rollBackUncommittedGeneration()` depends on - * this to legitimately un-write a vector a failed transaction added. This - * is deliberately NOT "preserve if absent" — a caller replaying an - * AFTER-IMAGE (the recovery fold, `GenerationStore`'s `replayFact`) must - * apply preserve-if-absent itself BEFORE calling this, by reading the - * current vector and carrying it forward when the after-image's own - * vector leg is null/undefined but its metadata is not (see `replayFact` - * for the implementation and full rationale). A caller that genuinely - * wants to unvector a row uses the sanctioned, ledger-correct path - * (`Brainy.unvectorNounForRootMigration`) — never this primitive. - * * @param id - The entity id. * @param record - Raw stored objects as returned by {@link BaseStorage.readNounRaw}. */ @@ -1535,9 +1489,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Restore a relationship's raw stored objects byte-for-byte (verb-side - * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats, - * same EXACT-RESTORE contract — `vector: null` deletes, on purpose; the - * fold's preserve-if-absent logic lives at its call site, not here). + * mirror of {@link BaseStorage.writeNounRaw}; same bookkeeping caveats). * * @param id - The relationship id. * @param record - Raw stored objects as returned by {@link BaseStorage.readVerbRaw}. @@ -2231,18 +2183,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Stable within-shard order (by noun id) so offset windows and cursor resume // are deterministic; ids come from the path so skipped nouns are never read. - // - // IDENTITY-KEYED WALK (population law, ADR-008 G1): the metadata record - // (not the vector) IS the population — a noun with metadata and no vector - // file (a fold-born after-image, see writeNounRaw's preserve-if-absent - // contract) must still enumerate. Keying on metadata.json here means the - // ledger recount (rebuildTypeCounts' `allNouns`, also metadata.json-keyed) - // and this walk agree on population by construction. Ordering is - // unaffected for a healthy store: every vectored noun has both legs, so - // the id set and sort order are identical to the old vectors.json keying. const entries = nounFiles - .filter((p) => p.includes('/metadata.json')) - .map((p) => ({ path: p, id: idFromMetadataPath(p) })) + .filter((p) => p.includes('/vectors.json')) + .map((p) => ({ path: p, id: idFromVectorPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor @@ -2267,24 +2210,13 @@ export abstract class BaseStorage extends BaseStorageAdapter { ) { const batch = toHydrate.slice(i, i + BaseStorage.HYDRATE_CONCURRENCY) const hydrated = await Promise.all( - batch.map(async ({ path: metadataPath, id }) => { + batch.map(async ({ path: nounPath }) => { try { - const metadata = await this.readCanonicalObject(metadataPath) + const noun = await this.readCanonicalObject(nounPath) + if (!noun) return null + const deserialized = this.deserializeNoun(noun) + const metadata = await this.getNounMetadata(deserialized.id) if (!metadata) return null - // The vector leg is OPTIONAL (population law): a metadata-only - // row hydrates with the sanctioned unvectored shape rather than - // being dropped from the walk. A fault reading the vector leg - // is treated the same as absence — best-effort, matching the - // canonical recount's tolerance for an unreadable vectors.json - // (rebuildTypeCounts) — a vector-leg problem never hides an - // otherwise-good identity record. - let deserialized: HNSWNoun - try { - const vectorRecord = await this.readCanonicalObject(getNounVectorPath(id)) - deserialized = vectorRecord ? this.deserializeNoun(vectorRecord) : unvectoredNoun(id) - } catch { - deserialized = unvectoredNoun(id) - } return { deserialized, metadata } } catch (error) { // A TORN record must surface typed — a paginated read that @@ -2294,9 +2226,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // walk's job is to HEAL PAST it — skip the victim, serve the rest. // Identity point-reads (get-by-id) still throw typed upstream. if (isTornRecordError(error)) { /* skip torn victim; loud floor already fired */ } - // Skip nouns whose IDENTITY record fails to load (the metadata - // read above) — that is the one leg this walk cannot proceed - // without. + // Skip nouns that fail to load return null } }) @@ -2417,14 +2347,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { const shardDir = `entities/nouns/${shardHex}` try { const nounFiles = await this.listCanonicalObjects(shardDir) - // IDENTITY-KEYED WALK (population law, ADR-008 G1) — see the matching - // comment in getNounsWithPagination: metadata.json is the population; - // the vector leg is optional, so a metadata-only row must still be - // listed (and here, for the unfiltered case, needs ZERO reads either - // way — the id comes straight from the path). const entries = nounFiles - .filter((p) => p.includes('/metadata.json')) - .map((p) => idFromMetadataPath(p)) + .filter((p) => p.includes('/vectors.json')) + .map((p) => idFromVectorPath(p)) .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)) const toWalk = cursor && shard === cursor.shard ? entries.filter((id) => id > cursor.id) : entries @@ -2635,79 +2560,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Stable within-shard order (by verb id) so offset windows and cursor resume // are deterministic and consistent across calls. Ids come from the path, so // verbs skipped by the cursor are never read. - // - // IDENTITY-KEYED WALK (population law, ADR-008 G1) — the noun mirror of - // this comment in getNounsWithPagination applies here too: metadata.json - // is the population; keying on it here means this walk and the ledger - // recount (rebuildTypeCounts' `allVerbs`, already metadata.json-keyed) - // agree on population by construction. Unchanged for a healthy store — - // `relate()` always writes both legs of a verb in the same commit, so - // the id set and order match the old vectors.json keying exactly; this - // only additionally surfaces a fold-born metadata-only row (see - // writeVerbRaw's preserve-if-absent contract). const entries = verbFiles - .filter((p) => p.includes('/metadata.json')) - .map((p) => ({ path: p, id: idFromMetadataPath(p) })) + .filter((p) => p.includes('/vectors.json')) + .map((p) => ({ path: p, id: idFromVectorPath(p) })) .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)) - for (const { path: metadataPath, id: verbId } of entries) { + for (const { path: verbPath, id: verbId } of entries) { if (collected.length >= peekCount) break // Resume: in the cursor's own shard, skip up to AND INCLUDING the cursor id // (later shards are processed in full). No read for skipped verbs. if (cursor && shard === cursor.shard && verbId <= cursor.id) continue try { - // Identity leg first — required. A verb this walk cannot read - // metadata for cannot be hydrated at all (same as before). - const metadata = await this.readCanonicalObject(metadataPath) - if (!metadata) continue + const rawVerb = await this.readCanonicalObject(verbPath) + if (!rawVerb) continue - // The vector leg is the verb's STRUCTURAL core (verb/sourceId/ - // targetId live there — see coreTypes.ts HNSWVerb), unlike a - // noun's vector, which is pure embedding data. `relate()` always - // writes both legs atomically and verbs have no deferred-embed - // path, so a healthy store's verbs always have both. A vector-leg - // absence here can only be a fold-born after-image (see - // writeVerbRaw's preserve-if-absent contract) — and unlike a - // noun, this walk cannot safely FABRICATE sourceId/targetId to - // synthesize a structurally-valid verb (an empty-string endpoint - // would silently create a phantom edge — worse than omission). - // If the metadata record happens to carry its own sourceId/ - // targetId (never true for current production writes, but not - // disallowed — e.g. a future schema or a repair tool could - // populate them), reconstruct from those; otherwise this row is - // loudly skipped — counted by the ledger, but not returned as an - // item, until a repair can supply the missing endpoints. - const rawVerb = await this.readCanonicalObject(getVerbVectorPath(verbId)) - let verb: HNSWVerb - if (rawVerb) { - verb = this.deserializeVerb(rawVerb) - } else { - const metaSourceId = (metadata as Record).sourceId - const metaTargetId = (metadata as Record).targetId - const metaVerbType = (metadata as Record).verb - if ( - typeof metaSourceId === 'string' && metaSourceId.length > 0 && - typeof metaTargetId === 'string' && metaTargetId.length > 0 && - typeof metaVerbType === 'string' && metaVerbType.length > 0 - ) { - verb = { - id: verbId, - vector: [], - connections: new Map>(), - verb: metaVerbType as VerbType, - sourceId: metaSourceId, - targetId: metaTargetId - } - } else { - prodLog.error( - `[BaseStorage] getVerbsWithPagination: verb ${verbId} has a metadata ` + - `record but no vector leg and no recoverable sourceId/targetId — ` + - `skipping (counted by the ledger, not yielded; needs repair).` - ) - continue - } - } + // Deserialize connections Map from JSON storage format + const verb = this.deserializeVerb(rawVerb) // Apply type filter if (filterVerbTypes && !filterVerbTypes.has(verb.verb)) { @@ -2724,6 +2593,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { continue } + // Load metadata + const metadata = await this.getVerbMetadata(verb.id) + // Apply subtype filter (requires metadata — checked AFTER load) if (filterSubtypes) { const subtype = metadata?.subtype as string | undefined @@ -4820,10 +4692,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.totalVerbCountAll = allVerbs this.totalVectoredNounCount = allVectoredNouns this.allCountsSuspect = false - // This walk counts one entity per metadata.json record (never per bare - // container) — the identity-record rule. Stamp it so a future load - // trusts these scalars instead of naming them suspect at open. - this.allCountsDerivedBy = 'identity-record' this.countCache.clear() await this.persistCounts() diff --git a/src/transaction/operations/IndexOperations.ts b/src/transaction/operations/IndexOperations.ts index 1bbbca88..139c67fe 100644 --- a/src/transaction/operations/IndexOperations.ts +++ b/src/transaction/operations/IndexOperations.ts @@ -13,8 +13,6 @@ import type { VectorIndexProvider, GraphIndexProvider } from '../../plugin.js' import type { MetadataIndexManager } from '../../utils/metadataIndex.js' import type { GraphVerb } from '../../coreTypes.js' import type { Operation, RollbackAction } from '../types.js' -import { isZeroNormVector } from '../../utils/distance.js' -import { prodLog } from '../../utils/logger.js' /** * Backend identity stamped into an operation's emitted `name` string (e.g. @@ -90,30 +88,6 @@ export class AddToVectorIndexOperation implements Operation { } async execute(): Promise { - // THE ZERO-NORM LAW (the live provider-write seam's belt): a zero-norm - // vector is not a vector — it never crosses an engine boundary. This - // engine's own cosine distance treats an all-zero vector safely (a - // zero-norm operand always scores MAXIMUM distance, see - // {@link isZeroNormVector}'s JSDoc), but a downstream engine serving - // squared-euclidean distance cannot tell it apart from a legitimate - // origin point — a false attractor that silently darkens real results. - // The canonical write already landed (SaveNoun/SaveNounMetadata - // operations are staged ahead of this one in every caller) — only the - // INDEX INSERT is refused here, loudly, never a throw. A length-0 - // vector is the unrelated "unvectored" shape and is skipped silently - // (the same contract callers already rely on for deferred embeds). - if (this.vector.length === 0) { - return async () => {} - } - if (isZeroNormVector(this.vector)) { - prodLog.warn( - `[vector-index] refusing to index a zero-norm vector for entity ${this.id} — ` + - `a zero-norm vector is not a vector and never crosses an engine boundary ` + - `(the canonical write is unaffected; only the vector-index insert is skipped)` - ) - return async () => {} - } - // Check if item already exists (for rollback decision) const existed = await this.itemExists(this.id) @@ -289,52 +263,14 @@ export class ReplaceInVectorIndexOperation implements Operation { // One commit generation for the whole replace (both branches + rollback). const generation = this.generationFn?.() - // THE ZERO-NORM LAW (see AddToVectorIndexOperation's matching JSDoc): a - // real all-zero replacement vector must never land in the index — refuse - // loudly, canonical write unaffected. The row must not be left stale - // either: if it was genuinely indexed under `oldVector`, remove it - // rather than pretend the old vector still describes the row. A - // length-0 `newVector` (the unrelated "unvectored" shape) is handled the - // same way, silently — no caller today reaches this with an empty - // replacement (update() rejects a dimension-mismatched empty vector), - // but the seam stays consistent in case one ever legitimately does. - if (isZeroNormVector(this.newVector) || this.newVector.length === 0) { - const wasIndexed = this.oldVector.length > 0 && !isZeroNormVector(this.oldVector) - if (isZeroNormVector(this.newVector)) { - prodLog.warn( - `[vector-index] refusing to replace with a zero-norm vector for entity ${this.id} — ` + - `a zero-norm vector is not a vector and never crosses an engine boundary ` + - `(the canonical write is unaffected; the row is removed from the vector index instead)` - ) - } - if (wasIndexed) { - await this.index.removeItem(this.id, generation) - } - return async () => { - // Restore the declared before-state. - if (wasIndexed) { - await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) - } - } - } - if (typeof index.updateItem === 'function') { // Atomic path: one in-place call, the row never leaves the index. await index.updateItem({ id: this.id, vector: this.newVector }, generation) return async () => { // Restore the declared before-state in place (see class JSDoc for - // the item-did-not-exist posture). A length-0 oldVector means the row - // was never actually indexed before this op ran (a length-0 vector is - // never a legal index member — see EmptyVectorIndexError) — there is - // no in-place "restore to empty" for the provider to perform, so - // rollback removes the row instead, leaving the same "not indexed" - // state the row was in before execute(). - if (this.oldVector.length > 0) { - await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) - } else { - await this.index.removeItem(this.id, generation) - } + // the item-did-not-exist posture). + await index.updateItem!({ id: this.id, vector: this.oldVector }, generation) } } @@ -345,14 +281,9 @@ export class ReplaceInVectorIndexOperation implements Operation { return async () => { // updateItem-style restore via the same adjacent pair, back to the - // declared before-state. Same length-0 carve-out as the updateItem - // path above: an empty oldVector was never a legal index member, so - // rollback just leaves the row removed rather than attempting an - // illegal empty re-add. + // declared before-state. await this.index.removeItem(this.id, generation) - if (this.oldVector.length > 0) { - await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) - } + await this.index.addItem({ id: this.id, vector: this.oldVector }, generation) } } } diff --git a/src/utils/distance.ts b/src/utils/distance.ts index d61bc12e..36e9e8e5 100644 --- a/src/utils/distance.ts +++ b/src/utils/distance.ts @@ -65,29 +65,6 @@ export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number = return 1 - similarity } -/** - * True when `vector` is a REAL (non-empty) all-zero vector — the "false - * attractor" shape this engine's own cosine distance treats safely (a - * zero-norm operand always scores the MAXIMUM distance, see - * {@link cosineDistance}) but a downstream engine serving squared-euclidean - * distance cannot distinguish from a legitimate origin point. THE LAW: a - * zero-norm vector is not a vector — it never crosses an engine boundary - * (never handed to a vector-index provider as a searchable item). - * - * A length-0 vector is the UNRELATED "unvectored, not yet embedded" shape - * (the deferred-embed stub, a permanently-vectorless system row) and is - * deliberately NOT zero-norm here — callers checking for "nothing to index" - * should test `vector.length === 0` separately; this only flags the - * dangerous non-empty all-zero case. - */ -export function isZeroNormVector(vector: readonly number[]): boolean { - if (vector.length === 0) return false - for (let i = 0; i < vector.length; i++) { - if (vector[i] !== 0) return false - } - return true -} - /** * Calculates the Manhattan (L1) distance between two vectors. * Lower values indicate higher similarity. diff --git a/src/utils/paramValidation.ts b/src/utils/paramValidation.ts index 00790a4a..d43559dc 100644 --- a/src/utils/paramValidation.ts +++ b/src/utils/paramValidation.ts @@ -588,14 +588,8 @@ export function validateAddParams(params: AddParams): void { ) } - // Validate vector dimensions if provided. A length-0 vector is the - // "unvectored" shape — an explicit `vector: []` (e.g. the VFS root's - // permanently-vectorless creation, see - // VirtualFileSystem.doInitializeRoot()'s zero-norm fix) carries no - // dimension information, exactly like an absent vector or a deferred - // embed's internal stub, so it is exempt from the dimension check rather - // than refused as a "0-dimensional vector". - if (params.vector && params.vector.length > 0) { + // Validate vector dimensions if provided + if (params.vector) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) @@ -613,17 +607,6 @@ export function validateUpdateParams(params: UpdateParams): void { // null/undefined means "no new data was given". const hasData = params.data !== undefined && params.data !== null if ((params as UpdateParams & { deferEmbedding?: boolean }).deferEmbedding === true) { - if (params.vector && params.vector.length === 0) { - // The nonsensical combination Leg D of the zero-norm/unvector-door law - // refuses: `vector: []` is the SANCTIONED UNVECTOR DOOR — an explicit - // instruction to remove the vector NOW, never "please embed" — so it - // cannot be paired with a request to defer an embed. - throw new Error( - `update(): 'vector: []' (the unvector door) cannot be combined with ` + - `'deferEmbedding: true' — an unvector is an explicit instruction to remove ` + - `the vector now, not a request to defer an embed. Drop one of the two.` - ) - } if (params.vector) { throw new Error( `update(): deferEmbedding cannot be combined with an explicit 'vector' — ` + @@ -660,16 +643,8 @@ export function validateUpdateParams(params: UpdateParams): void { throw new Error(`invalid NounType: ${params.type}`) } - // Validate vector dimensions if provided. A length-0 vector is the - // SANCTIONED UNVECTOR DOOR (see brainy.ts update()'s matching comment): an - // explicit `vector: []` — or a real all-zero vector, normalized to `[]` - // upstream by the zero-norm law — carries no dimension information, - // exactly like validateAddParams's identical exemption, so it is exempt - // from the dimension check rather than refused as a "0-dimensional - // vector". (The `deferEmbedding` combination above already refuses - // `vector: []` paired with `deferEmbedding: true` — an empty array is - // truthy, so that guard fires unconditionally on any explicit `vector`.) - if (params.vector && params.vector.length > 0) { + // Validate vector dimensions if provided + if (params.vector) { const config = ValidationConfig.getInstance() if (params.vector.length !== config.maxVectorDimensions) { throw new Error(`vector must have exactly ${config.maxVectorDimensions} dimensions`) diff --git a/src/vfs/VirtualFileSystem.ts b/src/vfs/VirtualFileSystem.ts index 59a16be4..90018863 100644 --- a/src/vfs/VirtualFileSystem.ts +++ b/src/vfs/VirtualFileSystem.ts @@ -11,7 +11,6 @@ import { v4 as uuidv4 } from '../universal/uuid.js' import { Brainy } from '../brainy.js' import { Entity, AddParams, RelateParams, FindParams, Relation } from '../types/brainy.types.js' import { NounType, VerbType } from '../types/graphTypes.js' -import { isZeroNormVector } from '../utils/distance.js' import { PathResolver } from './PathResolver.js' import { mimeDetector } from './MimeTypeDetector.js' import { @@ -90,6 +89,16 @@ export class VirtualFileSystem implements IVirtualFileSystem { // Uses deterministic UUID format for storage compatibility private static readonly VFS_ROOT_ID = '00000000-0000-0000-0000-000000000000' + // OPEN-PATH FIX: the dimension of the placeholder vector given to the VFS + // root when it is first created (see `doInitializeRoot`). Mirrors the + // built-in WASM embedding engine's fixed, hardcoded output size + // (all-MiniLM-L6-v2 — see src/embeddings/candle-wasm/src/lib.rs + // HIDDEN_SIZE and src/embeddings/EmbeddingManager.ts). Deliberately NOT + // derived from `brain.dimensions` — this constant only applies to the + // default-embedder branch, where the true output dimension is this fixed + // value by construction, never a moving target. + private static readonly VFS_ROOT_VECTOR_DIMENSIONS = 384 + /** * Construct a VFS bound to a Brainy instance. * @@ -232,11 +241,9 @@ export class VirtualFileSystem implements IVirtualFileSystem { private async doInitializeRoot(): Promise { const rootId = VirtualFileSystem.VFS_ROOT_ID - // Try to get existing root by fixed ID (O(1) lookup, not query). - // includeVectors: true — the zero-norm migration below (leg 2) needs to - // inspect the persisted vector to detect the legacy placeholder shape. + // Try to get existing root by fixed ID (O(1) lookup, not query) try { - const existingRoot = await this.brain.get(rootId, { includeVectors: true }) + const existingRoot = await this.brain.get(rootId) if (existingRoot) { // Root exists - verify metadata is correct @@ -253,34 +260,6 @@ export class VirtualFileSystem implements IVirtualFileSystem { }) } - // ZERO-NORM ROOT MIGRATION (one-time): a pre-fix store persisted the - // root with a REAL all-zero placeholder vector — lawful inside - // brainy (cosineDistance treats a zero-norm operand as MAXIMUM - // distance, see src/utils/distance.ts) but a "false attractor" for a - // downstream engine serving squared-euclidean distance, which cannot - // tell an all-zero vector apart from a legitimate origin point (a - // production incident silently darkened 150+ rows in a partner - // engine's index this way). THE LAW: a zero-norm vector is not a - // vector — it never crosses an engine boundary. Detect the legacy - // shape via NORM, not length or dimension (any real all-zero vector - // qualifies, not just the historical 384-dim one), and rewrite it to - // the "unvectored" `[]` shape through the sanctioned migration path - // (Brainy.unvectorNounForRootMigration — see its JSDoc), which keeps - // `getCanonicalCounts().vectors.all` honest and removes the row from - // the vector index. Idempotent: a store already on the new shape - // (vector.length === 0) takes the false branch below on every - // subsequent init() — a permanent no-op, not a one-time flag. - const existingVector = existingRoot.vector ?? [] - if (existingVector.length > 0 && isZeroNormVector(existingVector)) { - const migrated = await this.brain.unvectorNounForRootMigration(rootId) - if (migrated) { - console.log( - 'VFS: migrated root vector from the legacy all-zero placeholder to the ' + - 'unvectored shape (zero-norm vectors never cross an engine boundary)' - ) - } - } - return rootId } } catch (error) { @@ -298,42 +277,31 @@ export class VirtualFileSystem implements IVirtualFileSystem { // meant every writer's FIRST-EVER open forced the process-global WASM // engine to cold-compile its model (measured 90-140s on throttled // CPUs) before the brain could even finish init(). This branch only - // runs once per store (a previously-opened store already has a root — - // see the migration above for the pre-fix shape — reopening never - // re-adds it), so the fix applies only to brand-new stores. + // runs once per store (the root already exists — with a real vector — + // in every previously-opened production store; reopening never re-adds + // it), so the fix applies only to brand-new stores. // - // ZERO-NORM LAW (current shape, superseding the historical all-zero - // placeholder): the root's vector is `[]` — the SAME "unvectored" - // empty-array shape used for a deferred embed's stub and any other - // not-yet-embedded row — never a real all-zero vector. A zero-norm - // vector is lawful inside brainy (`cosineDistance`, see - // src/utils/distance.ts, returns the MAXIMUM distance whenever either - // operand's norm is zero) but is a "false attractor" for a downstream - // engine serving squared-euclidean distance, which cannot tell a real - // all-zero vector apart from a legitimate origin point — it silently - // darkened 150+ rows in a partner engine's index in production. THE - // LAW: a zero-norm vector is not a vector — it never crosses an engine - // boundary. `vector: []` achieves the SAME cold-compile avoidance the - // original placeholder did (`add()`'s dimension-pin and HNSW-insert - // gates both key off `vector.length > 0`, so an empty vector never - // calls embed(), never pins `brain.dimensions`, and never reaches the - // vector index — see brainy.ts add()'s matching comments) while never - // persisting a searchable zero vector for a downstream engine to trip - // over. Deliberately NOT `deferEmbedding: true`: that flag's landing - // path (`kickEmbedWorker()`, called synchronously right after commit — - // see brainy.ts add()/update()) would still force the WASM engine to - // cold-compile within milliseconds of open (just off the awaited path - // instead of never paying it at all) AND would eventually embed the - // root's data for real, which this fix forbids — the root must NEVER - // be embedded, not merely "not yet". + // Chosen fix: an explicit all-zero vector, not `deferEmbedding: true`. + // `deferEmbedding` looked attractive (ack now, embed later) but its + // landing path (`kickEmbedWorker()`, called synchronously right after + // commit — see brainy.ts add()/update()) would still force the WASM + // engine to cold-compile within milliseconds of open, just off the + // awaited path instead of never paying it at all — worse than the + // explicit-vector path, which never touches the engine for this row. + // An all-zero vector is safe: `cosineDistance` (src/utils/distance.ts) + // explicitly returns the MAXIMUM distance whenever either operand's + // norm is zero, so the root can never rank ahead of real content in a + // similarity search, and HNSW indexes it like any other vector. // - // Only the default WASM engine gets this treatment — a plugin- - // registered native 'embeddings' provider has no cold-compile cost and - // may use a different dimension, so it keeps embedding the root for - // real (same as before this fix) rather than leave Brainy's own - // plumbing permanently unvectored on a store where embedding is cheap. + // Only the default WASM engine gets this treatment — its output + // dimension (384) is fixed and hardcoded, so the placeholder can never + // mis-pin `brain.dimensions` for it. A plugin-registered native + // 'embeddings' provider has no cold-compile cost AND may use a + // different dimension, so it keeps embedding the root for real (same + // as before this fix) rather than risk pinning the wrong dimension + // ahead of the caller's own first real embed. const rootVector = this.brain.usesDefaultWasmEmbedder() - ? ([] as number[]) + ? new Array(VirtualFileSystem.VFS_ROOT_VECTOR_DIMENSIONS).fill(0) : undefined await this.brain.add({ diff --git a/tests/integration/canonical-count-ledger.test.ts b/tests/integration/canonical-count-ledger.test.ts index 3d292e97..f3c8ce20 100644 --- a/tests/integration/canonical-count-ledger.test.ts +++ b/tests/integration/canonical-count-ledger.test.ts @@ -225,12 +225,9 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s } /** Baseline vectored count right after a fresh open() — init() creates a - * hidden system VFS-root noun, but (the zero-norm root cure) it is - * deliberately UNVECTORED (`vector: []`, never a real all-zero - * placeholder — a zero-norm vector never crosses an engine boundary), so - * a brand-new store's `vectors.all` is 0. Tests still assert DELTAS off - * this baseline rather than hardcoding it away, in case that ever - * changes again. */ + * hidden system VFS-root noun that itself carries a real vector, so a + * brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off + * this baseline rather than hardcoding it away. */ let baseline: number beforeEach(async () => { @@ -238,7 +235,6 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-')) brain = await open() baseline = (await brain.storage.getCanonicalCounts()).vectors.all - expect(baseline).toBe(0) // the unvectored VFS root contributes nothing }) afterEach(async () => { vi.restoreAllMocks() @@ -352,7 +348,7 @@ describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s brain = await open() const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 1) // just the one non-deferred noun — the root is unvectored + expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) expect(persisted.totalVectoredNounCount).toBe(baseline + 1) diff --git a/tests/integration/enumeration-population-law.test.ts b/tests/integration/enumeration-population-law.test.ts deleted file mode 100644 index eaab7432..00000000 --- a/tests/integration/enumeration-population-law.test.ts +++ /dev/null @@ -1,333 +0,0 @@ -/** - * @module tests/integration/enumeration-population-law - * @description THE POPULATION LAW (ADR-008 G1): the unfiltered noun/verb walk - * and the canonical ALL scalar must agree on the population — a row's - * IDENTITY RECORD (metadata.json) is what defines membership; the vector leg - * is optional data, never a gate on visibility. Before this fix, the walk - * (getNounsWithPagination / getNounIdsWithPagination / getVerbsWithPagination) - * enumerated by keying on the VECTOR leg (`vectors.json`), so a row with - * metadata and no vector file was counted by the ledger (already - * metadata.json-keyed — see `rebuildTypeCounts`) but never yielded by the - * walk: a permanent "counted but invisible" phantom for any downstream - * consumer (a health-coverage row, an index-fill walk) that iterates the walk - * to account for the ledger's total. - * - * Two legs are pinned here: - * (a)/(b) LEG 1 — the walk re-keys on metadata.json. A fold-born - * metadata-only row (the exact shape `GenerationStore.replayFact` can - * leave behind, and the exact shape `writeNounRaw`/`writeVerbRaw` accept) - * must be YIELDED, hydrated with the sanctioned unvectored shape - * (`vector: []`) — not merely counted. - * - * For VERBS this closes only PARTIALLY: `sourceId`/`targetId` are - * HNSWVerb's structural core and live ONLY in the vector leg (never in - * metadata — see `RESERVED_RELATION_FIELDS` in reservedFields.ts, which - * does not include them). A metadata-only verb row therefore cannot be - * safely reconstructed without FABRICATING an edge's endpoints — which - * would silently create a phantom relationship, strictly worse than the - * original defect. The walk recovers the row when its metadata happens - * to carry `sourceId`/`targetId` (a defensive, forward-compatible - * fallback — never true for a CURRENT production write, but not - * disallowed either); otherwise it counts the row (ledger, unchanged) - * but loudly skips yielding it, logging the gap instead of hiding it. - * Closing this fully requires persisting `sourceId`/`targetId` in verb - * metadata — a schema change out of this task's scope; see the session - * report for the explicit call-out. - * - * (c)/(d) LEG 2 — the recovery fold's preserve-if-absent contract, exercised - * directly against `GenerationStore`/`FactLog` (below the `Brainy` API): - * a metadata-only after-image replayed over an already-vectored row must - * PRESERVE the existing vector leg (never delete it); a genuine tombstone - * (both legs absent) still removes both legs. - */ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { randomUUID } from 'node:crypto' -import { Brainy } from '../../src/index.js' -import { GenerationStore } from '../../src/db/generationStore.js' -import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' -import { LOG_AUTHORITY_PATH } from '../../src/db/logAuthority.js' -import type { CommitFact } from '../../src/db/factLog.js' - -describe('enumeration population law — LEG 1 (identity-keyed walk)', () => { - let dir: string - let brain: any - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-population-law-')) - brain = await open() - }) - afterEach(async () => { - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('(a) nouns.all equals the unfiltered walk-yield count with a fold-born metadata-only row present', async () => { - // Ordinary, fully-vectored background population. - await brain.add({ data: 'one', type: 'document' }) - await brain.add({ data: 'two', type: 'document' }) - await brain.flush() - - // THE EXACT PRE-FIX SHAPE: a metadata-only row against a FRESH id — no - // vector ever existed for it. Written through the raw primitive directly, - // exactly as `GenerationStore.replayFact` (the recovery fold) applies a - // replayed after-image whose vector leg came back null. - const freshId = randomUUID() - await brain.storage.writeNounRaw(freshId, { - metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 }, - vector: null - }) - - // writeNounRaw bypasses count bookkeeping on purpose (its own JSDoc) — the - // sanctioned recount brings the ledger scalar to ground truth. This walk - // was ALREADY metadata.json-keyed before this fix (rebuildTypeCounts), so - // the recount's answer does not depend on today's change. - await brain.repairIndex() - - const ledger = await brain.storage.getCanonicalCounts() - const walk = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } }) - - expect(walk.items.length).toBe(ledger.nouns.all) - expect(walk.totalCount).toBe(ledger.nouns.all) - - const yielded = walk.items.find((n: any) => n.id === freshId) - expect(yielded, 'the metadata-only row must be YIELDED, not merely counted').toBeDefined() - expect(yielded.vector).toEqual([]) - }) - - it('(a-ids) getNounIdsWithPagination (the zero-read unfiltered enumerator) also yields the metadata-only row', async () => { - await brain.add({ data: 'one', type: 'document' }) - await brain.flush() - - const freshId = randomUUID() - await brain.storage.writeNounRaw(freshId, { - metadata: { noun: 'document', createdAt: Date.now(), updatedAt: Date.now(), _rev: 1 }, - vector: null - }) - await brain.repairIndex() - - const ledger = await brain.storage.getCanonicalCounts() - const page = await brain.storage.getNounIdsWithPagination({ limit: 1000, offset: 0 }) - expect(page.ids.length).toBe(ledger.nouns.all) - expect(page.ids).toContain(freshId) - }) - - it('(b) verbs.all counts a fold-born metadata-only row; the walk yields it when endpoints are recoverable from metadata, and loudly skips (never fabricates) when they are not', async () => { - const a = await brain.add({ data: 'a', type: 'document' }) - const b = await brain.add({ data: 'b', type: 'document' }) - await brain.relate({ from: a, to: b, type: 'relatedTo' }) - await brain.flush() - - // Case 1 — the REALISTIC production shape: metadata carries the verb - // type (a reserved field, kept for backward compat) but never - // sourceId/targetId — those are HNSWVerb's structural core and live - // ONLY in the vector leg. The walk cannot safely fabricate them (an - // empty-string endpoint would silently create a phantom edge), so this - // row is counted by the ledger but not yielded — a documented, - // loudly-logged gap, not a silent one. - const gapId = randomUUID() - await brain.storage.writeVerbRaw(gapId, { - metadata: { verb: 'relatedTo', createdAt: Date.now(), updatedAt: Date.now(), weight: 1 }, - vector: null - }) - - // Case 2 — endpoints ARE recoverable from metadata (never true for a - // current production write; modeled here as what a repair tool or a - // future schema could supply): the walk reconstructs and yields it. - const recoveredId = randomUUID() - await brain.storage.writeVerbRaw(recoveredId, { - metadata: { - verb: 'relatedTo', - sourceId: a, - targetId: b, - createdAt: Date.now(), - updatedAt: Date.now(), - weight: 1 - }, - vector: null - }) - - await brain.repairIndex() - const ledger = await brain.storage.getCanonicalCounts() - const walk = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } }) - - // The ledger counts every identity record — the real edge plus both - // synthetic metadata-only rows — unaffected by whether the walk can - // safely hydrate them. - expect(ledger.verbs.all).toBe(3) - - const recovered = walk.items.find((v: any) => v.id === recoveredId) - expect(recovered, 'endpoints recoverable from metadata must be yielded').toBeDefined() - expect(recovered.sourceId).toBe(a) - expect(recovered.targetId).toBe(b) - expect(recovered.vector).toEqual([]) - - // The documented gap: counted, not yielded — this is the one corner of - // the population law this task does NOT close (see the session report). - const gapped = walk.items.find((v: any) => v.id === gapId) - expect(gapped).toBeUndefined() - expect(walk.items.length).toBeLessThan(ledger.verbs.all) - }) -}) - -describe('enumeration population law — LEG 2 (fold preserve-if-absent, below the Brainy API)', () => { - /** A GenerationStore whose brain has already flipped to log authority — the - * precondition for `replayFact` (the recovery fold) to run at open(). */ - async function openLogAuthorityStore(): Promise<{ storage: MemoryStorage; store: GenerationStore }> { - const storage = new MemoryStorage() - await storage.init() - await storage.writeRawObject(LOG_AUTHORITY_PATH, { authority: 'log' }) - const store = new GenerationStore(storage) - await store.open() - return { storage, store } - } - - it('(c) nouns: a metadata-only after-image replayed over a vectored row PRESERVES the vector; it stays readable and the vectored ledger is untouched either way', async () => { - const { storage, store } = await openLogAuthorityStore() - const id = randomUUID() - const vectorRecord = { id, vector: [0.1, 0.2, 0.3], connections: {}, level: 0 } - - // Generation 1 — a real, honest commit: both legs land together. - await store.commitTransaction({ - touched: { nouns: [id], verbs: [] }, - execute: async () => { - await storage.writeNounRaw(id, { - metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 }, - vector: vectorRecord - }) - } - }) - const beforeVectoredCount = (await storage.getCanonicalCounts()).vectors.all - - // THE ANOMALOUS FACT, crafted directly (bypassing commitTransaction, - // whose honest read-after-write could never produce this on its own): - // metadata changed, vector leg null, while the row is STILL vectored on - // disk. This is exactly the shape the recovery fold must tolerate — - // modeling the confirmed production defect at the replay boundary. - const factLog = store.getFactLog()! - const anomalousFact: CommitFact = { - generation: 2, - timestamp: Date.now(), - ops: [ - { - kind: 'noun', - id, - record: { - metadata: { noun: 'document', createdAt: 1000, updatedAt: 2000, _rev: 2 }, - vector: null - } - } - ] - } - await factLog.append(anomalousFact) - await factLog.sync() - - // Reopen — a fresh GenerationStore over the SAME storage. Generation 2's - // fact sits above the (still generation-1) manifest, so it replays - // through the recovery fold — `replayFact`'s own call site. - const store2 = new GenerationStore(storage) - await store2.open() - - const after = await storage.readNounRaw(id) - expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull() - expect((after.vector as { vector: number[] }).vector).toEqual([0.1, 0.2, 0.3]) - expect((after.metadata as { updatedAt: number }).updatedAt).toBe(2000) // the new metadata DID apply - - // writeNounRaw bypasses ledger bookkeeping either way (by design — see - // its JSDoc), so this scalar is unaffected by the replay regardless of - // outcome; asserted for completeness against the task's exact wording. - const afterVectoredCount = (await storage.getCanonicalCounts()).vectors.all - expect(afterVectoredCount).toBe(beforeVectoredCount) - }) - - it('(c-verb) verbs: a metadata-only after-image replayed over a vectored edge PRESERVES the vector leg (sourceId/targetId/verb intact)', async () => { - const { storage, store } = await openLogAuthorityStore() - const id = randomUUID() - const sourceId = randomUUID() - const targetId = randomUUID() - const vectorRecord = { id, vector: [0.7, 0.8], connections: {}, verb: 'relatedTo', sourceId, targetId } - - await store.commitTransaction({ - touched: { nouns: [], verbs: [id] }, - execute: async () => { - await storage.writeVerbRaw(id, { - metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 1000, weight: 1 }, - vector: vectorRecord - }) - } - }) - - const factLog = store.getFactLog()! - const anomalousFact: CommitFact = { - generation: 2, - timestamp: Date.now(), - ops: [ - { - kind: 'verb', - id, - record: { - metadata: { verb: 'relatedTo', createdAt: 1000, updatedAt: 2000, weight: 2 }, - vector: null - } - } - ] - } - await factLog.append(anomalousFact) - await factLog.sync() - - const store2 = new GenerationStore(storage) - await store2.open() - - const after = await storage.readVerbRaw(id) - expect(after.vector, 'the vector leg must survive the metadata-only replay').not.toBeNull() - expect((after.vector as { sourceId: string }).sourceId).toBe(sourceId) - expect((after.vector as { targetId: string }).targetId).toBe(targetId) - expect((after.metadata as { weight: number }).weight).toBe(2) - }) - - it('(d) a genuine tombstone replay removes BOTH legs (never preserved)', async () => { - const { storage, store } = await openLogAuthorityStore() - const id = randomUUID() - const vectorRecord = { id, vector: [0.4, 0.5, 0.6], connections: {}, level: 0 } - - await store.commitTransaction({ - touched: { nouns: [id], verbs: [] }, - execute: async () => { - await storage.writeNounRaw(id, { - metadata: { noun: 'document', createdAt: 1000, updatedAt: 1000, _rev: 1 }, - vector: vectorRecord - }) - } - }) - expect((await storage.readNounRaw(id)).vector).not.toBeNull() // sanity: it landed - - const factLog = store.getFactLog()! - await factLog.append({ - generation: 2, - timestamp: Date.now(), - ops: [{ kind: 'noun', id, record: null }] // a genuine tombstone — both legs absent - }) - await factLog.sync() - - const store2 = new GenerationStore(storage) - await store2.open() - - const after = await storage.readNounRaw(id) - expect(after.metadata, 'a genuine delete removes the metadata leg').toBeNull() - expect(after.vector, 'a genuine delete removes the vector leg too — preserve-if-absent never applies to a tombstone').toBeNull() - }) -}) diff --git a/tests/integration/index-skips-unvectored.test.ts b/tests/integration/index-skips-unvectored.test.ts deleted file mode 100644 index c65aaa01..00000000 --- a/tests/integration/index-skips-unvectored.test.ts +++ /dev/null @@ -1,253 +0,0 @@ -/** - * @module tests/integration/index-skips-unvectored - * @description THE UNVECTORED-ROW CURE — two integration tests - * (`tests/lifecycle/biography.test.ts`'s Ch4/5/6 chapter and - * `tests/integration/clear-persistence.test.ts`'s multi-cycle test) started - * failing after a canonical-storage change made a vector-less row (the - * class-J shape: `vector: []`, e.g. the VFS root, a deferred embed not yet - * landed, or any other legitimately-unvectored canonical record) VISIBLE to - * the enumeration walk `getNounsWithPagination()` for the first time — before - * that change such rows were simply invisible to the walk. `hnswIndex.ts`'s - * `rebuild()` never guarded against that shape: it inserted every row the - * walk yielded into the live in-memory index, including ones with a length-0 - * vector, because `storage.getVectorIndexData()` derives its {level, - * connections} answer straight from the noun's OWN record — it returns - * non-null for ANY existing noun, whether or not that noun was ever actually - * indexed via `addItem()`. A vector-less node admitted into the graph could - * become the entry point (or occupy any graph position), and the very next - * real-vectored `addItem()` then ran a distance calculation against it — - * `cosineDistance` throws "Vectors must have the same dimensions" the moment - * one operand is a length-0 array. - * - * THE FIX, at two layers (`src/hnsw/hnswIndex.ts`): - * (1) FILL/REBUILD/LOAD consumers treat `vector.length === 0` as "unvectored — - * nothing to index" and skip the row (normal, not an error; one summary - * count line, never per-row spam) — `rebuild()`'s loop now checks this - * BEFORE ever creating a graph node, so an unvectored row can never - * become an index member, entry point, or dimension-setter. - * (2) THE INDEX ITSELF refuses a length-0 vector in `addItem()` / - * `updateItem()` with a typed `EmptyVectorIndexError`, loudly, instead of - * ever pinning `dimension = 0` or storing a vector-less node — so no - * future fill/rebuild/load path can silently poison the index even if it - * forgets law (1). - * - * Four legs pinned here: - * (a) `rebuild()` over a store mixing real-vectored rows and `vector: []` - * rows indexes ONLY the vectored ones — size === vectored count, - * dimension pinned to the real (non-zero) length. - * (b) `clear()` then real adds afterward never trip a dimension mismatch — - * the exact `clear-persistence.test.ts` regression shape, reproduced - * directly against the index/storage seam this module owns. - * (c) `index.addItem({ id, vector: [] })` throws `EmptyVectorIndexError` - * (and `updateItem` does too, for an existing node). - * (d) crash -> repair: the crashed generation's entities survive, the ledger - * recounts honestly, and a fresh real-vectored add afterward never trips - * a dimension mismatch against a leftover vector-less phantom. - */ -import { describe, it, expect, afterEach } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' -import { EmptyVectorIndexError } from '../../src/hnsw/hnswIndex.js' -import { abandonAsCrashed, openBrain as openKillMatrixBrain, uid, vec } from '../helpers/durabilityKillMatrix.js' - -const tmpDirs: string[] = [] -function mkTmp(): string { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-')) - tmpDirs.push(d) - return d -} -afterEach(() => { - for (const d of tmpDirs.splice(0)) { - try { - fs.rmSync(d, { recursive: true, force: true }) - } catch { - /* best-effort cleanup */ - } - } -}) - -/** A filesystem-backed brain with explicit vectors (no embedder needed) and - * manual persistence — mirrors `durabilityKillMatrix.ts`'s `openBrain` so - * every write in this module is explicit and provably durable. */ -function openBrain(dir: string): any { - return new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - persistence: { policy: 'manual' } - }) -} - -describe('HNSW index skips unvectored rows', () => { - it('(a) rebuild() indexes only vectored rows: size === vectored count, dimension pinned to the real length', async () => { - const dir = mkTmp() - let brain = openBrain(dir) - await brain.init() - - // 5 real-vectored rows. - const vectoredIds: string[] = [] - for (let i = 0; i < 5; i++) { - const id = uid(`vectored-${i}`) - await brain.add({ id, data: `real entity ${i}`, type: NounType.Document, vector: vec(i) }) - vectoredIds.push(id) - } - // 3 explicit unvectored rows — the class-J "vector: []" shape, a normal, - // enumerable, countable canonical row that must never reach the index. - const unvectoredIds: string[] = [] - for (let i = 0; i < 3; i++) { - const id = uid(`unvectored-${i}`) - await brain.add({ id, data: `unvectored entity ${i}`, type: NounType.Document, vector: [] }) - unvectoredIds.push(id) - } - await brain.flush() - - // The canonical ledger already agrees before any rebuild: nouns.all - // counts every row (8 + the VFS root); vectors.all counts only the real - // ones (5) — the VFS root and the 3 explicit unvectored rows are excluded. - const ledgerBeforeReopen = await brain.storage.getCanonicalCounts() - expect(ledgerBeforeReopen.vectors.all).toBe(5) - expect(ledgerBeforeReopen.nouns.all).toBe(9) // 5 vectored + 3 unvectored + 1 VFS root - - await brain.close() - - // Reopen: open()'s index build IS hnswIndex.rebuild() run fresh from - // storage — this is the exact path that used to admit unvectored rows. - brain = openBrain(dir) - await brain.init() - - const status = await brain.getIndexStatus() - expect(status.hnswIndex.size, 'the rebuilt index must contain ONLY the 5 real-vectored rows').toBe(5) - - // Dimension is pinned to the REAL embedded length (384 via `vec()`), not - // 0 — adding a wrong-length vector must be refused naming that real - // dimension, proving no vector-less row ever set it. - const realDimension = vec(0).length - let mismatchMessage: string | undefined - try { - await brain.index.addItem({ id: uid('dimension-probe'), vector: vec(0).slice(0, realDimension - 1) }) - expect.fail('expected a dimension mismatch error') - } catch (err) { - mismatchMessage = (err as Error).message - } - expect(mismatchMessage).toContain(`expected ${realDimension}`) - - // Every unvectored row is still a normal, enumerable, readable canonical - // record — class-J semantics survive the rebuild fix untouched. - for (const id of unvectoredIds) { - const entity = await brain.get(id, { includeVectors: true }) - expect(entity, `unvectored entity ${id} must remain readable`).not.toBeNull() - expect(entity.vector).toEqual([]) - } - // A correct-dimension add succeeds cleanly against the pinned dimension. - const freshId = uid('post-reopen-fresh') - await expect(brain.add({ id: freshId, data: 'fresh', type: NounType.Document, vector: vec(50) })).resolves.toBe( - freshId - ) - - await brain.close() - }) - - it('(b) clear() then real adds afterward never trip a dimension mismatch (the clear-persistence regression shape)', async () => { - const dir = mkTmp() - let brain = openBrain(dir) - await brain.init() // the VFS root (vector: []) is the store's only row - - await brain.clear() - await brain.close() - - // Reopen over a store whose only surviving row is the recreated, - // unvectored VFS root — this is exactly the shape that used to poison - // the entry point / dimension in `clear-persistence.test.ts`. - brain = openBrain(dir) - await brain.init() - expect((await brain.getIndexStatus()).hnswIndex.size).toBe(0) - - const id1 = uid('after-clear-1') - await expect(brain.add({ id: id1, data: 'after clear 1', type: NounType.Document, vector: vec(1) })).resolves.toBe( - id1 - ) - const id2 = uid('after-clear-2') - await expect(brain.add({ id: id2, data: 'after clear 2', type: NounType.Document, vector: vec(2) })).resolves.toBe( - id2 - ) - expect((await brain.getIndexStatus()).hnswIndex.size).toBe(2) - - await brain.close() - }) - - it('(c) index.addItem/updateItem refuse a length-0 vector with EmptyVectorIndexError', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - await expect(brain.index.addItem({ id: uid('empty-add'), vector: [] })).rejects.toThrow(EmptyVectorIndexError) - - // updateItem on an EXISTING (real-vectored) node must refuse the same way. - const existingId = uid('existing-for-update') - await brain.add({ id: existingId, data: 'existing', type: NounType.Document, vector: vec(9) }) - await expect(brain.index.updateItem({ id: existingId, vector: [] })).rejects.toThrow(EmptyVectorIndexError) - - // The index was never disturbed by either refused call. - expect((await brain.getIndexStatus()).hnswIndex.size).toBe(1) - - await brain.close() - }) - - it('(d) crash -> repair: the crashed generation survives, the ledger recounts honestly, and a fresh add afterward never trips a dimension mismatch', async () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-index-skips-unvectored-crash-')) - try { - let brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) - - // Baseline: real-vectored entities, durably flushed. - const baselineIds: string[] = [] - for (let i = 0; i < 5; i++) { - const id = uid(`baseline-${i}`) - await brain.add({ id, data: `baseline entity ${i}`, type: NounType.Document, vector: vec(i) }) - baselineIds.push(id) - } - await brain.flush() - - // Crash window: at-ack writes that are never flushed before the crash. - const crashedIds: string[] = [] - for (let i = 0; i < 4; i++) { - const id = uid(`crashed-${i}`) - await brain.add({ id, data: `crash-window entity ${i}`, type: NounType.Document, vector: vec(100 + i) }) - crashedIds.push(id) - } - await abandonAsCrashed(brain) - - // Reopen — logAuthority: 'adopt' replays the at-ack log for the crash window. - brain = await openKillMatrixBrain(dir, { logAuthority: 'adopt' }) - for (const id of [...baselineIds, ...crashedIds]) { - expect(await brain.get(id), `entity ${id} must survive the crash`).not.toBeNull() - } - - // Repair — must not disturb any entity, and must recount the ledger honestly. - const report = await brain.repairIndex() - expect(report.families.length).toBeGreaterThan(0) - for (const id of [...baselineIds, ...crashedIds]) { - expect(await brain.get(id), `entity ${id} must survive repair`).not.toBeNull() - } - - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(false) - expect(ledger.vectors.all).toBe(baselineIds.length + crashedIds.length) - - // Second life: a fresh real-vectored add must never trip a dimension - // mismatch against a vector-less phantom left in the index — the exact - // mechanism `clear-persistence.test.ts` and the biography lane hit. - const secondLifeId = uid('second-life') - await expect( - brain.add({ id: secondLifeId, data: 'second life entity', type: NounType.Document, vector: vec(200) }) - ).resolves.toBe(secondLifeId) - expect(await brain.get(secondLifeId)).not.toBeNull() - - await brain.close() - } finally { - fs.rmSync(dir, { recursive: true, force: true }) - } - }) -}) diff --git a/tests/integration/ledger-derivation-identity.test.ts b/tests/integration/ledger-derivation-identity.test.ts deleted file mode 100644 index cb19af4a..00000000 --- a/tests/integration/ledger-derivation-identity.test.ts +++ /dev/null @@ -1,195 +0,0 @@ -/** - * @module tests/integration/ledger-derivation-identity - * @description The ALL-visibility ledger scalars are an IDENTITY-RECORD - * count, never a container count. A pre-8.3.1 partial-delete defect can - * leave a "ghost" container (a stale `vectors.json` with no metadata content - * leg) or a "scar" container (an empty `entities////` - * directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need - * the metadata content leg — yet the legacy derivation counted one entity - * per id DIRECTORY, so orphaned containers inflated the ALL scalars forever - * (they were never clamped and never re-derived). Laws under test: - * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per - * metadata content leg (`metadata.json` or `.json.gz`), the same test - * `pruneOrphanedEntities()` uses, so the two agree by construction. - * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that - * carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'` - * stamp predates this fix; loading it marks `suspect = true` from a - * single field read alone, never a directory walk, and warns exactly - * once naming the cause. - * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned - * containers, recounts from the canonical metadata.json walk, and - * re-stamps — suspect clears and the ALL scalar is exact again. - * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store - * with no counts.json stamps as it writes, so a brand-new store never - * carries the legacy signature. - */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy, FileSystemStorage } from '../../src/index.js' -import { prodLog } from '../../src/utils/logger.js' - -const countsPath = (root: string) => path.join(root, '_system', 'counts.json') - -/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */ -function plantGhost(root: string, shard: string, id: string): void { - const idDir = path.join(root, 'entities', 'nouns', shard, id) - fs.mkdirSync(idDir, { recursive: true }) - fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] })) -} - -/** Plant a scar container: an empty id directory, no legs at all. */ -function plantScar(root: string, shard: string, id: string): void { - fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true }) -} - -describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => { - let dir: string - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - beforeEach(() => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-')) - }) - afterEach(() => { - vi.restoreAllMocks() - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => { - let brain = await open() - const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone - for (let i = 0; i < 3; i++) { - await brain.add({ data: `real ${i}`, type: 'document' }) - } - await brain.flush() - const realTotal = baseline + 3 - await brain.close() - - // 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) — - // neither is a live entity. - for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) - for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) - - // Remove counts.json so open() re-derives from scratch (the one-time - // legacy/lost-file derivation path). - fs.rmSync(countsPath(dir), { force: true }) - - brain = await open() - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing - expect(ledger.suspect).toBe(false) - - const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - expect(raw.totalNounCountAll).toBe(realTotal) - expect(raw.allCountsDerivedBy).toBe('identity-record') - - await brain.close() - }) - - it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => { - let brain = await open() - await brain.add({ data: 'one', type: 'document' }) - await brain.add({ data: 'two', type: 'document' }) - await brain.flush() - await brain.close() - - // Confirm a normal close under the fix DOES stamp — then strip the stamp - // to simulate a counts.json produced before this fix existed. - const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - expect(raw.allCountsDerivedBy).toBe('identity-record') - expect(typeof raw.totalNounCountAll).toBe('number') - expect(typeof raw.totalVerbCountAll).toBe('number') - expect(typeof raw.totalVectoredNounCount).toBe('number') - delete raw.allCountsDerivedBy - fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) - - const warnSpy = vi.spyOn(prodLog, 'warn') - // The two derivation walks live on FileSystemStorage's prototype — - // spying here (rather than on fs.promises.readdir globally) isolates - // THIS code path's behavior from unrelated walks elsewhere in the open - // sequence (a separate, pre-existing engine's own O(store) cost — not - // this fix's concern, and not something this pin should be sensitive - // to). Neither derivation method may run: the stamp check is a field - // read on the already-parsed counts.json, nothing more. - const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities') - const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount') - - brain = await open() - - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(true) - - const stampWarnings = warnSpy.mock.calls.filter( - ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule') - ) - expect(stampWarnings.length).toBe(1) // exactly one, loud - - expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk - expect(scanVectoredSpy).not.toHaveBeenCalled() - - await brain.close() - }) - - it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => { - let brain = await open() - const baseline = (await brain.storage.getCanonicalCounts()).nouns.all - for (let i = 0; i < 3; i++) { - await brain.add({ data: `real ${i}`, type: 'document' }) - } - await brain.flush() - const realTotal = baseline + 3 - await brain.close() - - for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) - for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) - - // Force the legacy (unstamped, container-rule-inflated) shape directly — - // the shape a pre-existing production store actually carries. - const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars - delete raw.allCountsDerivedBy - fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) - - brain = await open() - expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load - - await brain.repairIndex() - - let ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(false) - expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again - - const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - expect(persisted.allCountsDerivedBy).toBe('identity-record') - expect(persisted.allCountsSuspect).toBe(false) - expect(persisted.totalNounCountAll).toBe(realTotal) - - await brain.close() - brain = await open() - ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(false) - expect(ledger.nouns.all).toBe(realTotal) - await brain.close() - }) - - it('(d) a fresh store derives with the stamp and is never suspect', async () => { - const brain = await open() - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.suspect).toBe(false) - const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) - expect(raw.allCountsDerivedBy).toBe('identity-record') - await brain.close() - }) -}) diff --git a/tests/integration/vector-leg-open-build.test.ts b/tests/integration/vector-leg-open-build.test.ts index ea841545..a1ccaefa 100644 --- a/tests/integration/vector-leg-open-build.test.ts +++ b/tests/integration/vector-leg-open-build.test.ts @@ -176,22 +176,21 @@ describe('vector-leg open-build (two-engine gate, last red)', () => { }) it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => { - // ARCHITECTURAL NOTE (updated by the zero-norm root cure): every brainy - // store carries ONE permanent VFS root noun beyond user data - // (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, - // src/vfs/VirtualFileSystem.ts), created (or, on a pre-fix store, - // migrated) on every open — but it is deliberately UNVECTORED (vector - // `[]`), never a real all-zero placeholder: a zero-norm vector is not a - // vector and never crosses an engine boundary (see that file's - // doInitializeRoot() comment). It therefore contributes NOTHING to the - // vectored-noun ledger — a brand-new store's `vectors.all` floor is 0, - // not 1. This pin verifies the law the task names in the ACHIEVABLE - // form: nouns whose embed is still deferred/unlanded contribute NOTHING - // to the vectored-noun ledger either — the coverage-gap comparison sees - // exactly the baseline (the root, contributing 0), never - // baseline+deferred — and semantic search over deferred-only user - // content honestly returns `[]` (no error, no false "coverage restored" - // claim). + // ARCHITECTURAL NOTE (found while building this pin): every brainy store + // carries ONE permanent, always-vectored noun beyond user data — the VFS + // root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, + // src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero + // (but non-empty, length-384) vector on EVERY store's first open — never + // deferred (a deliberate WASM-cold-compile-avoidance fix, see that + // file's comment) — and VFS init unconditionally re-creates it if + // missing, before the rebuild gate ever runs. A literal "0 vectored + // nouns" store is therefore unreachable through the public API; a + // brand-new store's `vectors.all` floor is 1, not 0. This pin verifies + // the law the task names in the ACHIEVABLE form: nouns whose embed is + // still deferred/unlanded contribute NOTHING to the vectored-noun ledger + // — the coverage-gap comparison sees exactly the root (1), never + // root+deferred — and semantic search over deferred-only user content + // honestly returns `[]` (no error, no false "coverage restored" claim). const dir = mkTmp() const build: any = new Brainy({ @@ -203,8 +202,6 @@ describe('vector-leg open-build (two-engine gate, last red)', () => { }) await build.init() const rootOnlyLedger = await build.storage.getCanonicalCounts() - // THE NEW LAW: the root is unvectored — a brand-new store's floor is 0. - expect(rootOnlyLedger.vectors.all).toBe(0) // Block the embedder permanently so every add below stays deferred and // unlanded for the rest of this test (a fast deterministic embedder // could otherwise land it before we ever observe the "still 0 extra" diff --git a/tests/integration/vfs-root-zero-norm.test.ts b/tests/integration/vfs-root-zero-norm.test.ts deleted file mode 100644 index 577ae7ee..00000000 --- a/tests/integration/vfs-root-zero-norm.test.ts +++ /dev/null @@ -1,216 +0,0 @@ -/** - * @module tests/integration/vfs-root-zero-norm - * @description THE ZERO-NORM ROOT CURE — a production incident traced 150+ - * darkened rows in a downstream engine's index to the VFS root's persisted - * ALL-ZERO placeholder vector: lawful inside brainy (`cosineDistance` - * treats a zero-norm operand as MAXIMUM distance, src/utils/distance.ts) - * but a "false attractor" for an engine serving squared-euclidean distance, - * which cannot tell a real all-zero vector apart from a legitimate origin - * point. THE LAW: a zero-norm vector is not a vector — it never crosses an - * engine boundary. - * - * Three legs pinned here: - * (a) the root persists NO zeros — a brand-new store creates it with - * vector `[]` (the "unvectored" shape), absent from the HNSW index, and - * the canonical vectored-noun ledger does not count it. - * (b) a ONE-TIME migration heals an existing (pre-fix) store: an old-shape - * root (a REAL all-zero vector, genuinely indexed and ledgered — the - * harness reproduces exactly what a pre-fix store looked like on disk) - * is rewritten to `[]` on the next `init()`, the ledger is decremented - * through the sanctioned path, and a second `init()` is a no-op. - * (c) THE CANONICAL-WRITE NORMALIZATION (Leg A of the follow-up - * zero-norm/unvector-door fix): an entity added with an EXPLICIT - * all-zero vector (any dimension) is normalized to the "unvectored" - * `[]` shape BEFORE the canonical write, the ledger flag, and the index - * ops ever see it — the canonical write still succeeds, loudly, and the - * vector-index insert never happens (nothing to index). Supersedes the - * original "canonical keeps the zero vector, only the index refuses" - * shape: a downstream engine's health-report gate reads the canonical - * ledger directly, so leaving a zero-norm vector on the canonical side - * re-opened the exact false-attractor risk this whole fix closes. - * (d) the migrated root never surfaces in `find()` results (it was already - * hidden behind `visibility: 'system'` — this pin holds regardless). - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' -import { prodLog } from '../../src/utils/logger.js' - -const ROOT_ID = '00000000-0000-0000-0000-000000000000' - -process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - -const tmpDirs: string[] = [] -function mkTmp(): string { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vfs-root-zero-norm-')) - tmpDirs.push(d) - return d -} -afterEach(() => { - vi.restoreAllMocks() - for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) -}) - -function openBrain(dir: string): any { - return new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) -} - -describe('VFS root zero-norm cure', () => { - it('(a) a brand-new store persists the root with vector [], absent from the HNSW index, and the canonical ledger counts it unvectored', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const root = await brain.get(ROOT_ID, { includeVectors: true }) - expect(root).not.toBeNull() - expect(root.vector).toEqual([]) - - const status = await brain.getIndexStatus() - expect(status.hnswIndex.size).toBe(0) - - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(0) - - await brain.close() - }) - - it('(b) an old-shape store (a real all-zero placeholder root) migrates to [] exactly once on init; the ledger is decremented through the sanctioned path; a second init is a no-op', async () => { - const dir = mkTmp() - - // SESSION 1 — build the store, then hand-rewrite the root to the LEGACY - // shape: a REAL all-zero 384-dim vector, genuinely inserted into the - // vector index and genuinely counted by the vectored-noun ledger — - // reproducing exactly what a pre-fix store's root looked like on disk - // (the pre-fix add() always indexed + counted it). `index.addItem` is - // called directly (bypassing AddToVectorIndexOperation's own zero-norm - // belt, added by this same fix) precisely because the pre-fix code path - // had no such belt — this harness must match history, not the cure. - let brain = openBrain(dir) - await brain.init() - const oldVector = new Array(384).fill(0) - await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) - await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) - await brain.storage.noteVectorLanded(ROOT_ID) - await brain.storage.persistCounts() - await brain.flush() - - const ledgerBeforeMigration = await brain.storage.getCanonicalCounts() - expect(ledgerBeforeMigration.vectors.all).toBe(1) - await brain.close() - - // SESSION 2 — reopen: VFS init must detect the legacy shape and migrate. - // Spy on the sanctioned migration method itself (not console output — - // `silent: true` monkey-patches `console.log` to a no-op INSIDE init(), - // which would silently swallow any pre-installed console spy too). - brain = openBrain(dir) - const migrateSpy = vi.spyOn(brain, 'unvectorNounForRootMigration') - await brain.init() - - expect(migrateSpy).toHaveBeenCalledTimes(1) - expect(migrateSpy).toHaveBeenCalledWith(ROOT_ID) - await expect(migrateSpy.mock.results[0].value).resolves.toBe(true) - - const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) - expect(migratedRoot.vector).toEqual([]) - - const ledgerAfterMigration = await brain.storage.getCanonicalCounts() - expect(ledgerAfterMigration.vectors.all).toBe(0) - - const statusAfterMigration = await brain.getIndexStatus() - expect(statusAfterMigration.hnswIndex.size).toBe(0) - - await brain.flush() - await brain.close() - - // SESSION 3 — reopen again: the migration is a permanent no-op, not a - // one-time flag that silently re-drifts or re-fires. The zero-norm - // detection at the VFS init site never even calls the migration method - // again — the root's vector is already `[]`. - brain = openBrain(dir) - const migrateSpy2 = vi.spyOn(brain, 'unvectorNounForRootMigration') - await brain.init() - - expect(migrateSpy2).not.toHaveBeenCalled() - - const rootAfterSecondInit = await brain.get(ROOT_ID, { includeVectors: true }) - expect(rootAfterSecondInit.vector).toEqual([]) - - const ledgerAfterSecondInit = await brain.storage.getCanonicalCounts() - expect(ledgerAfterSecondInit.vectors.all).toBe(0) - - await brain.close() - }) - - it('(c) canonical-write normalization: an entity added with an explicit all-zero vector persists UNVECTORED ([]), loudly, and never reaches the vector index', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const warnSpy = vi.spyOn(prodLog, 'warn') - - const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size - const ledgerBefore = await brain.storage.getCanonicalCounts() - const zeroVector = new Array(384).fill(0) - const id = await brain.add({ data: 'poisoned entity', type: NounType.Document, vector: zeroVector }) - - // The canonical write succeeded — but the zero-norm vector was - // normalized to the "unvectored" `[]` shape BEFORE it was persisted - // (Leg A: a zero-norm vector is not a vector — it never crosses an - // engine boundary, canonical side included). - const entity = await brain.get(id, { includeVectors: true }) - expect(entity).not.toBeNull() - expect(entity.vector).toEqual([]) - - // Nothing to index — the vector-index size never moved, and the - // vectored-noun ledger never counted this row. - const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size - expect(sizeAfter).toBe(sizeBefore) - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - - // The normalization was LOUD and named the entity. - const loudCall = warnSpy.mock.calls.find( - (call) => typeof call[0] === 'string' && call[0].includes(id) && call[0].toLowerCase().includes('zero-norm') - ) - expect(loudCall).toBeDefined() - - await brain.close() - }) - - it('(d) find() over a store whose root has been migrated never returns the root (already hidden behind visibility: system — pinned anyway)', async () => { - const dir = mkTmp() - - // Build an old-shape store (same harness as pin (b)) and let it migrate. - let brain = openBrain(dir) - await brain.init() - const oldVector = new Array(384).fill(0) - await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) - await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) - await brain.storage.noteVectorLanded(ROOT_ID) - await brain.storage.persistCounts() - await brain.add({ data: 'a document about technology', type: NounType.Document }) - await brain.flush() - await brain.close() - - brain = openBrain(dir) // migrates on init() - await brain.init() - - const results = await brain.find({ query: 'technology', limit: 10 }) - expect(results.some((r: any) => r.id === ROOT_ID)).toBe(false) - - // Even asking explicitly for system-tier entities must never surface the - // root as a semantic-search HIT (it carries no vector to match against). - const resultsIncludingSystem = await brain.find({ query: 'technology', limit: 10, includeSystem: true }) - expect(resultsIncludingSystem.some((r: any) => r.id === ROOT_ID)).toBe(false) - - await brain.close() - }) -}) diff --git a/tests/integration/zero-norm-unvector-door.test.ts b/tests/integration/zero-norm-unvector-door.test.ts deleted file mode 100644 index d3290010..00000000 --- a/tests/integration/zero-norm-unvector-door.test.ts +++ /dev/null @@ -1,399 +0,0 @@ -/** - * @module tests/integration/zero-norm-unvector-door - * @description THE SEAM LAW, GENERALIZED: "a zero-norm vector is not a - * vector — it never crosses an engine boundary." `tests/integration/ - * vfs-root-zero-norm.test.ts` pins the VFS-root-specific cure; this file - * pins the follow-up that generalizes it to every write path plus the - * sanctioned door for shedding a vector on purpose. - * - * Four legs pinned here: - * (A) THE CANONICAL WRITE NORMALIZES ZERO-NORM TO `[]` — `add()` (single and - * `transact()`) persists an explicit real all-zero vector as the - * "unvectored" `[]` shape, loudly, before the ledger flag/dimension - * pin/index ops ever see it. The canonical write still succeeds. - * (B) THE LEGACY DERIVATION IS ZERO-NORM-AWARE — a lost/corrupted - * `counts.json`'s one-time re-derivation walk excludes a persisted - * zero-norm row from the vectored-noun scalar, matching the live - * ledger's definition of "vectored". - * (C) THE LEGACY VFS ROOT MIGRATES AT OPEN, BEFORE THE GATE, IN O(1) — a - * store whose ONLY vectored row is a legacy all-zero VFS root opens - * clean (no `VectorIndexNotReadyError`), via one fixed-path read, never - * a listing. - * (D) THE UNVECTOR DOOR — `update({ id, vector: [] })` (and the same op - * inside `transact()`) is the sanctioned, idempotent way to shed a - * vector on purpose: ledger decrement exactly once, index removal, no - * re-embed, and a pending deferred-embed marker is cleared rather than - * left to re-vectorize the row later. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { NounType } from '../../src/types/graphTypes.js' -import { prodLog } from '../../src/utils/logger.js' -import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' -import { BaseStorage } from '../../src/storage/baseStorage.js' - -const ROOT_ID = '00000000-0000-0000-0000-000000000000' - -process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - -const tmpDirs: string[] = [] -function mkTmp(): string { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-zero-norm-unvector-')) - tmpDirs.push(d) - return d -} -afterEach(() => { - vi.restoreAllMocks() - for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) -}) - -function openBrain(dir: string): any { - return new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) -} - -const countsPath = (root: string) => path.join(root, '_system', 'counts.json') - -describe('zero-norm canonical write + the sanctioned unvector door', () => { - it('(A1) add() with an explicit all-zero vector persists [], warns loudly, never indexes, and the ledger is unchanged', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const ledgerBefore = await brain.storage.getCanonicalCounts() - const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size - const warnSpy = vi.spyOn(prodLog, 'warn') - - const zeroVector = new Array(384).fill(0) - const id = await brain.add({ data: 'zero-norm add', type: NounType.Document, vector: zeroVector }) - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity).not.toBeNull() - expect(entity.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - - const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size - expect(sizeAfter).toBe(sizeBefore) - - const loud = warnSpy.mock.calls.find( - (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') - ) - expect(loud).toBeDefined() - - await brain.close() - }) - - it('(A2) transact() add with an explicit all-zero vector — the same canonical normalization', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const ledgerBefore = await brain.storage.getCanonicalCounts() - const warnSpy = vi.spyOn(prodLog, 'warn') - const zeroVector = new Array(384).fill(0) - const id = 'aaaaaaaa-0000-4000-8000-000000000001' - - await brain.transact([ - { op: 'add', id, type: NounType.Document, data: 'zero-norm transact add', vector: zeroVector } - ]) - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity).not.toBeNull() - expect(entity.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all) - - const loud = warnSpy.mock.calls.find( - (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('zero-norm') - ) - expect(loud).toBeDefined() - - await brain.close() - }) - - it('(B) the legacy counts.json derivation excludes a persisted zero-norm row from the vectored-noun scalar', async () => { - const dir = mkTmp() - let brain = openBrain(dir) - await brain.init() - - // The VFS root alone (unvectored — []) — the floor. - const baseline = (await brain.storage.getCanonicalCounts()).vectors.all - - const realId = await brain.add({ data: 'a real vectored document', type: NounType.Document }) - - // Plant the legacy all-zero shape BY HAND: a genuine identity record - // (via add(), so it has real metadata) whose vector leg is then - // overwritten directly through the raw storage primitive — bypassing - // Leg A's canonical-write normalization entirely (brain.storage.saveNoun - // is not Brainy.add()/update()'s normalized path) — reproducing exactly - // what a pre-fix store could have persisted on disk. - const zeroId = await brain.add({ data: 'a legacy zero-norm document', type: NounType.Document }) - const zeroVector = new Array(384).fill(0) - await brain.storage.saveNoun({ id: zeroId, vector: zeroVector, connections: new Map(), level: 0 }) - - await brain.flush() - await brain.close() - - // Remove counts.json so the next open re-derives from scratch (the - // one-time legacy/lost-file derivation path — Leg B). - fs.rmSync(countsPath(dir), { force: true }) - - brain = openBrain(dir) - await brain.init() - const ledger = await brain.storage.getCanonicalCounts() - // Only realId counts; zeroId's persisted all-zero vector does not. - expect(ledger.vectors.all).toBe(baseline + 1) - - await brain.close() - }) - - it('(C) a legacy all-zero VFS root as the ONLY vectored row: open succeeds with no not-ready error, via an O(1) fixed-path read (no entities-tree readdir), and the ledger is 0 after open', async () => { - const dir = mkTmp() - - // SESSION 1 — build the legacy shape: the root is a REAL all-zero - // 384-dim vector, genuinely indexed and genuinely ledgered — exactly - // what a pre-fix store's root looked like on disk (see - // vfs-root-zero-norm.test.ts pin (b) for the identical harness). - // `index.addItem` is called directly (bypassing the transactional - // zero-norm belt) because the pre-fix code path had no such belt — this - // harness must match history, not the cure. No other entity is added, - // so the root is the store's ONLY vectored row. - let brain = openBrain(dir) - await brain.init() - const oldVector = new Array(384).fill(0) - await brain.storage.saveNoun({ id: ROOT_ID, vector: oldVector, connections: new Map(), level: 0 }) - await brain.index.addItem({ id: ROOT_ID, vector: oldVector }) - await brain.storage.noteVectorLanded(ROOT_ID) - await brain.storage.persistCounts() - await brain.flush() - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(1) - await brain.close() - - // SESSION 2 — reopen with a FAKE native vector provider that claims - // `serving: true` at `size()===0` (the exact shape a downstream - // engine's own health report can legitimately carry — same technique as - // tests/integration/vector-leg-open-build.test.ts). This is the ONLY - // codepath where the vector-leg open gate's FAIL-TYPED throw - // (VectorIndexNotReadyError) can fire; the built-in JS engine alone - // never reaches it (the size-heuristic branch just rebuilds instead) — - // so this is the faithful reproduction of the incident Leg C closes. - const readdirCalls: string[] = [] - const originalReaddir = fs.promises.readdir.bind(fs.promises) - vi.spyOn(fs.promises, 'readdir').mockImplementation(((...args: any[]) => { - readdirCalls.push(String(args[0])) - return (originalReaddir as any)(...args) - }) as any) - - // Spy at the PROTOTYPE level (BaseStorage.getNoun) — the new brain's - // storage instance does not exist until init() runs, so an - // instance-level spy cannot be installed beforehand. Records the - // readdir-call delta across the FIRST call made with the root id — - // Leg C's own fixed-path read — proving it needs no directory listing. - let readdirDeltaDuringRootRead: number | null = null - const originalGetNoun = BaseStorage.prototype.getNoun - vi.spyOn(BaseStorage.prototype, 'getNoun').mockImplementation(async function ( - this: unknown, - id: string - ) { - const before = readdirCalls.length - const result = await originalGetNoun.call(this as BaseStorage, id) - if (id === ROOT_ID && readdirDeltaDuringRootRead === null) { - readdirDeltaDuringRootRead = readdirCalls.length - before - } - return result - }) - - brain = openBrain(dir) - brain.use({ - name: 'fake-native-vector-unledgered-coverage', - activate: async (ctx: any) => { - ctx.registerProvider('vector', (config: any, distance: any, options: any) => { - const real = new JsHnswVectorIndex(config, distance, options) - let rebuilt = false - const originalRebuild = real.rebuild.bind(real) - ;(real as any).rebuild = async (...args: any[]) => { - const r = await originalRebuild(...args) - rebuilt = true - return r - } - const originalSize = real.size.bind(real) - ;(real as any).size = () => (rebuilt ? originalSize() : 0) - ;(real as any).healthReport = () => ({ - provider: 'vector', - healthy: true, - serving: true, - invariants: [], - checkedAt: Date.now(), - durationMs: 0, - generation: 1, - unledgered: ['vector-coverage'] - }) - return real - }) - return true - } - }) - - // Must NOT throw VectorIndexNotReadyError (or anything else) — a - // near-empty store whose only vectored row is the zero-norm root must - // never go dark. - await brain.init() - - const migratedRoot = await brain.get(ROOT_ID, { includeVectors: true }) - expect(migratedRoot.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(0) - - expect(readdirDeltaDuringRootRead).toBe(0) - - await brain.close() - }) - - describe('the sanctioned unvector door', () => { - it('(D1) update({ id, vector: [] }) unvectors a real vectored row — canonical [], removed from the index, ledger decremented by exactly 1, no embed call', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const id = await brain.add({ data: 'a real document', type: NounType.Document }) - await brain.flush() - - const ledgerBefore = await brain.storage.getCanonicalCounts() - const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size - - const embedSpy = vi.spyOn(brain, 'embed') - await brain.update({ id, vector: [] }) - expect(embedSpy).not.toHaveBeenCalled() - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) - - const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size - expect(sizeAfter).toBe(sizeBefore - 1) - - await brain.close() - }) - - it('(D2) idempotent: a second update({ id, vector: [] }) on an already-unvectored row is a true no-op — no error, no further decrement', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const id = await brain.add({ data: 'a real document', type: NounType.Document }) - await brain.flush() - - await brain.update({ id, vector: [] }) - const ledgerAfterFirst = await brain.storage.getCanonicalCounts() - - await brain.update({ id, vector: [] }) - const ledgerAfterSecond = await brain.storage.getCanonicalCounts() - expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfterFirst.vectors.all) - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity.vector).toEqual([]) - - await brain.close() - }) - - it('(D3) a PENDING deferred-embed row: the unvector door clears the marker; awaitPendingEmbeds() then leaves it unvectored', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - // Prevent the background worker from ever actually running — it is - // fire-and-forget from add(), and a real run would race this test's - // own assertions (see tests/integration/vector-leg-open-build.test.ts - // for the same concern). This isolates exactly the marker-clearing - // behavior under test. - vi.spyOn(brain as any, 'kickEmbedWorker').mockImplementation(() => {}) - - const id = await brain.add({ - data: 'deferred content, never embedded', - type: NounType.Document, - deferEmbedding: true - }) - expect(brain.pendingEmbedCount()).toBe(1) - - const warnSpy = vi.spyOn(prodLog, 'warn') - await brain.update({ id, vector: [] }) - - expect(brain.pendingEmbedCount()).toBe(0) - const clearedWarn = warnSpy.mock.calls.find( - (c) => typeof c[0] === 'string' && c[0].includes(id) && c[0].toLowerCase().includes('pending') - ) - expect(clearedWarn).toBeDefined() - - // The barrier must not hang and must not re-vectorize the row — the - // worker (still mocked to a no-op) never runs again. - await brain.awaitPendingEmbeds() - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity.vector).toEqual([]) - - await brain.close() - }) - - it('(D4) update({ vector: [], deferEmbedding: true }) is a typed refusal — the unvector door cannot be paired with a deferred embed', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const id = await brain.add({ data: 'a real document', type: NounType.Document }) - const before = await brain.get(id, { includeVectors: true }) - - await expect( - brain.update({ id, vector: [], deferEmbedding: true }) - ).rejects.toThrow(/unvector door/i) - - // Refused before any write — the row is untouched. - const after = await brain.get(id, { includeVectors: true }) - expect(after.vector).toEqual(before.vector) - - await brain.close() - }) - - it('(D5) the transact() twin of the unvector door decrements the ledger exactly once, and is idempotent on a second call', async () => { - const dir = mkTmp() - const brain = openBrain(dir) - await brain.init() - - const id = await brain.add({ data: 'a real document for transact unvector', type: NounType.Document }) - await brain.flush() - - const ledgerBefore = await brain.storage.getCanonicalCounts() - const sizeBefore = (await brain.getIndexStatus()).hnswIndex.size - - await brain.transact([{ op: 'update', id, vector: [] }]) - - const entity = await brain.get(id, { includeVectors: true }) - expect(entity.vector).toEqual([]) - - const ledgerAfter = await brain.storage.getCanonicalCounts() - expect(ledgerAfter.vectors.all).toBe(ledgerBefore.vectors.all - 1) - - const sizeAfter = (await brain.getIndexStatus()).hnswIndex.size - expect(sizeAfter).toBe(sizeBefore - 1) - - // Idempotent through transact() too. - await brain.transact([{ op: 'update', id, vector: [] }]) - const ledgerAfterSecond = await brain.storage.getCanonicalCounts() - expect(ledgerAfterSecond.vectors.all).toBe(ledgerAfter.vectors.all) - - await brain.close() - }) - }) -}) diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts index 274f0ef0..8b274fce 100644 --- a/tests/lifecycle/biography.test.ts +++ b/tests/lifecycle/biography.test.ts @@ -382,17 +382,9 @@ describe.sequential('lifecycle — the working store', () => { }, // Every noun this biography ever adds carries an explicit/computed // vector (the harness never defers an embed), so the vectored-noun - // scalar tracks nouns.all exactly EXCEPT for the VFS root counted - // in `vfsBaselineNouns`: the root is deliberately persisted with - // `vector: []` (the sanctioned "unvectored" shape — see - // VirtualFileSystem.doInitializeRoot()'s zero-norm-avoidance - // comment) so it never pays the WASM engine's cold-compile cost and - // never crosses an engine boundary as a false attractor. It is the - // ONE hidden-tier record `vfsBaselineNouns` represents (see - // biographyHarness's module header), so it is excluded here even - // though it counts toward `nouns.all`. + // scalar tracks nouns.all exactly. vectors: { - all: aliveEntities.length + model.vfsFileNouns + all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns }, suspect: false })