diff --git a/CHANGELOG.md b/CHANGELOG.md index 494ae950..c2342d59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,6 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. -### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25) - -- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b) - - ### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25) - fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e) diff --git a/package-lock.json b/package-lock.json index 9daf75f3..519bbfac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.4", + "version": "10.4.0-rc.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.4", + "version": "10.4.0-rc.3", "license": "MIT", "dependencies": { "@msgpack/msgpack": "^3.1.2", diff --git a/package.json b/package.json index fdee766f..467b1fef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@soulcraft/brainy", - "version": "10.4.0-rc.4", + "version": "10.4.0-rc.3", "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.", "main": "dist/index.js", "module": "dist/index.js", diff --git a/src/brainy.ts b/src/brainy.ts index 543286a2..069a77c0 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -2382,17 +2382,6 @@ export class Brainy implements BrainyInterface { [{ type: 'embed.landed', id, vector: newVector }], 'system:embed-landing' ) - // Vectored-noun ledger: the landing commit above carries a vector - // write with NO accompanying metadata operation, so the - // saveNounMetadata(..., hasVector) seam never fires for it — the - // narrow storage hook is the only seam left. `oldVector.length===0` - // (already known for free from the pre-embed read above) proves this - // is a GENUINE first landing, not a re-embed of an already-vectored - // row (e.g. a deferred update() on a row that already had a real - // vector) — the latter must never double-count. - if (oldVector.length === 0) { - await this.storage.noteVectorLanded?.(id) - } this.clearPendingEmbed(id) } catch (err) { prodLog.warn( @@ -3011,11 +3000,8 @@ export class Brainy implements BrainyInterface { const runInsert: TransactionFunction = async (tx) => { // Operation 1: Save metadata FIRST (TypeAwareStorage caching) // isNew=true: skip pre-read for rollback (entity doesn't exist yet) - // hasVector: the vectored-noun ledger counts this insert iff its - // vector is real/non-empty (never true for a deferred embed, whose - // stub `vector` is `[]` — it counts later, at landing). tx.addOperation( - new SaveNounMetadataOperation(this.storage, id, storageMetadata, true, vector.length > 0) + new SaveNounMetadataOperation(this.storage, id, storageMetadata, true) ) // Operation 2: Save vector data @@ -10582,9 +10568,7 @@ export class Brainy implements BrainyInterface { plan.postCommit.push(() => this.kickEmbedWorker()) } plan.operations.push( - // hasVector: see the single-add() insert path's comment — never true - // for a deferred embed (stub vector `[]`; counted later at landing). - new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0), + new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), ...(deferringEmbed ? [] @@ -16916,92 +16900,9 @@ export class Brainy implements BrainyInterface { const shouldRebuildMetadata = !metadataMigrating && (epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0)) - - // VECTOR LEG — the two-engine gate's last red: a migrated 7.x-era store - // can hold canonical vectored nouns with NO derived vector index built. - // `legNeedsRebuild`'s size-heuristic fallback (below) only fires off - // `hnswIndexSize === 0`, and its health-report branch trusts a - // provider's own `serving` verdict verbatim — but a provider's health - // report can legitimately say `serving: true` while vector coverage is - // honestly UNLEDGERED on ITS side too (an unledgered invariant never - // flips serving), so neither signal alone can tell "genuinely empty" - // apart from "never built". The canonical vectored-noun ledger - // (`getCanonicalCounts().vectors.all` — Deliverable 1) is the - // denominator that CAN tell them apart, and is compared here: - // - a CONFIDENT (non-suspect) ledger `> 0` while the reported node - // count is 0 is a proven coverage gap — force the build regardless - // of what a health report claims; - // - a CONFIDENT ledger `=== 0` while the node count is 0 proves there - // is nothing to load (e.g. every noun's embed is still deferred) — - // skip the size-heuristic fallback's blunt "always rebuild when - // empty" trigger, which otherwise wastes a full canonical walk for - // zero benefit on every cold open of such a store; - // - an unavailable/suspect ledger changes nothing — loud errors never - // quiet losses, so a doubtful ledger must never suppress a rebuild - // the old heuristic would have run. - // The bare `isReady()` boolean (no report, no `unledgered` concept) is - // NOT overridden — that signal is what fixed the 48-seconds-per-restart - // regression pinned in tests/unit/cold-open-rebuild-gate.test.ts (a - // disk-native provider legitimately reporting 0 resident while durable - // on disk), and re-deriving it from a denominator the provider itself - // has no way to consult would reopen exactly that regression. - const vectorAssessment = assessProviderHealth(this.index) - const vectorLedger = await this.storage.getCanonicalCounts?.() - const vectorLedgerAll = vectorLedger?.vectors.all - const vectorLedgerConfident = vectorLedger !== undefined && !vectorLedger.suspect - const vectorHasCoverageProof = vectorLedgerConfident && (vectorLedgerAll as number) > 0 - const vectorConfirmedEmpty = vectorLedgerConfident && vectorLedgerAll === 0 - - let vectorNeedsRebuild: boolean - if (vectorAssessment.via === 'is-ready') { - // Bare isReady() stays authoritative and UNMODIFIED — see above. - vectorNeedsRebuild = vectorAssessment.readiness === 'not-ready' - } else if (vectorAssessment.via === 'health-report') { - vectorNeedsRebuild = - vectorAssessment.readiness !== 'ready' || - (hnswIndexSize === 0 && vectorHasCoverageProof) - } else { - // size-heuristic / no provider (the built-in JS engine's own posture). - vectorNeedsRebuild = hnswIndexSize === 0 && !vectorConfirmedEmpty - } - - const shouldRebuildVector = !vectorMigrating && (epochStale || vectorNeedsRebuild) - - // Narration (and the FAIL-TYPED backstop below) are scoped EXACTLY to - // the defect this gate closes: a provider whose OWN health report - // claims `serving: true` — an affirmative "I am ready" a caller would - // otherwise trust outright — while the canonical ledger proves vector - // coverage is missing. This is deliberately NARROWER than "any branch - // where the ledger contributed to the decision": - // - the bare isReady() branch is untouched, as above (never in scope); - // - the health-report branch's OWN `readiness !== 'ready'` case is - // already an ordinary, PRE-EXISTING rebuild trigger (the provider - // admits not-ready) — not a ledger override, so not a "gap"; - // - the size-heuristic/no-provider branch's rebuild-when-empty is the - // SAME blunt trigger the code always had (`hnswIndexSize === 0`) - // — the ledger only ever SUPPRESSES a rebuild there (the confirmed- - // empty case), it never forces one the old heuristic wouldn't - // already have run. Marking that branch a "gap" too made the - // FAIL-TYPED backstop fire on ordinary white-box tests that stub - // rebuild() as a no-op and pin `size()` at 0 to drive OTHER - // assertions (e.g. migration-deference's isMigrating() coverage) — - // those are not silent-empty defects, so they must open exactly as - // before (tests/unit/brainy/migration-deference.test.ts). - const vectorCoverageGap = + const shouldRebuildVector = !vectorMigrating && - vectorAssessment.via === 'health-report' && - vectorAssessment.readiness === 'ready' && - hnswIndexSize === 0 && - vectorHasCoverageProof - if (vectorCoverageGap) { - prodLog.warn( - `[Brainy] open(): vector index reports ${hnswIndexSize} node(s) but the canonical ` + - `ledger holds ${vectorLedgerAll} vectored noun(s) — the derived vector index is ` + - `missing or unbuilt on this store. Forcing the vector rebuild rather than serving ` + - `silent-empty search results.` - ) - } - + (epochStale || legNeedsRebuild(this.index, hnswIndexSize === 0)) const shouldRebuildGraph = !graphMigrating && (epochStale || legNeedsRebuild(this.graphIndex, false)) @@ -17054,27 +16955,9 @@ export class Brainy implements BrainyInterface { // provider running its own background migration is skipped here (it owns // its index until it verifies-and-swaps). const rebuildStartTime = Date.now() - // The vector leg's build door, by contract with the native provider: a - // provider exposing fillFromCanonical() gets THAT call — idempotent, the - // provider's own init runs it first so this is the backstop — never a - // full rebuild() for a coverage gap. A PARTIAL shortfall deliberately - // triggers nothing here: that is repair()'s operator door. The JS index - // has no fill door and keeps its rebuild. - const vectorBuild = (): Promise => { - const fillDoor = (this.index as unknown as { fillFromCanonical?: () => Promise }) - .fillFromCanonical - if (vectorCoverageGap && typeof fillDoor === 'function') { - prodLog.warn( - `[Brainy] open(): vector coverage gap routes through the provider's ` + - `fillFromCanonical() (idempotent canonical fill), not a full rebuild.` - ) - return fillDoor.call(this.index) - } - return this.index.rebuild() - } await Promise.all([ shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(), - shouldRebuildVector ? vectorBuild() : Promise.resolve(), + shouldRebuildVector ? this.index.rebuild() : Promise.resolve(), shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve() ]) @@ -17115,24 +16998,6 @@ export class Brainy implements BrainyInterface { console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`) } - // Vector coverage verification: the coverage-gap rebuild above (see - // `vectorCoverageGap`) MUST have actually restored the ledger's - // vectored nouns. A provider that STILL reports 0 nodes after its own - // rebuild() ran — no JS (or provider) fallback could build from what's - // on disk — cannot silently complete open(): search would then serve - // empty results with no signal, exactly the defect this gate closes. - // FAIL TYPED, pre-serve, rather than let a broken vector leg pass as a - // successful open. - if (vectorCoverageGap && this.index.size() === 0) { - throw new VectorIndexNotReadyError( - `open(): the canonical ledger holds ${vectorLedgerAll} vectored noun(s) but the vector ` + - `index still reports 0 node(s) after rebuild() — the derived vector index could not ` + - `be restored from canonical. Refusing to serve silent-empty search results; ` + - `investigate the vector provider/storage, or repairIndex({ rebuild: ['vector'] }) ` + - `after restoring the underlying data.` - ) - } - // 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes // have now rebuilt and verified, so they match this build's epoch — // re-stamp the marker LAST, only here. A crash anywhere above leaves the diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 8112afd2..cf4a29e0 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -801,17 +801,6 @@ export interface DerivedFamilyDeclaration { export interface CanonicalCounts { nouns: { counted: number; all: number } verbs: { counted: number; all: number } - /** - * The count of canonical nouns holding a REAL (non-empty) vector — the - * coverage denominator a vector index's node-count ledger is measured - * against (`nodeCount === vectors.all` is the whole-store coverage - * verdict for the vector leg, the vector-side mirror of `nouns.all` for - * metadata/graph). A deferred-embed noun (`add({ deferEmbedding: true })`) - * counts only once its vector actually LANDS — its canonical record exists - * (counted in `nouns.all`) with an empty vector until then, so it is - * deliberately NOT counted here in the interim. - */ - vectors: { all: number } /** An unprovable delete has left the `all` scalars unverified since the last recount. */ suspect: boolean } @@ -830,47 +819,14 @@ export interface StorageAdapter { * Save noun metadata separately * @param id Noun ID * @param metadata Noun metadata - * @param hasVector - OPTIONAL vectored-noun ledger hint: `true` when this - * write is a FRESH insert (`isNew`) whose vector is a real, non-empty - * array — the caller already knows this for free (the insert's own - * `vector` local), so the increment rides the SAME isNew gate that - * already protects `totalNounCountAll` from double-counting on HNSW - * neighbor-link re-saves (`saveNoun_internal` re-runs on every link - * change; this metadata seam does not). Absent/`false` ⇒ no ledger - * action. A deferred-embed insert passes `false` (its vector lands - * later — see {@link StorageAdapter.noteVectorLanded}). */ - saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise + saveNounMetadata(id: string, metadata: NounMetadata): Promise /** * Delete noun metadata * @param id Noun ID - * @param priorRecord - OPTIONAL already-known metadata (the caller's - * pre-delete read) — see {@link StorageAdapter.deleteNoun}. - * @param hadVector - OPTIONAL vectored-noun ledger hint: `true`/`false` - * when the caller already knows (read as a side effect of ITS OWN delete - * flow — e.g. `remove()`'s pre-read for the vector-index removal — never - * a read added FOR this ledger), `undefined` when genuinely unknown. A - * known `true` decrements the vectored-noun ledger; a known `false` is a - * no-op (it was never counted); `undefined` marks the ledger SUSPECT - * rather than guessing — the delete path must never add a canonical read - * to answer this question. */ - deleteNounMetadata(id: string, priorRecord?: NounMetadata | null, hadVector?: boolean): Promise - - /** - * OPTIONAL narrow ledger hook: record that a canonical noun's vector just - * LANDED for the first time. Exists ONLY for the deferred-embedding - * lifecycle — the landing commit (`system:embed-landing`) carries a vector - * write with no accompanying metadata operation, so the normal - * `saveNounMetadata(..., hasVector)` seam never fires for it. Callers MUST - * call this only when the noun held NO real vector before this write (the - * deferred-embed worker already holds that fact for free, from its own - * pre-embed read — never an added read). A backend without vectored-noun - * tracking is a no-op via this method's absence (feature-detected). - * @param id - The noun whose vector just landed. - */ - noteVectorLanded?(id: string): Promise + deleteNounMetadata(id: string): Promise /** * Get noun with metadata combined @@ -919,11 +875,8 @@ export interface StorageAdapter { * REQUIRE re-reading the record being removed: when the internal read * returns `null` (replace race, or a ghost left by an earlier version) the * decrement falls back to this record instead of being silently skipped. - * @param hadVector OPTIONAL vectored-noun ledger hint — see - * {@link StorageAdapter.deleteNounMetadata}'s `hadVector` param, which - * this forwards to unchanged. */ - deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise + deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise /** * Save verb - Pure HNSW verb with core fields only diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 22bf1366..7c080b4c 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1041,27 +1041,12 @@ export abstract class BaseStorageAdapter implements StorageAdapter { */ protected totalNounCountAll = 0 protected totalVerbCountAll = 0 - /** - * The count of canonical nouns holding a REAL (non-empty) vector — the - * vector-side mirror of `totalNounCountAll` and the coverage denominator a - * vector index's node-count ledger is measured against. A deferred-embed - * noun (`add({ deferEmbedding: true })`) counts only once its vector - * LANDS (the `system:embed-landing` commit) — its canonical record exists - * (already counted in `totalNounCountAll`) with an empty vector until - * then. Maintained on the write path (a fresh insert whose vector is - * non-empty +1, a deferred embed's landing +1, a PROVEN delete of a - * vectored noun −1), persisted beside the other ALL scalars, recomputed by - * the sanctioned recount. Shares `allCountsSuspect` — no separate flag. - */ - protected totalVectoredNounCount = 0 /** * `true` when a delete could not prove whether the record existed (no * canonical read, no caller-provided prior) — the ALL scalar may be off by * the unprovable deletes since. Loud, persisted, and cleared only by the * sanctioned recount; a consumer reading the scalar as a ledger denominator - * must treat a suspect scalar as unverified, never as exact. Also covers - * `totalVectoredNounCount` — a delete whose vector-presence fact was - * unknowable marks this SAME flag rather than minting a second one. + * must treat a suspect scalar as unverified, never as exact. */ protected allCountsSuspect = false /** One narration per session for the suspect transition (never per delete). */ @@ -1098,19 +1083,15 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * The canonical count ledger — O(1), no I/O. `counted` is the user-facing * scalar (public/internal tiers, what `getNounCount()` returns); `all` is * the ALL-visibility scalar every unfiltered storage walk is measured - * against (the coverage-ledger denominator for derived-index providers); - * `vectors.all` is the vectored-noun scalar — the coverage denominator for - * a vector index's node-count ledger specifically. - * `suspect` is `true` when an unprovable delete has made `all` (any - * family, including `vectors`) unverified since the last sanctioned - * recount (`rebuildTypeCounts`). - * @returns All scalars per family plus the suspect flag. + * against (the coverage-ledger denominator for derived-index providers). + * `suspect` is `true` when an unprovable delete has made `all` unverified + * since the last sanctioned recount (`rebuildTypeCounts`). + * @returns Both scalars per family plus the suspect flag. */ async getCanonicalCounts(): Promise { return { nouns: { counted: this.totalNounCount, all: this.totalNounCountAll }, verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll }, - vectors: { all: this.totalVectoredNounCount }, suspect: this.allCountsSuspect } } @@ -1122,7 +1103,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter { * @param family - Which family's delete was unprovable. * @param id - The id whose existence could not be established. */ - protected markAllCountsSuspect(family: 'noun' | 'verb' | 'noun-vector', id: string): void { + protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void { this.allCountsSuspect = true if (!this.allCountsSuspectNarrated) { this.allCountsSuspectNarrated = true @@ -1135,23 +1116,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } } - /** - * OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorLanded}): - * record a deferred-embed noun's FIRST real vector landing. The caller - * (the deferred-embed worker) proves this is a genuine landing — not a - * re-embed of an already-vectored row — by observing its own pre-embed - * read's vector was empty, at no added storage cost. - * @param id - The noun whose vector just landed (retained for a future - * narration seam; the count itself needs no id-keyed state). - */ - async noteVectorLanded(id: string): Promise { - void id - this.totalVectoredNounCount++ - this.scheduleCountPersist().catch(() => { - // Ignore persist errors — the in-memory count is authoritative; a later op retries. - }) - } - /** * Increment count for entity type - O(1) operation. * Concurrency is handled by the process-global mutex diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index fd9dbb4c..965e97b9 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -2583,7 +2583,6 @@ export class FileSystemStorage extends BaseStorage { // record reads), persist, and never scan again. Absent keys are a // legacy file, not a zero — a zero here would make every provider's // coverage ledger read "over-posted" on a populated store. - let needsPersist = false if ( typeof counts.totalNounCountAll === 'number' && typeof counts.totalVerbCountAll === 'number' @@ -2602,29 +2601,6 @@ export class FileSystemStorage extends BaseStorage { `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + `every tier) and persisted; no further scan.` ) - needsPersist = true - } - - // The vectored-noun scalar (shipped after the ALL scalars above — a - // counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but - // still predate THIS key). Unlike the ALL scalars, presence cannot be - // decided from the id-directory listing alone: a deferred-embed - // noun's `vectors.json` EXISTS with an empty `vector: []` until its - // embed lands, so this derivation reads every noun's `vectors.json` - // ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost. - if (typeof counts.totalVectoredNounCount === 'number') { - this.totalVectoredNounCount = counts.totalVectoredNounCount - } else { - const vectored = await this.scanVectoredNounCount() - this.totalVectoredNounCount = vectored - console.warn( - `[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` + - `derived once by reading every noun's vectors.json (${vectored} vectored) and ` + - `persisted; no further scan.` - ) - needsPersist = true - } - if (needsPersist) { await this.persistCounts() } @@ -2667,12 +2643,6 @@ export class FileSystemStorage extends BaseStorage { this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false - // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT - // (a deferred-embed noun's file exists but holds an empty vector until - // its embed lands), so this is a full O(nouns) content scan — see - // scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here, - // alongside the rest of this from-disk recovery. - this.totalVectoredNounCount = await this.scanVectoredNounCount() // Sample some entities for the type distribution (don't read all). // Read the metadata files DIRECTLY with fs — this runs inside init(), @@ -2758,61 +2728,6 @@ export class FileSystemStorage extends BaseStorage { } } - /** - * Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs - * — the vector-side mirror of {@link readEntityMetadataRaw}, same - * reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`). - * @param entityDir - Absolute `entities/nouns//` directory. - * @returns The parsed vector record, or null when absent/unreadable. - */ - private async readEntityVectorRaw(entityDir: string): Promise { - const base = path.join(entityDir, 'vectors.json') - try { - return JSON.parse(await fs.promises.readFile(base, 'utf-8')) - } catch { - // fall through to the compressed variant - } - try { - const gz = await fs.promises.readFile(`${base}.gz`) - return JSON.parse(zlib.gunzipSync(gz).toString('utf-8')) - } catch { - return null - } - } - - /** - * Count canonical nouns holding a REAL (non-empty) vector — the vectored- - * noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence - * cannot be decided from the id-directory listing alone: a deferred-embed - * noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`) - * until its embed LANDS, so this walk reads every noun's `vectors.json` - * CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time - * legacy-counts.json derivation or a lost/corrupted counts.json recovery; - * the result is persisted so this scan never repeats. - */ - private async scanVectoredNounCount(): Promise { - const base = path.join(this.rootDir, 'entities', 'nouns') - let vectored = 0 - try { - const shards = await fs.promises.readdir(base, { withFileTypes: true }) - for (const shard of shards) { - if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue - const shardPath = path.join(base, shard.name) - const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) - for (const entry of ids) { - if (!entry.isDirectory()) continue - const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name)) - if (record && Array.isArray(record.vector) && record.vector.length > 0) { - vectored++ - } - } - } - } catch (error: any) { - if (error?.code !== 'ENOENT') throw error - } - return vectored - } - /** * Persist counts to filesystem storage */ @@ -2829,10 +2744,6 @@ export class FileSystemStorage extends BaseStorage { // written before the ledger existed; initializeCounts() derives them once. totalNounCountAll: this.totalNounCountAll, totalVerbCountAll: this.totalVerbCountAll, - // Vectored-noun ledger scalar — absent in files written before it - // existed; initializeCounts() derives it once (a content scan, see - // scanVectoredNounCount()'s JSDoc). - totalVectoredNounCount: this.totalVectoredNounCount, allCountsSuspect: this.allCountsSuspect, lastUpdated: new Date().toISOString() } diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index ab2a52d3..f55a5626 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -520,12 +520,6 @@ export class MemoryStorage extends BaseStorage { let totalNouns = 0 let totalVerbs = 0 - // Vectored-noun scalar: unlike the bare presence check above, this needs - // the vectors.json RECORD'S content — a deferred-embed noun's record - // exists with an empty `vector: []` until its embed lands. In-memory this - // is a free field access (no I/O), unlike the filesystem adapter's - // per-noun disk read. - let totalVectoredNouns = 0 // Scan all paths in objectStore for (const path of this.objectStore.keys()) { @@ -534,10 +528,6 @@ export class MemoryStorage extends BaseStorage { if (nounMatch) { // Type is in metadata, not path - just count total totalNouns++ - const record = this.objectStore.get(path) as { vector?: unknown } | undefined - if (Array.isArray(record?.vector) && record.vector.length > 0) { - totalVectoredNouns++ - } } // Count verbs (entities/verbs/{shard}/{id}/vectors.json) @@ -553,7 +543,6 @@ export class MemoryStorage extends BaseStorage { // A scan of every canonical record IS the ALL-visibility count. this.totalNounCountAll = totalNouns this.totalVerbCountAll = totalVerbs - this.totalVectoredNounCount = totalVectoredNouns this.allCountsSuspect = false } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index d6ccc5fa..6f4c7f0e 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1762,10 +1762,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Delete a noun from storage - * @param hadVector - OPTIONAL vectored-noun ledger hint, forwarded to - * {@link deleteNounMetadata} unchanged — see its JSDoc. */ - public async deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise { + public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise { await this.ensureInitialized() // FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the @@ -1782,7 +1780,7 @@ export abstract class BaseStorage extends BaseStorageAdapter { // LONGER wrapped in a blind catch that masked faults as "file didn't exist". // `priorMetadata` (the caller's pre-delete read) keeps the decrement honest // even when the canonical read inside returns null (replace race / ghost). - await this.deleteNounMetadata(id, priorMetadata, hadVector) + await this.deleteNounMetadata(id, priorMetadata) // Remove the now-empty entity container (a no-op for key/prefix stores). await this.removeCanonicalContainer(getNounVectorPath(id)) @@ -3394,12 +3392,11 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Save noun metadata to storage (now typed) * Routes to correct sharded location based on UUID - * @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc. */ - public async saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise { + public async saveNounMetadata(id: string, metadata: NounMetadata): Promise { // Validate noun type in metadata - storage boundary protection validateNounType(metadata.noun) - return this.saveNounMetadata_internal(id, metadata, hasVector) + return this.saveNounMetadata_internal(id, metadata) } /** @@ -3410,10 +3407,9 @@ export abstract class BaseStorage extends BaseStorageAdapter { * This ensures counts are updated AFTER metadata exists, fixing the race condition * where storage adapters tried to read metadata before it was saved. * - * @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc. * @protected */ - protected async saveNounMetadata_internal(id: string, metadata: NounMetadata, hasVector?: boolean): Promise { + protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise { await this.ensureInitialized() // ID-first path - no type needed! @@ -3469,14 +3465,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { // record persists here so the ALL scalar never lags the tree. if (isNew) { this.totalNounCountAll++ - // Vectored-noun ledger: rides the SAME isNew gate (once per id, at - // creation) — this seam is metadata-write-driven and never re-runs on - // the HNSW neighbor-link re-saves that hit saveNoun_internal, so it - // cannot double-count. A deferred-embed insert passes hasVector=false - // (or omits it); its vector lands later via noteVectorLanded(). - if (hasVector) { - this.totalVectoredNounCount++ - } if (!(metadata.noun && isCounted)) { this.scheduleCountPersist().catch(() => { // Ignore persist errors — the in-memory count is authoritative; a later op retries. @@ -3872,18 +3860,8 @@ export abstract class BaseStorage extends BaseStorageAdapter { * the skip permanently inflated the persisted totals (adds counted, paired * removals not decremented), and `Math.max(totalNounCount, scanned)` made * the inflation unfixable by any disk cleanup. - * @param hadVector - OPTIONAL vectored-noun ledger hint — see - * {@link StorageAdapter.deleteNounMetadata}'s JSDoc. This method never - * reads `vectors.json` to answer the question itself (a canonical read - * the delete path must never add); a caller that cannot supply the fact - * for free leaves it `undefined`, and the ledger goes SUSPECT rather - * than guessing. */ - public async deleteNounMetadata( - id: string, - priorRecord?: NounMetadata | null, - hadVector?: boolean - ): Promise { + public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise { await this.ensureInitialized() // Direct O(1) delete with ID-first path. Read the canonical record BEFORE @@ -3907,15 +3885,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { } else { this.markAllCountsSuspect('noun', id) } - - // Vectored-noun ledger: a KNOWN vector fact decrements (or no-ops); - // an UNKNOWN one goes suspect rather than guessing — see @param hadVector. - if (hadVector === true) { - if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount-- - else this.markAllCountsSuspect('noun-vector', id) - } else if (hadVector === undefined) { - this.markAllCountsSuspect('noun-vector', id) - } this.scheduleCountPersist().catch(() => { // Ignore persist errors — the in-memory count is authoritative; a later op retries. }) @@ -4580,16 +4549,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { // construction. let allNouns = 0 let allVerbs = 0 - // Vectored-noun scalar: unlike `allNouns` (decided from the metadata.json - // LISTING alone), presence cannot be decided from the vectors.json - // listing alone — a deferred-embed noun's vectors.json EXISTS with an - // empty `vector: []` until its embed lands, so the file's CONTENT must be - // read. This walk already lists every path per shard (including - // vectors.json entries — `listCanonicalObjects` yields both legs), so - // reading them here costs one EXTRA read per noun beyond the metadata.json - // read above (doubling this walk's per-noun I/O) — honest cost, paid only - // by this diagnostic/repair recount, never on the hot path. - let allVectoredNouns = 0 // Scan noun shards for (let shard = 0; shard < 256; shard++) { @@ -4600,18 +4559,6 @@ export abstract class BaseStorage extends BaseStorageAdapter { const paths = await this.listCanonicalObjects(shardDir) for (const path of paths) { - if (path.includes('/vectors.json')) { - try { - const vectorRecord = await this.readCanonicalObject(path) - if (vectorRecord && Array.isArray(vectorRecord.vector) && vectorRecord.vector.length > 0) { - allVectoredNouns++ - } - } catch (error) { - // Skip vector records that fail to load — best-effort ground truth, - // same as the metadata read below. - } - continue - } if (!path.includes('/metadata.json')) continue allNouns++ @@ -4687,19 +4634,17 @@ export abstract class BaseStorage extends BaseStorageAdapter { // IS the proof an unprovable delete could not give. const nounsAllBefore = this.totalNounCountAll const verbsAllBefore = this.totalVerbCountAll - const vectoredBefore = this.totalVectoredNounCount this.totalNounCountAll = allNouns this.totalVerbCountAll = allVerbs - this.totalVectoredNounCount = allVectoredNouns this.allCountsSuspect = false this.countCache.clear() await this.persistCounts() prodLog.info( `[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` + - `ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs / ${allVectoredNouns} vectored nouns` + - (nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs || vectoredBefore !== allVectoredNouns - ? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore} / ${vectoredBefore})` + `ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` + + (nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs + ? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})` : ' (unchanged)') + ` — scalar + per-type persisted` ) diff --git a/src/transaction/operations/StorageOperations.ts b/src/transaction/operations/StorageOperations.ts index 8b2ebffe..c1e9f1c1 100644 --- a/src/transaction/operations/StorageOperations.ts +++ b/src/transaction/operations/StorageOperations.ts @@ -52,15 +52,7 @@ export class SaveNounMetadataOperation implements Operation { private readonly storage: StorageAdapter, private readonly id: string, private readonly metadata: NounMetadata, - private readonly isNew: boolean = false, - /** - * OPTIONAL vectored-noun ledger hint: `true` when this write's paired - * vector (the SAME insert's `vector` local) is real/non-empty — see - * {@link StorageAdapter.saveNounMetadata}'s JSDoc for the isNew-gated, - * double-count-proof seam this rides. Default `false`: a deferred-embed - * insert (or any caller that doesn't know) never counts here. - */ - private readonly hasVector: boolean = false + private readonly isNew: boolean = false ) {} async execute(): Promise { @@ -70,7 +62,7 @@ export class SaveNounMetadataOperation implements Operation { : await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata') // Save new metadata - await this.storage.saveNounMetadata(this.id, this.metadata, this.hasVector) + await this.storage.saveNounMetadata(this.id, this.metadata) // Return rollback action return async () => { @@ -78,10 +70,8 @@ export class SaveNounMetadataOperation implements Operation { // Restore previous metadata await this.storage.saveNounMetadata(this.id, previousMetadata) } else { - // Delete newly created metadata. `this.hasVector` is the SAME fact - // this operation's own execute() used to (maybe) count the vectored - // ledger — reversing with it on rollback needs no new read. - await this.storage.deleteNounMetadata(this.id, undefined, this.hasVector) + // Delete newly created metadata + await this.storage.deleteNounMetadata(this.id) } } } @@ -150,9 +140,7 @@ export class SaveNounOperation implements Operation { // Note: Not all adapters implement deleteNoun // This is acceptable - metadata deletion makes entity invisible if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') { - // `this.noun.vector` is the SAME record just written — the - // vectored-noun ledger fact is free (no added read) and exact. - await this.storage.deleteNoun(this.noun.id, undefined, this.noun.vector.length > 0) + await this.storage.deleteNoun(this.noun.id) } } } @@ -210,21 +198,14 @@ export class DeleteNounMetadataOperation implements Operation { return async () => {} } - // Vectored-noun ledger fact: `previousNoun` is already read above for the - // before-image capture — no added read. `undefined` (noun genuinely - // absent, metadata-only ghost) is passed through honestly; the storage - // layer marks the ledger suspect rather than guessing. - const hadVector = previousNoun ? previousNoun.vector.length > 0 : undefined - // Full removal: both canonical legs + the entity container + count decrement // (the prior record keeps the decrement honest on a null canonical read). - await this.storage.deleteNoun(this.id, previousMetadata, hadVector) + await this.storage.deleteNoun(this.id, previousMetadata) // Return rollback action return async () => { // Restore the vector leg, then the metadata leg through the count-aware - // save so deleteNoun()'s decrement is reversed (hadVector's mirror: - // re-increments the vectored ledger iff the restored vector is real). + // save so deleteNoun()'s decrement is reversed. if (previousNoun) { await this.storage.saveNoun({ id: previousNoun.id, @@ -234,7 +215,7 @@ export class DeleteNounMetadataOperation implements Operation { }) } if (previousMetadata) { - await this.storage.saveNounMetadata(this.id, previousMetadata, hadVector === true) + await this.storage.saveNounMetadata(this.id, previousMetadata) } } } diff --git a/tests/integration/canonical-count-ledger.test.ts b/tests/integration/canonical-count-ledger.test.ts index f3c8ce20..8d0f46b8 100644 --- a/tests/integration/canonical-count-ledger.test.ts +++ b/tests/integration/canonical-count-ledger.test.ts @@ -17,11 +17,10 @@ * (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is * upgraded from the canonical id tree at open, then persisted. */ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { describe, it, expect, beforeEach, afterEach } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' -import * as zlib from 'node:zlib' import { Brainy } from '../../src/index.js' /** Count canonical `/entities///` directories — every tier. */ @@ -181,176 +180,3 @@ describe('canonical count ledger — ALL-visibility scalars, unclamped totals, r expect(ledger.nouns.all).toBe(truth) }) }) - -/** Count `/entities/nouns///vectors.json[.gz]` files holding a non-empty `vector`. */ -function countVectoredNouns(root: string): number { - const base = path.join(root, 'entities', 'nouns') - if (!fs.existsSync(base)) return 0 - let n = 0 - for (const shard of fs.readdirSync(base)) { - const shardDir = path.join(base, shard) - if (!fs.statSync(shardDir).isDirectory()) continue - for (const id of fs.readdirSync(shardDir)) { - const idDir = path.join(shardDir, id) - if (!fs.statSync(idDir).isDirectory()) continue - const plainPath = path.join(idDir, 'vectors.json') - const gzPath = `${plainPath}.gz` - let record: any = null - if (fs.existsSync(plainPath)) { - record = JSON.parse(fs.readFileSync(plainPath, 'utf-8')) - } else if (fs.existsSync(gzPath)) { - record = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf-8')) - } else { - continue - } - if (Array.isArray(record.vector) && record.vector.length > 0) n++ - } - } - return n -} - -describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s coverage denominator)', () => { - let dir: string - let brain: any - - const open = async () => { - const b: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - silent: true, - dimensions: 384 - }) - await b.init() - return b - } - - /** Baseline vectored count right after a fresh open() — init() creates a - * hidden system VFS-root noun that itself carries a real vector, so a - * brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off - * this baseline rather than hardcoding it away. */ - let baseline: number - - beforeEach(async () => { - process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' - dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-')) - brain = await open() - baseline = (await brain.storage.getCanonicalCounts()).vectors.all - }) - afterEach(async () => { - vi.restoreAllMocks() - await brain.close?.().catch(() => {}) - fs.rmSync(dir, { recursive: true, force: true }) - }) - - it('an explicit-vector add counts immediately; the ledger matches the on-disk vectors.json content', async () => { - await brain.add({ data: 'a', type: 'document', vector: Array(384).fill(0).map((_, i) => Math.sin(i)) }) - await brain.add({ data: 'b', type: 'document' }) // embedded (non-deferred) — also a real vector - await brain.flush() - - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 2) - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - expect(ledger.suspect).toBe(false) - }) - - it('a deferred-embed add does NOT count until its embed LANDS', async () => { - // Hold the background worker's embed call open under manual control — a - // deterministic embedder is fast enough that the landing could otherwise - // race ahead of the "still unlanded" assertion below. - let resolveEmbed: ((v: number[]) => void) | undefined - vi.spyOn(brain, 'embed').mockImplementation( - () => new Promise((resolve) => { resolveEmbed = resolve }) - ) - - const id = await brain.add({ data: 'deferred content', type: 'document', deferEmbedding: true }) - await brain.flush() - - // Landed nothing yet — the ledger must not count the stub. - let ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline) - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - - // Release the held embed, then cross the barrier: the vector lands - // (system:embed-landing). - resolveEmbed!(Array(384).fill(0).map((_, i) => Math.cos(i))) - await brain.awaitPendingEmbeds() - const landed = await brain.get(id, { includeVectors: true }) - expect((landed!.vector as number[]).length).toBeGreaterThan(0) - - ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 1) - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - expect(ledger.suspect).toBe(false) - }) - - it('a proven delete of a vectored noun decrements; a non-vectored (unlanded) delete does not', async () => { - const vectoredId = await brain.add({ data: 'v', type: 'document' }) // real embed, unmocked - // Block the embed worker AFTER the real add above — a deterministic - // embedder is fast enough that the deferred noun below could otherwise - // land before this test observes its "still unlanded" state. - vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) - const deferredId = await brain.add({ data: 'd', type: 'document', deferEmbedding: true }) - await brain.flush() - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(baseline + 1) - - await brain.remove(vectoredId) - await brain.flush() - let ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline) - expect(ledger.suspect).toBe(false) - - await brain.remove(deferredId) // never had a real vector — no decrement, still unsuspect - await brain.flush() - ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline) - expect(ledger.suspect).toBe(false) - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - }) - - it('the recount corrects a tampered vectors.all scalar, surviving reopen', async () => { - await brain.add({ data: 'real 1', type: 'document' }) - await brain.add({ data: 'real 2', type: 'document' }) - await brain.flush() - const truth = countVectoredNouns(dir) - expect(truth).toBe(baseline + 2) - - ;(brain.storage as any).totalVectoredNounCount = truth + 40 - await (brain.storage as any).persistCounts() - await brain.close() - brain = await open() - - // The lie survives reopen (never clamped). - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth + 40) - - await brain.repairIndex() - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth) - - await brain.close() - brain = await open() - expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth) - }) - - it('a legacy counts.json without totalVectoredNounCount is derived once from vectors.json content and persisted', async () => { - await brain.add({ data: 'one', type: 'document' }) // real embed, unmocked - // Block the embed worker AFTER the real add above — a deterministic - // embedder is fast enough that the deferred noun below could otherwise - // land before close(), which would inflate this test's expected count. - vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) - await brain.add({ data: 'two deferred', type: 'document', deferEmbedding: true }) - await brain.flush() - await brain.close() - - const countsPath = path.join(dir, '_system', 'counts.json') - const raw = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) - expect(typeof raw.totalVectoredNounCount).toBe('number') - delete raw.totalVectoredNounCount - fs.writeFileSync(countsPath, JSON.stringify(raw, null, 2)) - - brain = await open() - const ledger = await brain.storage.getCanonicalCounts() - expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun - expect(ledger.vectors.all).toBe(countVectoredNouns(dir)) - const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8')) - expect(persisted.totalVectoredNounCount).toBe(baseline + 1) - }) -}) diff --git a/tests/integration/vector-leg-open-build.test.ts b/tests/integration/vector-leg-open-build.test.ts deleted file mode 100644 index a1ccaefa..00000000 --- a/tests/integration/vector-leg-open-build.test.ts +++ /dev/null @@ -1,247 +0,0 @@ -/** - * @module tests/integration/vector-leg-open-build - * @description THE LAST RED of the two-engine release gate: a migrated - * store can hold canonical vectored nouns with NO derived vector index - * built. `open()` owns building the derived indexes (reads never build — - * see `rebuildIndexesIfNeeded`'s JSDoc); the defect this pins is the vector - * leg's decision silently skipping that build, so `find`/search served `[]` - * with no error and no narration. - * - * A downstream deployment measures this through a native vector provider - * whose own health report can legitimately say `serving: true` even while - * vector COVERAGE is honestly unledgered on its side (an unledgered - * invariant never flips `serving` — see `HealthReport`'s derivation laws). - * This repo ships only the JS engine, so the reproduction here uses the - * SAME plugin seam a native provider would (`brain.use({ activate: ctx => - * ctx.registerProvider('vector', factory) })`, the pattern - * `tests/unit/cold-open-rebuild-gate.test.ts` already established for this - * exact class of gate-decision bug) with a stub that WRAPS the real - * `JsHnswVectorIndex` — every method delegates to a genuine engine (so a - * successful rebuild restores REAL, searchable vectors), except `size()` - * (fakes 0 until rebuild runs — the "never built" posture) and - * `healthReport()` (always reports `serving: true`, `unledgered: - * ['vector-coverage']` — the "I don't track this yet" posture). This is - * "as close as the JS engine allows": the gap is reproduced at the exact - * decision the fix changes, not approximated by deleting files the JS - * engine's own cold-start heuristic already recovers from unaided (see the - * inverse pin below and cold-open-rebuild-gate.test.ts's already-pinned - * "isReady()===true, size()===0" contract, which this fix deliberately does - * NOT touch — bare isReady() has no unledgered concept to hide behind, and - * overriding it would reopen the 48-seconds-per-restart regression pinned - * there). - * - * Pins: - * (1) COVERAGE GAP FORCES THE BUILD: N vectored nouns, a provider that - * claims `serving: true` at `size()===0` — open() builds anyway (the - * ledger proves there is something to cover), and search returns real - * results, never `[]`. - * (2) THE INVERSE, HONEST EMPTY: 0 vectored nouns (every embed still - * deferred/unlanded) — open() does NOT attempt a rebuild (nothing to - * load; the old blunt "always rebuild when size()===0" heuristic wasted - * a full canonical walk here for zero benefit), and search honestly - * returns `[]` — no error, no false coverage-gap narration. - * - * SEARCH VERIFICATION NOTE: pin (1) verifies "search returns real results" - * via `find({ query: })` (semantic search — embeds the query, then - * searches), matching the pattern `tests/integration/hnsw-rebuild.test.ts` - * already uses for exactly this "post-rebuild search works" class of pin. - * A raw `find({ vector: })` / `index.search(vector, k)` call was - * tried first and found to reproducibly return only 1 hit after a - * FROM-CANONICAL rebuild (never the full requested `limit`, sometimes not - * even a real neighbor) — REGARDLESS of this task's changes: it reproduces - * identically on a plain, unwrapped, un-stubbed reopen with the stock JS - * engine (verified against `hnsw-rebuild.test.ts`'s own construction) and - * is therefore a PRE-EXISTING, orthogonal defect in the JS HNSW engine's - * rebuilt-graph connectivity — outside this task's two deliverables (the - * count ledger and the open-gate REBUILD DECISION, not rebuild()'s internal - * search quality). Left for a separate investigation; not touched here. - */ -import { describe, it, expect, afterEach, vi } from 'vitest' -import * as fs from 'node:fs' -import * as os from 'node:os' -import * as path from 'node:path' -import { Brainy } from '../../src/index.js' -import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js' - -const tmpDirs: string[] = [] -function mkTmp(): string { - const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vector-leg-open-')) - tmpDirs.push(d) - return d -} -afterEach(() => { - vi.restoreAllMocks() - for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true }) -}) - -const V = (seed: number) => - Array.from({ length: 384 }, (_, i) => Math.sin((seed + 1) * 7919 + i * 131) * 0.5 + 0.5) - -/** - * Build a store with N explicit-vector (non-deferred) nouns, flush, close. - * Each noun also carries embeddable text (`technology`/`science`, matching - * the query used below) so the semantic-search verification exercises real - * retrieval, not a coincidental match. The default JS engine builds a fully - * current store — the epoch marker is stamped current at this open's - * completion, so a later reopen's `_indexEpochStale` is honestly false and - * cannot mask the ledger-gap decision under test (nothing here manufactures - * epoch drift). - */ -async function buildVectoredStore(dir: string, n: number): Promise { - const brain: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - plugins: [], - silent: true, - dimensions: 384 - }) - await brain.init() - const ids: string[] = [] - for (let i = 0; i < n; i++) { - ids.push( - await brain.add({ - data: `doc ${i} about ${i % 2 === 0 ? 'technology' : 'science'}`, - type: 'document', - vector: V(i) - }) - ) - } - await brain.flush() - await brain.close() - return ids -} - -describe('vector-leg open-build (two-engine gate, last red)', () => { - it('coverage gap: a provider reporting serving:true at size()===0 is overridden by the vectored-noun ledger — open() builds, search returns real results', async () => { - const dir = mkTmp() - const ids = await buildVectoredStore(dir, 12) - - // Wrap the REAL JS engine so a successful rebuild restores genuine, - // searchable vectors — only `size()` and `healthReport()` are faked, - // simulating a native provider that has never built its own coverage of - // an unledgered invariant. - const calls = { rebuild: 0 } - const brain: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - plugins: [], - silent: true, - dimensions: 384 - }) - brain.use({ - name: 'fake-native-vector-unledgered-coverage', - activate: async (ctx: any) => { - ctx.registerProvider('vector', (config: any, distance: any, options: any) => { - const real = new JsHnswVectorIndex(config, distance, options) - let rebuilt = false - const originalRebuild = real.rebuild.bind(real) - ;(real as any).rebuild = async (...args: any[]) => { - const r = await originalRebuild(...args) - calls.rebuild++ - rebuilt = true - return r - } - const originalSize = real.size.bind(real) - ;(real as any).size = () => (rebuilt ? originalSize() : 0) - ;(real as any).healthReport = () => ({ - provider: 'vector', - healthy: true, - serving: true, - invariants: [], - checkedAt: Date.now(), - durationMs: 0, - generation: 1, - unledgered: ['vector-coverage'] - }) - return real - }) - return true - } - }) - await brain.init() - - // WITHOUT any find() first: open() itself must have built the leg. - expect(calls.rebuild, 'open() forced the rebuild despite serving:true').toBe(1) - const status = await brain.getIndexStatus() - expect(status.hnswIndex.size).toBeGreaterThanOrEqual(ids.length) - - // Real, searchable results — never [] (see the module doc's SEARCH - // VERIFICATION NOTE for why this is a semantic `query`, not a raw - // `vector`, call). - const results = await brain.find({ query: 'technology document', limit: 5 }) - expect(results.length).toBeGreaterThan(0) - expect(results.length).not.toBe(0) - - await brain.close() - }) - - it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => { - // ARCHITECTURAL NOTE (found while building this pin): every brainy store - // carries ONE permanent, always-vectored noun beyond user data — the VFS - // root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`, - // src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero - // (but non-empty, length-384) vector on EVERY store's first open — never - // deferred (a deliberate WASM-cold-compile-avoidance fix, see that - // file's comment) — and VFS init unconditionally re-creates it if - // missing, before the rebuild gate ever runs. A literal "0 vectored - // nouns" store is therefore unreachable through the public API; a - // brand-new store's `vectors.all` floor is 1, not 0. This pin verifies - // the law the task names in the ACHIEVABLE form: nouns whose embed is - // still deferred/unlanded contribute NOTHING to the vectored-noun ledger - // — the coverage-gap comparison sees exactly the root (1), never - // root+deferred — and semantic search over deferred-only user content - // honestly returns `[]` (no error, no false "coverage restored" claim). - const dir = mkTmp() - - const build: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - plugins: [], - silent: true, - dimensions: 384 - }) - await build.init() - const rootOnlyLedger = await build.storage.getCanonicalCounts() - // Block the embedder permanently so every add below stays deferred and - // unlanded for the rest of this test (a fast deterministic embedder - // could otherwise land it before we ever observe the "still 0 extra" - // state). - vi.spyOn(build, 'embed').mockImplementation(() => new Promise(() => {})) - for (let i = 0; i < 5; i++) { - await build.add({ data: `deferred ${i}`, type: 'document', deferEmbedding: true }) - } - await build.flush() - const ledgerWithDeferred = await build.storage.getCanonicalCounts() - // The five deferred adds contributed ZERO to the vectored-noun ledger. - expect(ledgerWithDeferred.vectors.all).toBe(rootOnlyLedger.vectors.all) - await build.close() - - // Reopen (default JS engine — no stub needed): the root is the ONLY - // thing the vector leg has to load; the deferred nouns are correctly - // invisible to it. Block the embedder again BEFORE init() — reopen - // recovers the durable pending-embed markers and kicks the worker as - // part of init() itself, and an unblocked deterministic embedder could - // land all five before this test observes the open-time ledger. - const brain: any = new Brainy({ - requireSubtype: false, - storage: { type: 'filesystem', path: dir }, - plugins: [], - silent: true, - dimensions: 384 - }) - vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {})) - await brain.init() - // The ledger is exactly the root — the five deferred, still-unlanded - // nouns (which the rebuild above DOES insert into the graph, each with - // its stub empty vector — `index.size()` counts EVERY canonical noun's - // graph node, deferred or not, so it is not the coverage metric) never - // inflate the VECTORED count. - const ledgerAfterReopen = await brain.storage.getCanonicalCounts() - expect(ledgerAfterReopen.vectors.all).toBe(rootOnlyLedger.vectors.all) - - const results = await brain.find({ vector: V(3), limit: 5 }) - expect(results).toEqual([]) - - await brain.close() - }) -}) diff --git a/tests/lifecycle/biography.test.ts b/tests/lifecycle/biography.test.ts index 8b274fce..9fda62a5 100644 --- a/tests/lifecycle/biography.test.ts +++ b/tests/lifecycle/biography.test.ts @@ -380,12 +380,6 @@ describe.sequential('lifecycle — the working store', () => { counted: aliveVerbs + model.vfsContainsVerbs, all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs }, - // Every noun this biography ever adds carries an explicit/computed - // vector (the harness never defers an embed), so the vectored-noun - // scalar tracks nouns.all exactly. - vectors: { - all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns - }, suspect: false }) } finally {