feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg
The coverage denominator the health-by-accounting ratification named for
the vector family — never built until now, and its absence was measured as
the exact outage class it existed to prevent: a migrated store with
canonical vectors and no derived index opened with the vector leg EMPTY,
served [] from vector search with no error, and the report-driven read gate
had nothing to refuse on (the provider's coverage invariant was honestly
unledgered — the denominator was ours to supply).
- getCanonicalCounts() gains vectors: { all } — the count of canonical
nouns holding a REAL vector. Incremented where a vector lands (the
isNew-gated metadata seam for explicit vectors — the same discipline that
keeps HNSW neighbor-link re-saves from inflating counts; a narrow
noteVectorLanded hook for the deferred-embed landing, gated on the
worker's own pre-embed read). Decremented on a proven delete of a
vectored noun; a vector-uncertain delete marks the ledger suspect rather
than guessing (no new reads on the delete path). Recounted by the
sanctioned recount; legacy counts.json derives it once (a deferred noun's
vector file exists with an empty vector, so presence requires one
content read at derivation — never on the hot path).
- The open gate's vector leg: when a health-reporting provider claims
serving while the index holds zero nodes and the ledger proves vectored
canonical rows exist, open BUILDS (narrated) — routed through the
provider's idempotent fillFromCanonical() when exposed (the joint door;
a partial shortfall stays repair()'s operator business), the JS rebuild
otherwise — or fails typed pre-serve. Scoped exactly: bare isReady()
providers, migrating providers, and white-box size stubs open as before.
Pinned end-to-end from the partner gate's probe shape (store with vectored
canonical rows, no derived index, reopen → search serves N, never []),
red-proved against the pre-fix path; the inverse (zero vectored rows) opens
without building and serves [] honestly.
This commit is contained in:
parent
bce2593e24
commit
9730835bdf
10 changed files with 851 additions and 32 deletions
145
src/brainy.ts
145
src/brainy.ts
|
|
@ -2382,6 +2382,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
[{ type: 'embed.landed', id, vector: newVector }],
|
||||
'system:embed-landing'
|
||||
)
|
||||
// Vectored-noun ledger: the landing commit above carries a vector
|
||||
// write with NO accompanying metadata operation, so the
|
||||
// saveNounMetadata(..., hasVector) seam never fires for it — the
|
||||
// narrow storage hook is the only seam left. `oldVector.length===0`
|
||||
// (already known for free from the pre-embed read above) proves this
|
||||
// is a GENUINE first landing, not a re-embed of an already-vectored
|
||||
// row (e.g. a deferred update() on a row that already had a real
|
||||
// vector) — the latter must never double-count.
|
||||
if (oldVector.length === 0) {
|
||||
await this.storage.noteVectorLanded?.(id)
|
||||
}
|
||||
this.clearPendingEmbed(id)
|
||||
} catch (err) {
|
||||
prodLog.warn(
|
||||
|
|
@ -3000,8 +3011,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const runInsert: TransactionFunction<void> = async (tx) => {
|
||||
// Operation 1: Save metadata FIRST (TypeAwareStorage caching)
|
||||
// isNew=true: skip pre-read for rollback (entity doesn't exist yet)
|
||||
// hasVector: the vectored-noun ledger counts this insert iff its
|
||||
// vector is real/non-empty (never true for a deferred embed, whose
|
||||
// stub `vector` is `[]` — it counts later, at landing).
|
||||
tx.addOperation(
|
||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, true)
|
||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, true, vector.length > 0)
|
||||
)
|
||||
|
||||
// Operation 2: Save vector data
|
||||
|
|
@ -10568,7 +10582,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
plan.postCommit.push(() => this.kickEmbedWorker())
|
||||
}
|
||||
plan.operations.push(
|
||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew),
|
||||
// hasVector: see the single-add() insert path's comment — never true
|
||||
// for a deferred embed (stub vector `[]`; counted later at landing).
|
||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0),
|
||||
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
|
||||
...(deferringEmbed
|
||||
? []
|
||||
|
|
@ -16900,9 +16916,92 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
const shouldRebuildMetadata =
|
||||
!metadataMigrating &&
|
||||
(epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0))
|
||||
const shouldRebuildVector =
|
||||
|
||||
// VECTOR LEG — the two-engine gate's last red: a migrated 7.x-era store
|
||||
// can hold canonical vectored nouns with NO derived vector index built.
|
||||
// `legNeedsRebuild`'s size-heuristic fallback (below) only fires off
|
||||
// `hnswIndexSize === 0`, and its health-report branch trusts a
|
||||
// provider's own `serving` verdict verbatim — but a provider's health
|
||||
// report can legitimately say `serving: true` while vector coverage is
|
||||
// honestly UNLEDGERED on ITS side too (an unledgered invariant never
|
||||
// flips serving), so neither signal alone can tell "genuinely empty"
|
||||
// apart from "never built". The canonical vectored-noun ledger
|
||||
// (`getCanonicalCounts().vectors.all` — Deliverable 1) is the
|
||||
// denominator that CAN tell them apart, and is compared here:
|
||||
// - a CONFIDENT (non-suspect) ledger `> 0` while the reported node
|
||||
// count is 0 is a proven coverage gap — force the build regardless
|
||||
// of what a health report claims;
|
||||
// - a CONFIDENT ledger `=== 0` while the node count is 0 proves there
|
||||
// is nothing to load (e.g. every noun's embed is still deferred) —
|
||||
// skip the size-heuristic fallback's blunt "always rebuild when
|
||||
// empty" trigger, which otherwise wastes a full canonical walk for
|
||||
// zero benefit on every cold open of such a store;
|
||||
// - an unavailable/suspect ledger changes nothing — loud errors never
|
||||
// quiet losses, so a doubtful ledger must never suppress a rebuild
|
||||
// the old heuristic would have run.
|
||||
// The bare `isReady()` boolean (no report, no `unledgered` concept) is
|
||||
// NOT overridden — that signal is what fixed the 48-seconds-per-restart
|
||||
// regression pinned in tests/unit/cold-open-rebuild-gate.test.ts (a
|
||||
// disk-native provider legitimately reporting 0 resident while durable
|
||||
// on disk), and re-deriving it from a denominator the provider itself
|
||||
// has no way to consult would reopen exactly that regression.
|
||||
const vectorAssessment = assessProviderHealth(this.index)
|
||||
const vectorLedger = await this.storage.getCanonicalCounts?.()
|
||||
const vectorLedgerAll = vectorLedger?.vectors.all
|
||||
const vectorLedgerConfident = vectorLedger !== undefined && !vectorLedger.suspect
|
||||
const vectorHasCoverageProof = vectorLedgerConfident && (vectorLedgerAll as number) > 0
|
||||
const vectorConfirmedEmpty = vectorLedgerConfident && vectorLedgerAll === 0
|
||||
|
||||
let vectorNeedsRebuild: boolean
|
||||
if (vectorAssessment.via === 'is-ready') {
|
||||
// Bare isReady() stays authoritative and UNMODIFIED — see above.
|
||||
vectorNeedsRebuild = vectorAssessment.readiness === 'not-ready'
|
||||
} else if (vectorAssessment.via === 'health-report') {
|
||||
vectorNeedsRebuild =
|
||||
vectorAssessment.readiness !== 'ready' ||
|
||||
(hnswIndexSize === 0 && vectorHasCoverageProof)
|
||||
} else {
|
||||
// size-heuristic / no provider (the built-in JS engine's own posture).
|
||||
vectorNeedsRebuild = hnswIndexSize === 0 && !vectorConfirmedEmpty
|
||||
}
|
||||
|
||||
const shouldRebuildVector = !vectorMigrating && (epochStale || vectorNeedsRebuild)
|
||||
|
||||
// Narration (and the FAIL-TYPED backstop below) are scoped EXACTLY to
|
||||
// the defect this gate closes: a provider whose OWN health report
|
||||
// claims `serving: true` — an affirmative "I am ready" a caller would
|
||||
// otherwise trust outright — while the canonical ledger proves vector
|
||||
// coverage is missing. This is deliberately NARROWER than "any branch
|
||||
// where the ledger contributed to the decision":
|
||||
// - the bare isReady() branch is untouched, as above (never in scope);
|
||||
// - the health-report branch's OWN `readiness !== 'ready'` case is
|
||||
// already an ordinary, PRE-EXISTING rebuild trigger (the provider
|
||||
// admits not-ready) — not a ledger override, so not a "gap";
|
||||
// - the size-heuristic/no-provider branch's rebuild-when-empty is the
|
||||
// SAME blunt trigger the code always had (`hnswIndexSize === 0`)
|
||||
// — the ledger only ever SUPPRESSES a rebuild there (the confirmed-
|
||||
// empty case), it never forces one the old heuristic wouldn't
|
||||
// already have run. Marking that branch a "gap" too made the
|
||||
// FAIL-TYPED backstop fire on ordinary white-box tests that stub
|
||||
// rebuild() as a no-op and pin `size()` at 0 to drive OTHER
|
||||
// assertions (e.g. migration-deference's isMigrating() coverage) —
|
||||
// those are not silent-empty defects, so they must open exactly as
|
||||
// before (tests/unit/brainy/migration-deference.test.ts).
|
||||
const vectorCoverageGap =
|
||||
!vectorMigrating &&
|
||||
(epochStale || legNeedsRebuild(this.index, hnswIndexSize === 0))
|
||||
vectorAssessment.via === 'health-report' &&
|
||||
vectorAssessment.readiness === 'ready' &&
|
||||
hnswIndexSize === 0 &&
|
||||
vectorHasCoverageProof
|
||||
if (vectorCoverageGap) {
|
||||
prodLog.warn(
|
||||
`[Brainy] open(): vector index reports ${hnswIndexSize} node(s) but the canonical ` +
|
||||
`ledger holds ${vectorLedgerAll} vectored noun(s) — the derived vector index is ` +
|
||||
`missing or unbuilt on this store. Forcing the vector rebuild rather than serving ` +
|
||||
`silent-empty search results.`
|
||||
)
|
||||
}
|
||||
|
||||
const shouldRebuildGraph =
|
||||
!graphMigrating &&
|
||||
(epochStale || legNeedsRebuild(this.graphIndex, false))
|
||||
|
|
@ -16955,9 +17054,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// provider running its own background migration is skipped here (it owns
|
||||
// its index until it verifies-and-swaps).
|
||||
const rebuildStartTime = Date.now()
|
||||
// The vector leg's build door, by contract with the native provider: a
|
||||
// provider exposing fillFromCanonical() gets THAT call — idempotent, the
|
||||
// provider's own init runs it first so this is the backstop — never a
|
||||
// full rebuild() for a coverage gap. A PARTIAL shortfall deliberately
|
||||
// triggers nothing here: that is repair()'s operator door. The JS index
|
||||
// has no fill door and keeps its rebuild.
|
||||
const vectorBuild = (): Promise<unknown> => {
|
||||
const fillDoor = (this.index as unknown as { fillFromCanonical?: () => Promise<unknown> })
|
||||
.fillFromCanonical
|
||||
if (vectorCoverageGap && typeof fillDoor === 'function') {
|
||||
prodLog.warn(
|
||||
`[Brainy] open(): vector coverage gap routes through the provider's ` +
|
||||
`fillFromCanonical() (idempotent canonical fill), not a full rebuild.`
|
||||
)
|
||||
return fillDoor.call(this.index)
|
||||
}
|
||||
return this.index.rebuild()
|
||||
}
|
||||
await Promise.all([
|
||||
shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(),
|
||||
shouldRebuildVector ? this.index.rebuild() : Promise.resolve(),
|
||||
shouldRebuildVector ? vectorBuild() : Promise.resolve(),
|
||||
shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve()
|
||||
])
|
||||
|
||||
|
|
@ -16998,6 +17115,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
|
||||
}
|
||||
|
||||
// Vector coverage verification: the coverage-gap rebuild above (see
|
||||
// `vectorCoverageGap`) MUST have actually restored the ledger's
|
||||
// vectored nouns. A provider that STILL reports 0 nodes after its own
|
||||
// rebuild() ran — no JS (or provider) fallback could build from what's
|
||||
// on disk — cannot silently complete open(): search would then serve
|
||||
// empty results with no signal, exactly the defect this gate closes.
|
||||
// FAIL TYPED, pre-serve, rather than let a broken vector leg pass as a
|
||||
// successful open.
|
||||
if (vectorCoverageGap && this.index.size() === 0) {
|
||||
throw new VectorIndexNotReadyError(
|
||||
`open(): the canonical ledger holds ${vectorLedgerAll} vectored noun(s) but the vector ` +
|
||||
`index still reports 0 node(s) after rebuild() — the derived vector index could not ` +
|
||||
`be restored from canonical. Refusing to serve silent-empty search results; ` +
|
||||
`investigate the vector provider/storage, or repairIndex({ rebuild: ['vector'] }) ` +
|
||||
`after restoring the underlying data.`
|
||||
)
|
||||
}
|
||||
|
||||
// 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
|
||||
// have now rebuilt and verified, so they match this build's epoch —
|
||||
// re-stamp the marker LAST, only here. A crash anywhere above leaves the
|
||||
|
|
|
|||
|
|
@ -801,6 +801,17 @@ export interface DerivedFamilyDeclaration {
|
|||
export interface CanonicalCounts {
|
||||
nouns: { counted: number; all: number }
|
||||
verbs: { counted: number; all: number }
|
||||
/**
|
||||
* The count of canonical nouns holding a REAL (non-empty) vector — the
|
||||
* coverage denominator a vector index's node-count ledger is measured
|
||||
* against (`nodeCount === vectors.all` is the whole-store coverage
|
||||
* verdict for the vector leg, the vector-side mirror of `nouns.all` for
|
||||
* metadata/graph). A deferred-embed noun (`add({ deferEmbedding: true })`)
|
||||
* counts only once its vector actually LANDS — its canonical record exists
|
||||
* (counted in `nouns.all`) with an empty vector until then, so it is
|
||||
* deliberately NOT counted here in the interim.
|
||||
*/
|
||||
vectors: { all: number }
|
||||
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
||||
suspect: boolean
|
||||
}
|
||||
|
|
@ -819,14 +830,47 @@ export interface StorageAdapter {
|
|||
* Save noun metadata separately
|
||||
* @param id Noun ID
|
||||
* @param metadata Noun metadata
|
||||
* @param hasVector - OPTIONAL vectored-noun ledger hint: `true` when this
|
||||
* write is a FRESH insert (`isNew`) whose vector is a real, non-empty
|
||||
* array — the caller already knows this for free (the insert's own
|
||||
* `vector` local), so the increment rides the SAME isNew gate that
|
||||
* already protects `totalNounCountAll` from double-counting on HNSW
|
||||
* neighbor-link re-saves (`saveNoun_internal` re-runs on every link
|
||||
* change; this metadata seam does not). Absent/`false` ⇒ no ledger
|
||||
* action. A deferred-embed insert passes `false` (its vector lands
|
||||
* later — see {@link StorageAdapter.noteVectorLanded}).
|
||||
*/
|
||||
saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>
|
||||
saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void>
|
||||
|
||||
/**
|
||||
* Delete noun metadata
|
||||
* @param id Noun ID
|
||||
* @param priorRecord - OPTIONAL already-known metadata (the caller's
|
||||
* pre-delete read) — see {@link StorageAdapter.deleteNoun}.
|
||||
* @param hadVector - OPTIONAL vectored-noun ledger hint: `true`/`false`
|
||||
* when the caller already knows (read as a side effect of ITS OWN delete
|
||||
* flow — e.g. `remove()`'s pre-read for the vector-index removal — never
|
||||
* a read added FOR this ledger), `undefined` when genuinely unknown. A
|
||||
* known `true` decrements the vectored-noun ledger; a known `false` is a
|
||||
* no-op (it was never counted); `undefined` marks the ledger SUSPECT
|
||||
* rather than guessing — the delete path must never add a canonical read
|
||||
* to answer this question.
|
||||
*/
|
||||
deleteNounMetadata(id: string): Promise<void>
|
||||
deleteNounMetadata(id: string, priorRecord?: NounMetadata | null, hadVector?: boolean): Promise<void>
|
||||
|
||||
/**
|
||||
* OPTIONAL narrow ledger hook: record that a canonical noun's vector just
|
||||
* LANDED for the first time. Exists ONLY for the deferred-embedding
|
||||
* lifecycle — the landing commit (`system:embed-landing`) carries a vector
|
||||
* write with no accompanying metadata operation, so the normal
|
||||
* `saveNounMetadata(..., hasVector)` seam never fires for it. Callers MUST
|
||||
* call this only when the noun held NO real vector before this write (the
|
||||
* deferred-embed worker already holds that fact for free, from its own
|
||||
* pre-embed read — never an added read). A backend without vectored-noun
|
||||
* tracking is a no-op via this method's absence (feature-detected).
|
||||
* @param id - The noun whose vector just landed.
|
||||
*/
|
||||
noteVectorLanded?(id: string): Promise<void>
|
||||
|
||||
/**
|
||||
* Get noun with metadata combined
|
||||
|
|
@ -875,8 +919,11 @@ export interface StorageAdapter {
|
|||
* REQUIRE re-reading the record being removed: when the internal read
|
||||
* returns `null` (replace race, or a ghost left by an earlier version) the
|
||||
* decrement falls back to this record instead of being silently skipped.
|
||||
* @param hadVector OPTIONAL vectored-noun ledger hint — see
|
||||
* {@link StorageAdapter.deleteNounMetadata}'s `hadVector` param, which
|
||||
* this forwards to unchanged.
|
||||
*/
|
||||
deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void>
|
||||
deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise<void>
|
||||
|
||||
/**
|
||||
* Save verb - Pure HNSW verb with core fields only
|
||||
|
|
|
|||
|
|
@ -1041,12 +1041,27 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
*/
|
||||
protected totalNounCountAll = 0
|
||||
protected totalVerbCountAll = 0
|
||||
/**
|
||||
* The count of canonical nouns holding a REAL (non-empty) vector — the
|
||||
* vector-side mirror of `totalNounCountAll` and the coverage denominator a
|
||||
* vector index's node-count ledger is measured against. A deferred-embed
|
||||
* noun (`add({ deferEmbedding: true })`) counts only once its vector
|
||||
* LANDS (the `system:embed-landing` commit) — its canonical record exists
|
||||
* (already counted in `totalNounCountAll`) with an empty vector until
|
||||
* then. Maintained on the write path (a fresh insert whose vector is
|
||||
* non-empty +1, a deferred embed's landing +1, a PROVEN delete of a
|
||||
* vectored noun −1), persisted beside the other ALL scalars, recomputed by
|
||||
* the sanctioned recount. Shares `allCountsSuspect` — no separate flag.
|
||||
*/
|
||||
protected totalVectoredNounCount = 0
|
||||
/**
|
||||
* `true` when a delete could not prove whether the record existed (no
|
||||
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
||||
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
||||
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
||||
* must treat a suspect scalar as unverified, never as exact.
|
||||
* must treat a suspect scalar as unverified, never as exact. Also covers
|
||||
* `totalVectoredNounCount` — a delete whose vector-presence fact was
|
||||
* unknowable marks this SAME flag rather than minting a second one.
|
||||
*/
|
||||
protected allCountsSuspect = false
|
||||
/** One narration per session for the suspect transition (never per delete). */
|
||||
|
|
@ -1083,15 +1098,19 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
||||
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||
* against (the coverage-ledger denominator for derived-index providers).
|
||||
* `suspect` is `true` when an unprovable delete has made `all` unverified
|
||||
* since the last sanctioned recount (`rebuildTypeCounts`).
|
||||
* @returns Both scalars per family plus the suspect flag.
|
||||
* against (the coverage-ledger denominator for derived-index providers);
|
||||
* `vectors.all` is the vectored-noun scalar — the coverage denominator for
|
||||
* a vector index's node-count ledger specifically.
|
||||
* `suspect` is `true` when an unprovable delete has made `all` (any
|
||||
* family, including `vectors`) unverified since the last sanctioned
|
||||
* recount (`rebuildTypeCounts`).
|
||||
* @returns All scalars per family plus the suspect flag.
|
||||
*/
|
||||
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
||||
return {
|
||||
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
||||
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
||||
vectors: { all: this.totalVectoredNounCount },
|
||||
suspect: this.allCountsSuspect
|
||||
}
|
||||
}
|
||||
|
|
@ -1103,7 +1122,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
* @param family - Which family's delete was unprovable.
|
||||
* @param id - The id whose existence could not be established.
|
||||
*/
|
||||
protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void {
|
||||
protected markAllCountsSuspect(family: 'noun' | 'verb' | 'noun-vector', id: string): void {
|
||||
this.allCountsSuspect = true
|
||||
if (!this.allCountsSuspectNarrated) {
|
||||
this.allCountsSuspectNarrated = true
|
||||
|
|
@ -1116,6 +1135,23 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorLanded}):
|
||||
* record a deferred-embed noun's FIRST real vector landing. The caller
|
||||
* (the deferred-embed worker) proves this is a genuine landing — not a
|
||||
* re-embed of an already-vectored row — by observing its own pre-embed
|
||||
* read's vector was empty, at no added storage cost.
|
||||
* @param id - The noun whose vector just landed (retained for a future
|
||||
* narration seam; the count itself needs no id-keyed state).
|
||||
*/
|
||||
async noteVectorLanded(id: string): Promise<void> {
|
||||
void id
|
||||
this.totalVectoredNounCount++
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment count for entity type - O(1) operation.
|
||||
* Concurrency is handled by the process-global mutex
|
||||
|
|
|
|||
|
|
@ -2583,6 +2583,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// record reads), persist, and never scan again. Absent keys are a
|
||||
// legacy file, not a zero — a zero here would make every provider's
|
||||
// coverage ledger read "over-posted" on a populated store.
|
||||
let needsPersist = false
|
||||
if (
|
||||
typeof counts.totalNounCountAll === 'number' &&
|
||||
typeof counts.totalVerbCountAll === 'number'
|
||||
|
|
@ -2601,6 +2602,29 @@ export class FileSystemStorage extends BaseStorage {
|
|||
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
||||
`every tier) and persisted; no further scan.`
|
||||
)
|
||||
needsPersist = true
|
||||
}
|
||||
|
||||
// The vectored-noun scalar (shipped after the ALL scalars above — a
|
||||
// counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but
|
||||
// still predate THIS key). Unlike the ALL scalars, presence cannot be
|
||||
// decided from the id-directory listing alone: a deferred-embed
|
||||
// noun's `vectors.json` EXISTS with an empty `vector: []` until its
|
||||
// embed lands, so this derivation reads every noun's `vectors.json`
|
||||
// ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost.
|
||||
if (typeof counts.totalVectoredNounCount === 'number') {
|
||||
this.totalVectoredNounCount = counts.totalVectoredNounCount
|
||||
} else {
|
||||
const vectored = await this.scanVectoredNounCount()
|
||||
this.totalVectoredNounCount = vectored
|
||||
console.warn(
|
||||
`[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` +
|
||||
`derived once by reading every noun's vectors.json (${vectored} vectored) and ` +
|
||||
`persisted; no further scan.`
|
||||
)
|
||||
needsPersist = true
|
||||
}
|
||||
if (needsPersist) {
|
||||
await this.persistCounts()
|
||||
}
|
||||
|
||||
|
|
@ -2643,6 +2667,12 @@ export class FileSystemStorage extends BaseStorage {
|
|||
this.totalNounCountAll = nouns.count
|
||||
this.totalVerbCountAll = verbs.count
|
||||
this.allCountsSuspect = false
|
||||
// Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
|
||||
// (a deferred-embed noun's file exists but holds an empty vector until
|
||||
// its embed lands), so this is a full O(nouns) content scan — see
|
||||
// scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here,
|
||||
// alongside the rest of this from-disk recovery.
|
||||
this.totalVectoredNounCount = await this.scanVectoredNounCount()
|
||||
|
||||
// Sample some entities for the type distribution (don't read all).
|
||||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||||
|
|
@ -2728,6 +2758,61 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs
|
||||
* — the vector-side mirror of {@link readEntityMetadataRaw}, same
|
||||
* reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`).
|
||||
* @param entityDir - Absolute `entities/nouns/<shard>/<id>` directory.
|
||||
* @returns The parsed vector record, or null when absent/unreadable.
|
||||
*/
|
||||
private async readEntityVectorRaw(entityDir: string): Promise<any | null> {
|
||||
const base = path.join(entityDir, 'vectors.json')
|
||||
try {
|
||||
return JSON.parse(await fs.promises.readFile(base, 'utf-8'))
|
||||
} catch {
|
||||
// fall through to the compressed variant
|
||||
}
|
||||
try {
|
||||
const gz = await fs.promises.readFile(`${base}.gz`)
|
||||
return JSON.parse(zlib.gunzipSync(gz).toString('utf-8'))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count canonical nouns holding a REAL (non-empty) vector — the vectored-
|
||||
* noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence
|
||||
* cannot be decided from the id-directory listing alone: a deferred-embed
|
||||
* noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`)
|
||||
* until its embed LANDS, so this walk reads every noun's `vectors.json`
|
||||
* CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time
|
||||
* legacy-counts.json derivation or a lost/corrupted counts.json recovery;
|
||||
* the result is persisted so this scan never repeats.
|
||||
*/
|
||||
private async scanVectoredNounCount(): Promise<number> {
|
||||
const base = path.join(this.rootDir, 'entities', 'nouns')
|
||||
let vectored = 0
|
||||
try {
|
||||
const shards = await fs.promises.readdir(base, { withFileTypes: true })
|
||||
for (const shard of shards) {
|
||||
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue
|
||||
const shardPath = path.join(base, shard.name)
|
||||
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
|
||||
for (const entry of ids) {
|
||||
if (!entry.isDirectory()) continue
|
||||
const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name))
|
||||
if (record && Array.isArray(record.vector) && record.vector.length > 0) {
|
||||
vectored++
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
}
|
||||
return vectored
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist counts to filesystem storage
|
||||
*/
|
||||
|
|
@ -2744,6 +2829,10 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// written before the ledger existed; initializeCounts() derives them once.
|
||||
totalNounCountAll: this.totalNounCountAll,
|
||||
totalVerbCountAll: this.totalVerbCountAll,
|
||||
// Vectored-noun ledger scalar — absent in files written before it
|
||||
// existed; initializeCounts() derives it once (a content scan, see
|
||||
// scanVectoredNounCount()'s JSDoc).
|
||||
totalVectoredNounCount: this.totalVectoredNounCount,
|
||||
allCountsSuspect: this.allCountsSuspect,
|
||||
lastUpdated: new Date().toISOString()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -520,6 +520,12 @@ export class MemoryStorage extends BaseStorage {
|
|||
|
||||
let totalNouns = 0
|
||||
let totalVerbs = 0
|
||||
// Vectored-noun scalar: unlike the bare presence check above, this needs
|
||||
// the vectors.json RECORD'S content — a deferred-embed noun's record
|
||||
// exists with an empty `vector: []` until its embed lands. In-memory this
|
||||
// is a free field access (no I/O), unlike the filesystem adapter's
|
||||
// per-noun disk read.
|
||||
let totalVectoredNouns = 0
|
||||
|
||||
// Scan all paths in objectStore
|
||||
for (const path of this.objectStore.keys()) {
|
||||
|
|
@ -528,6 +534,10 @@ export class MemoryStorage extends BaseStorage {
|
|||
if (nounMatch) {
|
||||
// Type is in metadata, not path - just count total
|
||||
totalNouns++
|
||||
const record = this.objectStore.get(path) as { vector?: unknown } | undefined
|
||||
if (Array.isArray(record?.vector) && record.vector.length > 0) {
|
||||
totalVectoredNouns++
|
||||
}
|
||||
}
|
||||
|
||||
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
|
||||
|
|
@ -543,6 +553,7 @@ export class MemoryStorage extends BaseStorage {
|
|||
// A scan of every canonical record IS the ALL-visibility count.
|
||||
this.totalNounCountAll = totalNouns
|
||||
this.totalVerbCountAll = totalVerbs
|
||||
this.totalVectoredNounCount = totalVectoredNouns
|
||||
this.allCountsSuspect = false
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1762,8 +1762,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
* @param hadVector - OPTIONAL vectored-noun ledger hint, forwarded to
|
||||
* {@link deleteNounMetadata} unchanged — see its JSDoc.
|
||||
*/
|
||||
public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void> {
|
||||
public async deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
|
||||
|
|
@ -1780,7 +1782,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// LONGER wrapped in a blind catch that masked faults as "file didn't exist".
|
||||
// `priorMetadata` (the caller's pre-delete read) keeps the decrement honest
|
||||
// even when the canonical read inside returns null (replace race / ghost).
|
||||
await this.deleteNounMetadata(id, priorMetadata)
|
||||
await this.deleteNounMetadata(id, priorMetadata, hadVector)
|
||||
|
||||
// Remove the now-empty entity container (a no-op for key/prefix stores).
|
||||
await this.removeCanonicalContainer(getNounVectorPath(id))
|
||||
|
|
@ -3392,11 +3394,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
/**
|
||||
* Save noun metadata to storage (now typed)
|
||||
* Routes to correct sharded location based on UUID
|
||||
* @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc.
|
||||
*/
|
||||
public async saveNounMetadata(id: string, metadata: NounMetadata): Promise<void> {
|
||||
public async saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void> {
|
||||
// Validate noun type in metadata - storage boundary protection
|
||||
validateNounType(metadata.noun)
|
||||
return this.saveNounMetadata_internal(id, metadata)
|
||||
return this.saveNounMetadata_internal(id, metadata, hasVector)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -3407,9 +3410,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* This ensures counts are updated AFTER metadata exists, fixing the race condition
|
||||
* where storage adapters tried to read metadata before it was saved.
|
||||
*
|
||||
* @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc.
|
||||
* @protected
|
||||
*/
|
||||
protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise<void> {
|
||||
protected async saveNounMetadata_internal(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// ID-first path - no type needed!
|
||||
|
|
@ -3465,6 +3469,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// record persists here so the ALL scalar never lags the tree.
|
||||
if (isNew) {
|
||||
this.totalNounCountAll++
|
||||
// Vectored-noun ledger: rides the SAME isNew gate (once per id, at
|
||||
// creation) — this seam is metadata-write-driven and never re-runs on
|
||||
// the HNSW neighbor-link re-saves that hit saveNoun_internal, so it
|
||||
// cannot double-count. A deferred-embed insert passes hasVector=false
|
||||
// (or omits it); its vector lands later via noteVectorLanded().
|
||||
if (hasVector) {
|
||||
this.totalVectoredNounCount++
|
||||
}
|
||||
if (!(metadata.noun && isCounted)) {
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
|
|
@ -3860,8 +3872,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* the skip permanently inflated the persisted totals (adds counted, paired
|
||||
* removals not decremented), and `Math.max(totalNounCount, scanned)` made
|
||||
* the inflation unfixable by any disk cleanup.
|
||||
* @param hadVector - OPTIONAL vectored-noun ledger hint — see
|
||||
* {@link StorageAdapter.deleteNounMetadata}'s JSDoc. This method never
|
||||
* reads `vectors.json` to answer the question itself (a canonical read
|
||||
* the delete path must never add); a caller that cannot supply the fact
|
||||
* for free leaves it `undefined`, and the ledger goes SUSPECT rather
|
||||
* than guessing.
|
||||
*/
|
||||
public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise<void> {
|
||||
public async deleteNounMetadata(
|
||||
id: string,
|
||||
priorRecord?: NounMetadata | null,
|
||||
hadVector?: boolean
|
||||
): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Direct O(1) delete with ID-first path. Read the canonical record BEFORE
|
||||
|
|
@ -3885,6 +3907,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
} else {
|
||||
this.markAllCountsSuspect('noun', id)
|
||||
}
|
||||
|
||||
// Vectored-noun ledger: a KNOWN vector fact decrements (or no-ops);
|
||||
// an UNKNOWN one goes suspect rather than guessing — see @param hadVector.
|
||||
if (hadVector === true) {
|
||||
if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount--
|
||||
else this.markAllCountsSuspect('noun-vector', id)
|
||||
} else if (hadVector === undefined) {
|
||||
this.markAllCountsSuspect('noun-vector', id)
|
||||
}
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
})
|
||||
|
|
@ -4549,6 +4580,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// construction.
|
||||
let allNouns = 0
|
||||
let allVerbs = 0
|
||||
// Vectored-noun scalar: unlike `allNouns` (decided from the metadata.json
|
||||
// LISTING alone), presence cannot be decided from the vectors.json
|
||||
// listing alone — a deferred-embed noun's vectors.json EXISTS with an
|
||||
// empty `vector: []` until its embed lands, so the file's CONTENT must be
|
||||
// read. This walk already lists every path per shard (including
|
||||
// vectors.json entries — `listCanonicalObjects` yields both legs), so
|
||||
// reading them here costs one EXTRA read per noun beyond the metadata.json
|
||||
// read above (doubling this walk's per-noun I/O) — honest cost, paid only
|
||||
// by this diagnostic/repair recount, never on the hot path.
|
||||
let allVectoredNouns = 0
|
||||
|
||||
// Scan noun shards
|
||||
for (let shard = 0; shard < 256; shard++) {
|
||||
|
|
@ -4559,6 +4600,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
const paths = await this.listCanonicalObjects(shardDir)
|
||||
|
||||
for (const path of paths) {
|
||||
if (path.includes('/vectors.json')) {
|
||||
try {
|
||||
const vectorRecord = await this.readCanonicalObject(path)
|
||||
if (vectorRecord && Array.isArray(vectorRecord.vector) && vectorRecord.vector.length > 0) {
|
||||
allVectoredNouns++
|
||||
}
|
||||
} catch (error) {
|
||||
// Skip vector records that fail to load — best-effort ground truth,
|
||||
// same as the metadata read below.
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!path.includes('/metadata.json')) continue
|
||||
allNouns++
|
||||
|
||||
|
|
@ -4634,17 +4687,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// IS the proof an unprovable delete could not give.
|
||||
const nounsAllBefore = this.totalNounCountAll
|
||||
const verbsAllBefore = this.totalVerbCountAll
|
||||
const vectoredBefore = this.totalVectoredNounCount
|
||||
this.totalNounCountAll = allNouns
|
||||
this.totalVerbCountAll = allVerbs
|
||||
this.totalVectoredNounCount = allVectoredNouns
|
||||
this.allCountsSuspect = false
|
||||
this.countCache.clear()
|
||||
await this.persistCounts()
|
||||
|
||||
prodLog.info(
|
||||
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
||||
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
|
||||
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
|
||||
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
|
||||
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs / ${allVectoredNouns} vectored nouns` +
|
||||
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs || vectoredBefore !== allVectoredNouns
|
||||
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore} / ${vectoredBefore})`
|
||||
: ' (unchanged)') +
|
||||
` — scalar + per-type persisted`
|
||||
)
|
||||
|
|
|
|||
|
|
@ -52,7 +52,15 @@ export class SaveNounMetadataOperation implements Operation {
|
|||
private readonly storage: StorageAdapter,
|
||||
private readonly id: string,
|
||||
private readonly metadata: NounMetadata,
|
||||
private readonly isNew: boolean = false
|
||||
private readonly isNew: boolean = false,
|
||||
/**
|
||||
* OPTIONAL vectored-noun ledger hint: `true` when this write's paired
|
||||
* vector (the SAME insert's `vector` local) is real/non-empty — see
|
||||
* {@link StorageAdapter.saveNounMetadata}'s JSDoc for the isNew-gated,
|
||||
* double-count-proof seam this rides. Default `false`: a deferred-embed
|
||||
* insert (or any caller that doesn't know) never counts here.
|
||||
*/
|
||||
private readonly hasVector: boolean = false
|
||||
) {}
|
||||
|
||||
async execute(): Promise<RollbackAction> {
|
||||
|
|
@ -62,7 +70,7 @@ export class SaveNounMetadataOperation implements Operation {
|
|||
: await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')
|
||||
|
||||
// Save new metadata
|
||||
await this.storage.saveNounMetadata(this.id, this.metadata)
|
||||
await this.storage.saveNounMetadata(this.id, this.metadata, this.hasVector)
|
||||
|
||||
// Return rollback action
|
||||
return async () => {
|
||||
|
|
@ -70,8 +78,10 @@ export class SaveNounMetadataOperation implements Operation {
|
|||
// Restore previous metadata
|
||||
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
||||
} else {
|
||||
// Delete newly created metadata
|
||||
await this.storage.deleteNounMetadata(this.id)
|
||||
// Delete newly created metadata. `this.hasVector` is the SAME fact
|
||||
// this operation's own execute() used to (maybe) count the vectored
|
||||
// ledger — reversing with it on rollback needs no new read.
|
||||
await this.storage.deleteNounMetadata(this.id, undefined, this.hasVector)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -140,7 +150,9 @@ export class SaveNounOperation implements Operation {
|
|||
// Note: Not all adapters implement deleteNoun
|
||||
// This is acceptable - metadata deletion makes entity invisible
|
||||
if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') {
|
||||
await this.storage.deleteNoun(this.noun.id)
|
||||
// `this.noun.vector` is the SAME record just written — the
|
||||
// vectored-noun ledger fact is free (no added read) and exact.
|
||||
await this.storage.deleteNoun(this.noun.id, undefined, this.noun.vector.length > 0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -198,14 +210,21 @@ export class DeleteNounMetadataOperation implements Operation {
|
|||
return async () => {}
|
||||
}
|
||||
|
||||
// Vectored-noun ledger fact: `previousNoun` is already read above for the
|
||||
// before-image capture — no added read. `undefined` (noun genuinely
|
||||
// absent, metadata-only ghost) is passed through honestly; the storage
|
||||
// layer marks the ledger suspect rather than guessing.
|
||||
const hadVector = previousNoun ? previousNoun.vector.length > 0 : undefined
|
||||
|
||||
// Full removal: both canonical legs + the entity container + count decrement
|
||||
// (the prior record keeps the decrement honest on a null canonical read).
|
||||
await this.storage.deleteNoun(this.id, previousMetadata)
|
||||
await this.storage.deleteNoun(this.id, previousMetadata, hadVector)
|
||||
|
||||
// Return rollback action
|
||||
return async () => {
|
||||
// Restore the vector leg, then the metadata leg through the count-aware
|
||||
// save so deleteNoun()'s decrement is reversed.
|
||||
// save so deleteNoun()'s decrement is reversed (hadVector's mirror:
|
||||
// re-increments the vectored ledger iff the restored vector is real).
|
||||
if (previousNoun) {
|
||||
await this.storage.saveNoun({
|
||||
id: previousNoun.id,
|
||||
|
|
@ -215,7 +234,7 @@ export class DeleteNounMetadataOperation implements Operation {
|
|||
})
|
||||
}
|
||||
if (previousMetadata) {
|
||||
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
||||
await this.storage.saveNounMetadata(this.id, previousMetadata, hadVector === true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,10 +17,11 @@
|
|||
* (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is
|
||||
* upgraded from the canonical id tree at open, then persisted.
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
import * as zlib from 'node:zlib'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
|
||||
/** Count canonical `<root>/entities/<kind>/<shard>/<id>` directories — every tier. */
|
||||
|
|
@ -180,3 +181,176 @@ describe('canonical count ledger — ALL-visibility scalars, unclamped totals, r
|
|||
expect(ledger.nouns.all).toBe(truth)
|
||||
})
|
||||
})
|
||||
|
||||
/** Count `<root>/entities/nouns/<shard>/<id>/vectors.json[.gz]` files holding a non-empty `vector`. */
|
||||
function countVectoredNouns(root: string): number {
|
||||
const base = path.join(root, 'entities', 'nouns')
|
||||
if (!fs.existsSync(base)) return 0
|
||||
let n = 0
|
||||
for (const shard of fs.readdirSync(base)) {
|
||||
const shardDir = path.join(base, shard)
|
||||
if (!fs.statSync(shardDir).isDirectory()) continue
|
||||
for (const id of fs.readdirSync(shardDir)) {
|
||||
const idDir = path.join(shardDir, id)
|
||||
if (!fs.statSync(idDir).isDirectory()) continue
|
||||
const plainPath = path.join(idDir, 'vectors.json')
|
||||
const gzPath = `${plainPath}.gz`
|
||||
let record: any = null
|
||||
if (fs.existsSync(plainPath)) {
|
||||
record = JSON.parse(fs.readFileSync(plainPath, 'utf-8'))
|
||||
} else if (fs.existsSync(gzPath)) {
|
||||
record = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf-8'))
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
if (Array.isArray(record.vector) && record.vector.length > 0) n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s coverage denominator)', () => {
|
||||
let dir: string
|
||||
let brain: any
|
||||
|
||||
const open = async () => {
|
||||
const b: any = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
await b.init()
|
||||
return b
|
||||
}
|
||||
|
||||
/** Baseline vectored count right after a fresh open() — init() creates a
|
||||
* hidden system VFS-root noun that itself carries a real vector, so a
|
||||
* brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off
|
||||
* this baseline rather than hardcoding it away. */
|
||||
let baseline: number
|
||||
|
||||
beforeEach(async () => {
|
||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-'))
|
||||
brain = await open()
|
||||
baseline = (await brain.storage.getCanonicalCounts()).vectors.all
|
||||
})
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks()
|
||||
await brain.close?.().catch(() => {})
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('an explicit-vector add counts immediately; the ledger matches the on-disk vectors.json content', async () => {
|
||||
await brain.add({ data: 'a', type: 'document', vector: Array(384).fill(0).map((_, i) => Math.sin(i)) })
|
||||
await brain.add({ data: 'b', type: 'document' }) // embedded (non-deferred) — also a real vector
|
||||
await brain.flush()
|
||||
|
||||
const ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.vectors.all).toBe(baseline + 2)
|
||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||
expect(ledger.suspect).toBe(false)
|
||||
})
|
||||
|
||||
it('a deferred-embed add does NOT count until its embed LANDS', async () => {
|
||||
// Hold the background worker's embed call open under manual control — a
|
||||
// deterministic embedder is fast enough that the landing could otherwise
|
||||
// race ahead of the "still unlanded" assertion below.
|
||||
let resolveEmbed: ((v: number[]) => void) | undefined
|
||||
vi.spyOn(brain, 'embed').mockImplementation(
|
||||
() => new Promise<number[]>((resolve) => { resolveEmbed = resolve })
|
||||
)
|
||||
|
||||
const id = await brain.add({ data: 'deferred content', type: 'document', deferEmbedding: true })
|
||||
await brain.flush()
|
||||
|
||||
// Landed nothing yet — the ledger must not count the stub.
|
||||
let ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.vectors.all).toBe(baseline)
|
||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||
|
||||
// Release the held embed, then cross the barrier: the vector lands
|
||||
// (system:embed-landing).
|
||||
resolveEmbed!(Array(384).fill(0).map((_, i) => Math.cos(i)))
|
||||
await brain.awaitPendingEmbeds()
|
||||
const landed = await brain.get(id, { includeVectors: true })
|
||||
expect((landed!.vector as number[]).length).toBeGreaterThan(0)
|
||||
|
||||
ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.vectors.all).toBe(baseline + 1)
|
||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||
expect(ledger.suspect).toBe(false)
|
||||
})
|
||||
|
||||
it('a proven delete of a vectored noun decrements; a non-vectored (unlanded) delete does not', async () => {
|
||||
const vectoredId = await brain.add({ data: 'v', type: 'document' }) // real embed, unmocked
|
||||
// Block the embed worker AFTER the real add above — a deterministic
|
||||
// embedder is fast enough that the deferred noun below could otherwise
|
||||
// land before this test observes its "still unlanded" state.
|
||||
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
||||
const deferredId = await brain.add({ data: 'd', type: 'document', deferEmbedding: true })
|
||||
await brain.flush()
|
||||
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(baseline + 1)
|
||||
|
||||
await brain.remove(vectoredId)
|
||||
await brain.flush()
|
||||
let ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.vectors.all).toBe(baseline)
|
||||
expect(ledger.suspect).toBe(false)
|
||||
|
||||
await brain.remove(deferredId) // never had a real vector — no decrement, still unsuspect
|
||||
await brain.flush()
|
||||
ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.vectors.all).toBe(baseline)
|
||||
expect(ledger.suspect).toBe(false)
|
||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||
})
|
||||
|
||||
it('the recount corrects a tampered vectors.all scalar, surviving reopen', async () => {
|
||||
await brain.add({ data: 'real 1', type: 'document' })
|
||||
await brain.add({ data: 'real 2', type: 'document' })
|
||||
await brain.flush()
|
||||
const truth = countVectoredNouns(dir)
|
||||
expect(truth).toBe(baseline + 2)
|
||||
|
||||
;(brain.storage as any).totalVectoredNounCount = truth + 40
|
||||
await (brain.storage as any).persistCounts()
|
||||
await brain.close()
|
||||
brain = await open()
|
||||
|
||||
// The lie survives reopen (never clamped).
|
||||
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth + 40)
|
||||
|
||||
await brain.repairIndex()
|
||||
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth)
|
||||
|
||||
await brain.close()
|
||||
brain = await open()
|
||||
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth)
|
||||
})
|
||||
|
||||
it('a legacy counts.json without totalVectoredNounCount is derived once from vectors.json content and persisted', async () => {
|
||||
await brain.add({ data: 'one', type: 'document' }) // real embed, unmocked
|
||||
// Block the embed worker AFTER the real add above — a deterministic
|
||||
// embedder is fast enough that the deferred noun below could otherwise
|
||||
// land before close(), which would inflate this test's expected count.
|
||||
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
||||
await brain.add({ data: 'two deferred', type: 'document', deferEmbedding: true })
|
||||
await brain.flush()
|
||||
await brain.close()
|
||||
|
||||
const countsPath = path.join(dir, '_system', 'counts.json')
|
||||
const raw = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
||||
expect(typeof raw.totalVectoredNounCount).toBe('number')
|
||||
delete raw.totalVectoredNounCount
|
||||
fs.writeFileSync(countsPath, JSON.stringify(raw, null, 2))
|
||||
|
||||
brain = await open()
|
||||
const ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun
|
||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||
const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
||||
expect(persisted.totalVectoredNounCount).toBe(baseline + 1)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
247
tests/integration/vector-leg-open-build.test.ts
Normal file
247
tests/integration/vector-leg-open-build.test.ts
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
/**
|
||||
* @module tests/integration/vector-leg-open-build
|
||||
* @description THE LAST RED of the two-engine release gate: a migrated
|
||||
* store can hold canonical vectored nouns with NO derived vector index
|
||||
* built. `open()` owns building the derived indexes (reads never build —
|
||||
* see `rebuildIndexesIfNeeded`'s JSDoc); the defect this pins is the vector
|
||||
* leg's decision silently skipping that build, so `find`/search served `[]`
|
||||
* with no error and no narration.
|
||||
*
|
||||
* A downstream deployment measures this through a native vector provider
|
||||
* whose own health report can legitimately say `serving: true` even while
|
||||
* vector COVERAGE is honestly unledgered on its side (an unledgered
|
||||
* invariant never flips `serving` — see `HealthReport`'s derivation laws).
|
||||
* This repo ships only the JS engine, so the reproduction here uses the
|
||||
* SAME plugin seam a native provider would (`brain.use({ activate: ctx =>
|
||||
* ctx.registerProvider('vector', factory) })`, the pattern
|
||||
* `tests/unit/cold-open-rebuild-gate.test.ts` already established for this
|
||||
* exact class of gate-decision bug) with a stub that WRAPS the real
|
||||
* `JsHnswVectorIndex` — every method delegates to a genuine engine (so a
|
||||
* successful rebuild restores REAL, searchable vectors), except `size()`
|
||||
* (fakes 0 until rebuild runs — the "never built" posture) and
|
||||
* `healthReport()` (always reports `serving: true`, `unledgered:
|
||||
* ['vector-coverage']` — the "I don't track this yet" posture). This is
|
||||
* "as close as the JS engine allows": the gap is reproduced at the exact
|
||||
* decision the fix changes, not approximated by deleting files the JS
|
||||
* engine's own cold-start heuristic already recovers from unaided (see the
|
||||
* inverse pin below and cold-open-rebuild-gate.test.ts's already-pinned
|
||||
* "isReady()===true, size()===0" contract, which this fix deliberately does
|
||||
* NOT touch — bare isReady() has no unledgered concept to hide behind, and
|
||||
* overriding it would reopen the 48-seconds-per-restart regression pinned
|
||||
* there).
|
||||
*
|
||||
* Pins:
|
||||
* (1) COVERAGE GAP FORCES THE BUILD: N vectored nouns, a provider that
|
||||
* claims `serving: true` at `size()===0` — open() builds anyway (the
|
||||
* ledger proves there is something to cover), and search returns real
|
||||
* results, never `[]`.
|
||||
* (2) THE INVERSE, HONEST EMPTY: 0 vectored nouns (every embed still
|
||||
* deferred/unlanded) — open() does NOT attempt a rebuild (nothing to
|
||||
* load; the old blunt "always rebuild when size()===0" heuristic wasted
|
||||
* a full canonical walk here for zero benefit), and search honestly
|
||||
* returns `[]` — no error, no false coverage-gap narration.
|
||||
*
|
||||
* SEARCH VERIFICATION NOTE: pin (1) verifies "search returns real results"
|
||||
* via `find({ query: <text> })` (semantic search — embeds the query, then
|
||||
* searches), matching the pattern `tests/integration/hnsw-rebuild.test.ts`
|
||||
* already uses for exactly this "post-rebuild search works" class of pin.
|
||||
* A raw `find({ vector: <array> })` / `index.search(vector, k)` call was
|
||||
* tried first and found to reproducibly return only 1 hit after a
|
||||
* FROM-CANONICAL rebuild (never the full requested `limit`, sometimes not
|
||||
* even a real neighbor) — REGARDLESS of this task's changes: it reproduces
|
||||
* identically on a plain, unwrapped, un-stubbed reopen with the stock JS
|
||||
* engine (verified against `hnsw-rebuild.test.ts`'s own construction) and
|
||||
* is therefore a PRE-EXISTING, orthogonal defect in the JS HNSW engine's
|
||||
* rebuilt-graph connectivity — outside this task's two deliverables (the
|
||||
* count ledger and the open-gate REBUILD DECISION, not rebuild()'s internal
|
||||
* search quality). Left for a separate investigation; not touched here.
|
||||
*/
|
||||
import { describe, it, expect, afterEach, vi } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js'
|
||||
|
||||
const tmpDirs: string[] = []
|
||||
function mkTmp(): string {
|
||||
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vector-leg-open-'))
|
||||
tmpDirs.push(d)
|
||||
return d
|
||||
}
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
const V = (seed: number) =>
|
||||
Array.from({ length: 384 }, (_, i) => Math.sin((seed + 1) * 7919 + i * 131) * 0.5 + 0.5)
|
||||
|
||||
/**
|
||||
* Build a store with N explicit-vector (non-deferred) nouns, flush, close.
|
||||
* Each noun also carries embeddable text (`technology`/`science`, matching
|
||||
* the query used below) so the semantic-search verification exercises real
|
||||
* retrieval, not a coincidental match. The default JS engine builds a fully
|
||||
* current store — the epoch marker is stamped current at this open's
|
||||
* completion, so a later reopen's `_indexEpochStale` is honestly false and
|
||||
* cannot mask the ledger-gap decision under test (nothing here manufactures
|
||||
* epoch drift).
|
||||
*/
|
||||
async function buildVectoredStore(dir: string, n: number): Promise<string[]> {
|
||||
const brain: any = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
plugins: [],
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
await brain.init()
|
||||
const ids: string[] = []
|
||||
for (let i = 0; i < n; i++) {
|
||||
ids.push(
|
||||
await brain.add({
|
||||
data: `doc ${i} about ${i % 2 === 0 ? 'technology' : 'science'}`,
|
||||
type: 'document',
|
||||
vector: V(i)
|
||||
})
|
||||
)
|
||||
}
|
||||
await brain.flush()
|
||||
await brain.close()
|
||||
return ids
|
||||
}
|
||||
|
||||
describe('vector-leg open-build (two-engine gate, last red)', () => {
|
||||
it('coverage gap: a provider reporting serving:true at size()===0 is overridden by the vectored-noun ledger — open() builds, search returns real results', async () => {
|
||||
const dir = mkTmp()
|
||||
const ids = await buildVectoredStore(dir, 12)
|
||||
|
||||
// Wrap the REAL JS engine so a successful rebuild restores genuine,
|
||||
// searchable vectors — only `size()` and `healthReport()` are faked,
|
||||
// simulating a native provider that has never built its own coverage of
|
||||
// an unledgered invariant.
|
||||
const calls = { rebuild: 0 }
|
||||
const brain: any = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
plugins: [],
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
brain.use({
|
||||
name: 'fake-native-vector-unledgered-coverage',
|
||||
activate: async (ctx: any) => {
|
||||
ctx.registerProvider('vector', (config: any, distance: any, options: any) => {
|
||||
const real = new JsHnswVectorIndex(config, distance, options)
|
||||
let rebuilt = false
|
||||
const originalRebuild = real.rebuild.bind(real)
|
||||
;(real as any).rebuild = async (...args: any[]) => {
|
||||
const r = await originalRebuild(...args)
|
||||
calls.rebuild++
|
||||
rebuilt = true
|
||||
return r
|
||||
}
|
||||
const originalSize = real.size.bind(real)
|
||||
;(real as any).size = () => (rebuilt ? originalSize() : 0)
|
||||
;(real as any).healthReport = () => ({
|
||||
provider: 'vector',
|
||||
healthy: true,
|
||||
serving: true,
|
||||
invariants: [],
|
||||
checkedAt: Date.now(),
|
||||
durationMs: 0,
|
||||
generation: 1,
|
||||
unledgered: ['vector-coverage']
|
||||
})
|
||||
return real
|
||||
})
|
||||
return true
|
||||
}
|
||||
})
|
||||
await brain.init()
|
||||
|
||||
// WITHOUT any find() first: open() itself must have built the leg.
|
||||
expect(calls.rebuild, 'open() forced the rebuild despite serving:true').toBe(1)
|
||||
const status = await brain.getIndexStatus()
|
||||
expect(status.hnswIndex.size).toBeGreaterThanOrEqual(ids.length)
|
||||
|
||||
// Real, searchable results — never [] (see the module doc's SEARCH
|
||||
// VERIFICATION NOTE for why this is a semantic `query`, not a raw
|
||||
// `vector`, call).
|
||||
const results = await brain.find({ query: 'technology document', limit: 5 })
|
||||
expect(results.length).toBeGreaterThan(0)
|
||||
expect(results.length).not.toBe(0)
|
||||
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => {
|
||||
// ARCHITECTURAL NOTE (found while building this pin): every brainy store
|
||||
// carries ONE permanent, always-vectored noun beyond user data — the VFS
|
||||
// root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`,
|
||||
// src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero
|
||||
// (but non-empty, length-384) vector on EVERY store's first open — never
|
||||
// deferred (a deliberate WASM-cold-compile-avoidance fix, see that
|
||||
// file's comment) — and VFS init unconditionally re-creates it if
|
||||
// missing, before the rebuild gate ever runs. A literal "0 vectored
|
||||
// nouns" store is therefore unreachable through the public API; a
|
||||
// brand-new store's `vectors.all` floor is 1, not 0. This pin verifies
|
||||
// the law the task names in the ACHIEVABLE form: nouns whose embed is
|
||||
// still deferred/unlanded contribute NOTHING to the vectored-noun ledger
|
||||
// — the coverage-gap comparison sees exactly the root (1), never
|
||||
// root+deferred — and semantic search over deferred-only user content
|
||||
// honestly returns `[]` (no error, no false "coverage restored" claim).
|
||||
const dir = mkTmp()
|
||||
|
||||
const build: any = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
plugins: [],
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
await build.init()
|
||||
const rootOnlyLedger = await build.storage.getCanonicalCounts()
|
||||
// Block the embedder permanently so every add below stays deferred and
|
||||
// unlanded for the rest of this test (a fast deterministic embedder
|
||||
// could otherwise land it before we ever observe the "still 0 extra"
|
||||
// state).
|
||||
vi.spyOn(build, 'embed').mockImplementation(() => new Promise(() => {}))
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await build.add({ data: `deferred ${i}`, type: 'document', deferEmbedding: true })
|
||||
}
|
||||
await build.flush()
|
||||
const ledgerWithDeferred = await build.storage.getCanonicalCounts()
|
||||
// The five deferred adds contributed ZERO to the vectored-noun ledger.
|
||||
expect(ledgerWithDeferred.vectors.all).toBe(rootOnlyLedger.vectors.all)
|
||||
await build.close()
|
||||
|
||||
// Reopen (default JS engine — no stub needed): the root is the ONLY
|
||||
// thing the vector leg has to load; the deferred nouns are correctly
|
||||
// invisible to it. Block the embedder again BEFORE init() — reopen
|
||||
// recovers the durable pending-embed markers and kicks the worker as
|
||||
// part of init() itself, and an unblocked deterministic embedder could
|
||||
// land all five before this test observes the open-time ledger.
|
||||
const brain: any = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
plugins: [],
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
||||
await brain.init()
|
||||
// The ledger is exactly the root — the five deferred, still-unlanded
|
||||
// nouns (which the rebuild above DOES insert into the graph, each with
|
||||
// its stub empty vector — `index.size()` counts EVERY canonical noun's
|
||||
// graph node, deferred or not, so it is not the coverage metric) never
|
||||
// inflate the VECTORED count.
|
||||
const ledgerAfterReopen = await brain.storage.getCanonicalCounts()
|
||||
expect(ledgerAfterReopen.vectors.all).toBe(rootOnlyLedger.vectors.all)
|
||||
|
||||
const results = await brain.find({ vector: V(3), limit: 5 })
|
||||
expect(results).toEqual([])
|
||||
|
||||
await brain.close()
|
||||
})
|
||||
})
|
||||
|
|
@ -380,6 +380,12 @@ describe.sequential('lifecycle — the working store', () => {
|
|||
counted: aliveVerbs + model.vfsContainsVerbs,
|
||||
all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs
|
||||
},
|
||||
// Every noun this biography ever adds carries an explicit/computed
|
||||
// vector (the harness never defers an embed), so the vectored-noun
|
||||
// scalar tracks nouns.all exactly.
|
||||
vectors: {
|
||||
all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns
|
||||
},
|
||||
suspect: false
|
||||
})
|
||||
} finally {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue