Compare commits
2 commits
v10.4.0-rc
...
v10.4.0-rc
| Author | SHA1 | Date | |
|---|---|---|---|
| d3aacfaf24 | |||
| 9730835bdf |
13 changed files with 859 additions and 35 deletions
|
|
@ -2,6 +2,11 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
|
### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25)
|
||||||
|
|
||||||
|
- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b)
|
||||||
|
|
||||||
|
|
||||||
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
|
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
|
||||||
|
|
||||||
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
|
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
|
||||||
|
|
|
||||||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.3",
|
"version": "10.4.0-rc.4",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.3",
|
"version": "10.4.0-rc.4",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.2",
|
"@msgpack/msgpack": "^3.1.2",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.3",
|
"version": "10.4.0-rc.4",
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"module": "dist/index.js",
|
"module": "dist/index.js",
|
||||||
|
|
|
||||||
145
src/brainy.ts
145
src/brainy.ts
|
|
@ -2382,6 +2382,17 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
[{ type: 'embed.landed', id, vector: newVector }],
|
[{ type: 'embed.landed', id, vector: newVector }],
|
||||||
'system:embed-landing'
|
'system:embed-landing'
|
||||||
)
|
)
|
||||||
|
// Vectored-noun ledger: the landing commit above carries a vector
|
||||||
|
// write with NO accompanying metadata operation, so the
|
||||||
|
// saveNounMetadata(..., hasVector) seam never fires for it — the
|
||||||
|
// narrow storage hook is the only seam left. `oldVector.length===0`
|
||||||
|
// (already known for free from the pre-embed read above) proves this
|
||||||
|
// is a GENUINE first landing, not a re-embed of an already-vectored
|
||||||
|
// row (e.g. a deferred update() on a row that already had a real
|
||||||
|
// vector) — the latter must never double-count.
|
||||||
|
if (oldVector.length === 0) {
|
||||||
|
await this.storage.noteVectorLanded?.(id)
|
||||||
|
}
|
||||||
this.clearPendingEmbed(id)
|
this.clearPendingEmbed(id)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
prodLog.warn(
|
prodLog.warn(
|
||||||
|
|
@ -3000,8 +3011,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
const runInsert: TransactionFunction<void> = async (tx) => {
|
const runInsert: TransactionFunction<void> = async (tx) => {
|
||||||
// Operation 1: Save metadata FIRST (TypeAwareStorage caching)
|
// Operation 1: Save metadata FIRST (TypeAwareStorage caching)
|
||||||
// isNew=true: skip pre-read for rollback (entity doesn't exist yet)
|
// isNew=true: skip pre-read for rollback (entity doesn't exist yet)
|
||||||
|
// hasVector: the vectored-noun ledger counts this insert iff its
|
||||||
|
// vector is real/non-empty (never true for a deferred embed, whose
|
||||||
|
// stub `vector` is `[]` — it counts later, at landing).
|
||||||
tx.addOperation(
|
tx.addOperation(
|
||||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, true)
|
new SaveNounMetadataOperation(this.storage, id, storageMetadata, true, vector.length > 0)
|
||||||
)
|
)
|
||||||
|
|
||||||
// Operation 2: Save vector data
|
// Operation 2: Save vector data
|
||||||
|
|
@ -10568,7 +10582,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
plan.postCommit.push(() => this.kickEmbedWorker())
|
plan.postCommit.push(() => this.kickEmbedWorker())
|
||||||
}
|
}
|
||||||
plan.operations.push(
|
plan.operations.push(
|
||||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew),
|
// hasVector: see the single-add() insert path's comment — never true
|
||||||
|
// for a deferred embed (stub vector `[]`; counted later at landing).
|
||||||
|
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0),
|
||||||
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
|
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
|
||||||
...(deferringEmbed
|
...(deferringEmbed
|
||||||
? []
|
? []
|
||||||
|
|
@ -16900,9 +16916,92 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
const shouldRebuildMetadata =
|
const shouldRebuildMetadata =
|
||||||
!metadataMigrating &&
|
!metadataMigrating &&
|
||||||
(epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0))
|
(epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0))
|
||||||
const shouldRebuildVector =
|
|
||||||
|
// VECTOR LEG — the two-engine gate's last red: a migrated 7.x-era store
|
||||||
|
// can hold canonical vectored nouns with NO derived vector index built.
|
||||||
|
// `legNeedsRebuild`'s size-heuristic fallback (below) only fires off
|
||||||
|
// `hnswIndexSize === 0`, and its health-report branch trusts a
|
||||||
|
// provider's own `serving` verdict verbatim — but a provider's health
|
||||||
|
// report can legitimately say `serving: true` while vector coverage is
|
||||||
|
// honestly UNLEDGERED on ITS side too (an unledgered invariant never
|
||||||
|
// flips serving), so neither signal alone can tell "genuinely empty"
|
||||||
|
// apart from "never built". The canonical vectored-noun ledger
|
||||||
|
// (`getCanonicalCounts().vectors.all` — Deliverable 1) is the
|
||||||
|
// denominator that CAN tell them apart, and is compared here:
|
||||||
|
// - a CONFIDENT (non-suspect) ledger `> 0` while the reported node
|
||||||
|
// count is 0 is a proven coverage gap — force the build regardless
|
||||||
|
// of what a health report claims;
|
||||||
|
// - a CONFIDENT ledger `=== 0` while the node count is 0 proves there
|
||||||
|
// is nothing to load (e.g. every noun's embed is still deferred) —
|
||||||
|
// skip the size-heuristic fallback's blunt "always rebuild when
|
||||||
|
// empty" trigger, which otherwise wastes a full canonical walk for
|
||||||
|
// zero benefit on every cold open of such a store;
|
||||||
|
// - an unavailable/suspect ledger changes nothing — loud errors never
|
||||||
|
// quiet losses, so a doubtful ledger must never suppress a rebuild
|
||||||
|
// the old heuristic would have run.
|
||||||
|
// The bare `isReady()` boolean (no report, no `unledgered` concept) is
|
||||||
|
// NOT overridden — that signal is what fixed the 48-seconds-per-restart
|
||||||
|
// regression pinned in tests/unit/cold-open-rebuild-gate.test.ts (a
|
||||||
|
// disk-native provider legitimately reporting 0 resident while durable
|
||||||
|
// on disk), and re-deriving it from a denominator the provider itself
|
||||||
|
// has no way to consult would reopen exactly that regression.
|
||||||
|
const vectorAssessment = assessProviderHealth(this.index)
|
||||||
|
const vectorLedger = await this.storage.getCanonicalCounts?.()
|
||||||
|
const vectorLedgerAll = vectorLedger?.vectors.all
|
||||||
|
const vectorLedgerConfident = vectorLedger !== undefined && !vectorLedger.suspect
|
||||||
|
const vectorHasCoverageProof = vectorLedgerConfident && (vectorLedgerAll as number) > 0
|
||||||
|
const vectorConfirmedEmpty = vectorLedgerConfident && vectorLedgerAll === 0
|
||||||
|
|
||||||
|
let vectorNeedsRebuild: boolean
|
||||||
|
if (vectorAssessment.via === 'is-ready') {
|
||||||
|
// Bare isReady() stays authoritative and UNMODIFIED — see above.
|
||||||
|
vectorNeedsRebuild = vectorAssessment.readiness === 'not-ready'
|
||||||
|
} else if (vectorAssessment.via === 'health-report') {
|
||||||
|
vectorNeedsRebuild =
|
||||||
|
vectorAssessment.readiness !== 'ready' ||
|
||||||
|
(hnswIndexSize === 0 && vectorHasCoverageProof)
|
||||||
|
} else {
|
||||||
|
// size-heuristic / no provider (the built-in JS engine's own posture).
|
||||||
|
vectorNeedsRebuild = hnswIndexSize === 0 && !vectorConfirmedEmpty
|
||||||
|
}
|
||||||
|
|
||||||
|
const shouldRebuildVector = !vectorMigrating && (epochStale || vectorNeedsRebuild)
|
||||||
|
|
||||||
|
// Narration (and the FAIL-TYPED backstop below) are scoped EXACTLY to
|
||||||
|
// the defect this gate closes: a provider whose OWN health report
|
||||||
|
// claims `serving: true` — an affirmative "I am ready" a caller would
|
||||||
|
// otherwise trust outright — while the canonical ledger proves vector
|
||||||
|
// coverage is missing. This is deliberately NARROWER than "any branch
|
||||||
|
// where the ledger contributed to the decision":
|
||||||
|
// - the bare isReady() branch is untouched, as above (never in scope);
|
||||||
|
// - the health-report branch's OWN `readiness !== 'ready'` case is
|
||||||
|
// already an ordinary, PRE-EXISTING rebuild trigger (the provider
|
||||||
|
// admits not-ready) — not a ledger override, so not a "gap";
|
||||||
|
// - the size-heuristic/no-provider branch's rebuild-when-empty is the
|
||||||
|
// SAME blunt trigger the code always had (`hnswIndexSize === 0`)
|
||||||
|
// — the ledger only ever SUPPRESSES a rebuild there (the confirmed-
|
||||||
|
// empty case), it never forces one the old heuristic wouldn't
|
||||||
|
// already have run. Marking that branch a "gap" too made the
|
||||||
|
// FAIL-TYPED backstop fire on ordinary white-box tests that stub
|
||||||
|
// rebuild() as a no-op and pin `size()` at 0 to drive OTHER
|
||||||
|
// assertions (e.g. migration-deference's isMigrating() coverage) —
|
||||||
|
// those are not silent-empty defects, so they must open exactly as
|
||||||
|
// before (tests/unit/brainy/migration-deference.test.ts).
|
||||||
|
const vectorCoverageGap =
|
||||||
!vectorMigrating &&
|
!vectorMigrating &&
|
||||||
(epochStale || legNeedsRebuild(this.index, hnswIndexSize === 0))
|
vectorAssessment.via === 'health-report' &&
|
||||||
|
vectorAssessment.readiness === 'ready' &&
|
||||||
|
hnswIndexSize === 0 &&
|
||||||
|
vectorHasCoverageProof
|
||||||
|
if (vectorCoverageGap) {
|
||||||
|
prodLog.warn(
|
||||||
|
`[Brainy] open(): vector index reports ${hnswIndexSize} node(s) but the canonical ` +
|
||||||
|
`ledger holds ${vectorLedgerAll} vectored noun(s) — the derived vector index is ` +
|
||||||
|
`missing or unbuilt on this store. Forcing the vector rebuild rather than serving ` +
|
||||||
|
`silent-empty search results.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
const shouldRebuildGraph =
|
const shouldRebuildGraph =
|
||||||
!graphMigrating &&
|
!graphMigrating &&
|
||||||
(epochStale || legNeedsRebuild(this.graphIndex, false))
|
(epochStale || legNeedsRebuild(this.graphIndex, false))
|
||||||
|
|
@ -16955,9 +17054,27 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// provider running its own background migration is skipped here (it owns
|
// provider running its own background migration is skipped here (it owns
|
||||||
// its index until it verifies-and-swaps).
|
// its index until it verifies-and-swaps).
|
||||||
const rebuildStartTime = Date.now()
|
const rebuildStartTime = Date.now()
|
||||||
|
// The vector leg's build door, by contract with the native provider: a
|
||||||
|
// provider exposing fillFromCanonical() gets THAT call — idempotent, the
|
||||||
|
// provider's own init runs it first so this is the backstop — never a
|
||||||
|
// full rebuild() for a coverage gap. A PARTIAL shortfall deliberately
|
||||||
|
// triggers nothing here: that is repair()'s operator door. The JS index
|
||||||
|
// has no fill door and keeps its rebuild.
|
||||||
|
const vectorBuild = (): Promise<unknown> => {
|
||||||
|
const fillDoor = (this.index as unknown as { fillFromCanonical?: () => Promise<unknown> })
|
||||||
|
.fillFromCanonical
|
||||||
|
if (vectorCoverageGap && typeof fillDoor === 'function') {
|
||||||
|
prodLog.warn(
|
||||||
|
`[Brainy] open(): vector coverage gap routes through the provider's ` +
|
||||||
|
`fillFromCanonical() (idempotent canonical fill), not a full rebuild.`
|
||||||
|
)
|
||||||
|
return fillDoor.call(this.index)
|
||||||
|
}
|
||||||
|
return this.index.rebuild()
|
||||||
|
}
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(),
|
shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(),
|
||||||
shouldRebuildVector ? this.index.rebuild() : Promise.resolve(),
|
shouldRebuildVector ? vectorBuild() : Promise.resolve(),
|
||||||
shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve()
|
shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve()
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
@ -16998,6 +17115,24 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
|
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Vector coverage verification: the coverage-gap rebuild above (see
|
||||||
|
// `vectorCoverageGap`) MUST have actually restored the ledger's
|
||||||
|
// vectored nouns. A provider that STILL reports 0 nodes after its own
|
||||||
|
// rebuild() ran — no JS (or provider) fallback could build from what's
|
||||||
|
// on disk — cannot silently complete open(): search would then serve
|
||||||
|
// empty results with no signal, exactly the defect this gate closes.
|
||||||
|
// FAIL TYPED, pre-serve, rather than let a broken vector leg pass as a
|
||||||
|
// successful open.
|
||||||
|
if (vectorCoverageGap && this.index.size() === 0) {
|
||||||
|
throw new VectorIndexNotReadyError(
|
||||||
|
`open(): the canonical ledger holds ${vectorLedgerAll} vectored noun(s) but the vector ` +
|
||||||
|
`index still reports 0 node(s) after rebuild() — the derived vector index could not ` +
|
||||||
|
`be restored from canonical. Refusing to serve silent-empty search results; ` +
|
||||||
|
`investigate the vector provider/storage, or repairIndex({ rebuild: ['vector'] }) ` +
|
||||||
|
`after restoring the underlying data.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
|
// 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
|
||||||
// have now rebuilt and verified, so they match this build's epoch —
|
// have now rebuilt and verified, so they match this build's epoch —
|
||||||
// re-stamp the marker LAST, only here. A crash anywhere above leaves the
|
// re-stamp the marker LAST, only here. A crash anywhere above leaves the
|
||||||
|
|
|
||||||
|
|
@ -801,6 +801,17 @@ export interface DerivedFamilyDeclaration {
|
||||||
export interface CanonicalCounts {
|
export interface CanonicalCounts {
|
||||||
nouns: { counted: number; all: number }
|
nouns: { counted: number; all: number }
|
||||||
verbs: { counted: number; all: number }
|
verbs: { counted: number; all: number }
|
||||||
|
/**
|
||||||
|
* The count of canonical nouns holding a REAL (non-empty) vector — the
|
||||||
|
* coverage denominator a vector index's node-count ledger is measured
|
||||||
|
* against (`nodeCount === vectors.all` is the whole-store coverage
|
||||||
|
* verdict for the vector leg, the vector-side mirror of `nouns.all` for
|
||||||
|
* metadata/graph). A deferred-embed noun (`add({ deferEmbedding: true })`)
|
||||||
|
* counts only once its vector actually LANDS — its canonical record exists
|
||||||
|
* (counted in `nouns.all`) with an empty vector until then, so it is
|
||||||
|
* deliberately NOT counted here in the interim.
|
||||||
|
*/
|
||||||
|
vectors: { all: number }
|
||||||
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
||||||
suspect: boolean
|
suspect: boolean
|
||||||
}
|
}
|
||||||
|
|
@ -819,14 +830,47 @@ export interface StorageAdapter {
|
||||||
* Save noun metadata separately
|
* Save noun metadata separately
|
||||||
* @param id Noun ID
|
* @param id Noun ID
|
||||||
* @param metadata Noun metadata
|
* @param metadata Noun metadata
|
||||||
|
* @param hasVector - OPTIONAL vectored-noun ledger hint: `true` when this
|
||||||
|
* write is a FRESH insert (`isNew`) whose vector is a real, non-empty
|
||||||
|
* array — the caller already knows this for free (the insert's own
|
||||||
|
* `vector` local), so the increment rides the SAME isNew gate that
|
||||||
|
* already protects `totalNounCountAll` from double-counting on HNSW
|
||||||
|
* neighbor-link re-saves (`saveNoun_internal` re-runs on every link
|
||||||
|
* change; this metadata seam does not). Absent/`false` ⇒ no ledger
|
||||||
|
* action. A deferred-embed insert passes `false` (its vector lands
|
||||||
|
* later — see {@link StorageAdapter.noteVectorLanded}).
|
||||||
*/
|
*/
|
||||||
saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>
|
saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete noun metadata
|
* Delete noun metadata
|
||||||
* @param id Noun ID
|
* @param id Noun ID
|
||||||
|
* @param priorRecord - OPTIONAL already-known metadata (the caller's
|
||||||
|
* pre-delete read) — see {@link StorageAdapter.deleteNoun}.
|
||||||
|
* @param hadVector - OPTIONAL vectored-noun ledger hint: `true`/`false`
|
||||||
|
* when the caller already knows (read as a side effect of ITS OWN delete
|
||||||
|
* flow — e.g. `remove()`'s pre-read for the vector-index removal — never
|
||||||
|
* a read added FOR this ledger), `undefined` when genuinely unknown. A
|
||||||
|
* known `true` decrements the vectored-noun ledger; a known `false` is a
|
||||||
|
* no-op (it was never counted); `undefined` marks the ledger SUSPECT
|
||||||
|
* rather than guessing — the delete path must never add a canonical read
|
||||||
|
* to answer this question.
|
||||||
*/
|
*/
|
||||||
deleteNounMetadata(id: string): Promise<void>
|
deleteNounMetadata(id: string, priorRecord?: NounMetadata | null, hadVector?: boolean): Promise<void>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OPTIONAL narrow ledger hook: record that a canonical noun's vector just
|
||||||
|
* LANDED for the first time. Exists ONLY for the deferred-embedding
|
||||||
|
* lifecycle — the landing commit (`system:embed-landing`) carries a vector
|
||||||
|
* write with no accompanying metadata operation, so the normal
|
||||||
|
* `saveNounMetadata(..., hasVector)` seam never fires for it. Callers MUST
|
||||||
|
* call this only when the noun held NO real vector before this write (the
|
||||||
|
* deferred-embed worker already holds that fact for free, from its own
|
||||||
|
* pre-embed read — never an added read). A backend without vectored-noun
|
||||||
|
* tracking is a no-op via this method's absence (feature-detected).
|
||||||
|
* @param id - The noun whose vector just landed.
|
||||||
|
*/
|
||||||
|
noteVectorLanded?(id: string): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get noun with metadata combined
|
* Get noun with metadata combined
|
||||||
|
|
@ -875,8 +919,11 @@ export interface StorageAdapter {
|
||||||
* REQUIRE re-reading the record being removed: when the internal read
|
* REQUIRE re-reading the record being removed: when the internal read
|
||||||
* returns `null` (replace race, or a ghost left by an earlier version) the
|
* returns `null` (replace race, or a ghost left by an earlier version) the
|
||||||
* decrement falls back to this record instead of being silently skipped.
|
* decrement falls back to this record instead of being silently skipped.
|
||||||
|
* @param hadVector OPTIONAL vectored-noun ledger hint — see
|
||||||
|
* {@link StorageAdapter.deleteNounMetadata}'s `hadVector` param, which
|
||||||
|
* this forwards to unchanged.
|
||||||
*/
|
*/
|
||||||
deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void>
|
deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save verb - Pure HNSW verb with core fields only
|
* Save verb - Pure HNSW verb with core fields only
|
||||||
|
|
|
||||||
|
|
@ -1041,12 +1041,27 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
*/
|
*/
|
||||||
protected totalNounCountAll = 0
|
protected totalNounCountAll = 0
|
||||||
protected totalVerbCountAll = 0
|
protected totalVerbCountAll = 0
|
||||||
|
/**
|
||||||
|
* The count of canonical nouns holding a REAL (non-empty) vector — the
|
||||||
|
* vector-side mirror of `totalNounCountAll` and the coverage denominator a
|
||||||
|
* vector index's node-count ledger is measured against. A deferred-embed
|
||||||
|
* noun (`add({ deferEmbedding: true })`) counts only once its vector
|
||||||
|
* LANDS (the `system:embed-landing` commit) — its canonical record exists
|
||||||
|
* (already counted in `totalNounCountAll`) with an empty vector until
|
||||||
|
* then. Maintained on the write path (a fresh insert whose vector is
|
||||||
|
* non-empty +1, a deferred embed's landing +1, a PROVEN delete of a
|
||||||
|
* vectored noun −1), persisted beside the other ALL scalars, recomputed by
|
||||||
|
* the sanctioned recount. Shares `allCountsSuspect` — no separate flag.
|
||||||
|
*/
|
||||||
|
protected totalVectoredNounCount = 0
|
||||||
/**
|
/**
|
||||||
* `true` when a delete could not prove whether the record existed (no
|
* `true` when a delete could not prove whether the record existed (no
|
||||||
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
||||||
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
||||||
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
||||||
* must treat a suspect scalar as unverified, never as exact.
|
* must treat a suspect scalar as unverified, never as exact. Also covers
|
||||||
|
* `totalVectoredNounCount` — a delete whose vector-presence fact was
|
||||||
|
* unknowable marks this SAME flag rather than minting a second one.
|
||||||
*/
|
*/
|
||||||
protected allCountsSuspect = false
|
protected allCountsSuspect = false
|
||||||
/** One narration per session for the suspect transition (never per delete). */
|
/** One narration per session for the suspect transition (never per delete). */
|
||||||
|
|
@ -1083,15 +1098,19 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
||||||
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
||||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||||
* against (the coverage-ledger denominator for derived-index providers).
|
* against (the coverage-ledger denominator for derived-index providers);
|
||||||
* `suspect` is `true` when an unprovable delete has made `all` unverified
|
* `vectors.all` is the vectored-noun scalar — the coverage denominator for
|
||||||
* since the last sanctioned recount (`rebuildTypeCounts`).
|
* a vector index's node-count ledger specifically.
|
||||||
* @returns Both scalars per family plus the suspect flag.
|
* `suspect` is `true` when an unprovable delete has made `all` (any
|
||||||
|
* family, including `vectors`) unverified since the last sanctioned
|
||||||
|
* recount (`rebuildTypeCounts`).
|
||||||
|
* @returns All scalars per family plus the suspect flag.
|
||||||
*/
|
*/
|
||||||
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
||||||
return {
|
return {
|
||||||
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
||||||
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
||||||
|
vectors: { all: this.totalVectoredNounCount },
|
||||||
suspect: this.allCountsSuspect
|
suspect: this.allCountsSuspect
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1103,7 +1122,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* @param family - Which family's delete was unprovable.
|
* @param family - Which family's delete was unprovable.
|
||||||
* @param id - The id whose existence could not be established.
|
* @param id - The id whose existence could not be established.
|
||||||
*/
|
*/
|
||||||
protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void {
|
protected markAllCountsSuspect(family: 'noun' | 'verb' | 'noun-vector', id: string): void {
|
||||||
this.allCountsSuspect = true
|
this.allCountsSuspect = true
|
||||||
if (!this.allCountsSuspectNarrated) {
|
if (!this.allCountsSuspectNarrated) {
|
||||||
this.allCountsSuspectNarrated = true
|
this.allCountsSuspectNarrated = true
|
||||||
|
|
@ -1116,6 +1135,23 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorLanded}):
|
||||||
|
* record a deferred-embed noun's FIRST real vector landing. The caller
|
||||||
|
* (the deferred-embed worker) proves this is a genuine landing — not a
|
||||||
|
* re-embed of an already-vectored row — by observing its own pre-embed
|
||||||
|
* read's vector was empty, at no added storage cost.
|
||||||
|
* @param id - The noun whose vector just landed (retained for a future
|
||||||
|
* narration seam; the count itself needs no id-keyed state).
|
||||||
|
*/
|
||||||
|
async noteVectorLanded(id: string): Promise<void> {
|
||||||
|
void id
|
||||||
|
this.totalVectoredNounCount++
|
||||||
|
this.scheduleCountPersist().catch(() => {
|
||||||
|
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment count for entity type - O(1) operation.
|
* Increment count for entity type - O(1) operation.
|
||||||
* Concurrency is handled by the process-global mutex
|
* Concurrency is handled by the process-global mutex
|
||||||
|
|
|
||||||
|
|
@ -2583,6 +2583,7 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
// record reads), persist, and never scan again. Absent keys are a
|
// record reads), persist, and never scan again. Absent keys are a
|
||||||
// legacy file, not a zero — a zero here would make every provider's
|
// legacy file, not a zero — a zero here would make every provider's
|
||||||
// coverage ledger read "over-posted" on a populated store.
|
// coverage ledger read "over-posted" on a populated store.
|
||||||
|
let needsPersist = false
|
||||||
if (
|
if (
|
||||||
typeof counts.totalNounCountAll === 'number' &&
|
typeof counts.totalNounCountAll === 'number' &&
|
||||||
typeof counts.totalVerbCountAll === 'number'
|
typeof counts.totalVerbCountAll === 'number'
|
||||||
|
|
@ -2601,6 +2602,29 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
||||||
`every tier) and persisted; no further scan.`
|
`every tier) and persisted; no further scan.`
|
||||||
)
|
)
|
||||||
|
needsPersist = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// The vectored-noun scalar (shipped after the ALL scalars above — a
|
||||||
|
// counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but
|
||||||
|
// still predate THIS key). Unlike the ALL scalars, presence cannot be
|
||||||
|
// decided from the id-directory listing alone: a deferred-embed
|
||||||
|
// noun's `vectors.json` EXISTS with an empty `vector: []` until its
|
||||||
|
// embed lands, so this derivation reads every noun's `vectors.json`
|
||||||
|
// ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost.
|
||||||
|
if (typeof counts.totalVectoredNounCount === 'number') {
|
||||||
|
this.totalVectoredNounCount = counts.totalVectoredNounCount
|
||||||
|
} else {
|
||||||
|
const vectored = await this.scanVectoredNounCount()
|
||||||
|
this.totalVectoredNounCount = vectored
|
||||||
|
console.warn(
|
||||||
|
`[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` +
|
||||||
|
`derived once by reading every noun's vectors.json (${vectored} vectored) and ` +
|
||||||
|
`persisted; no further scan.`
|
||||||
|
)
|
||||||
|
needsPersist = true
|
||||||
|
}
|
||||||
|
if (needsPersist) {
|
||||||
await this.persistCounts()
|
await this.persistCounts()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2643,6 +2667,12 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.totalNounCountAll = nouns.count
|
this.totalNounCountAll = nouns.count
|
||||||
this.totalVerbCountAll = verbs.count
|
this.totalVerbCountAll = verbs.count
|
||||||
this.allCountsSuspect = false
|
this.allCountsSuspect = false
|
||||||
|
// Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
|
||||||
|
// (a deferred-embed noun's file exists but holds an empty vector until
|
||||||
|
// its embed lands), so this is a full O(nouns) content scan — see
|
||||||
|
// scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here,
|
||||||
|
// alongside the rest of this from-disk recovery.
|
||||||
|
this.totalVectoredNounCount = await this.scanVectoredNounCount()
|
||||||
|
|
||||||
// Sample some entities for the type distribution (don't read all).
|
// Sample some entities for the type distribution (don't read all).
|
||||||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||||||
|
|
@ -2728,6 +2758,61 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs
|
||||||
|
* — the vector-side mirror of {@link readEntityMetadataRaw}, same
|
||||||
|
* reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`).
|
||||||
|
* @param entityDir - Absolute `entities/nouns/<shard>/<id>` directory.
|
||||||
|
* @returns The parsed vector record, or null when absent/unreadable.
|
||||||
|
*/
|
||||||
|
private async readEntityVectorRaw(entityDir: string): Promise<any | null> {
|
||||||
|
const base = path.join(entityDir, 'vectors.json')
|
||||||
|
try {
|
||||||
|
return JSON.parse(await fs.promises.readFile(base, 'utf-8'))
|
||||||
|
} catch {
|
||||||
|
// fall through to the compressed variant
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const gz = await fs.promises.readFile(`${base}.gz`)
|
||||||
|
return JSON.parse(zlib.gunzipSync(gz).toString('utf-8'))
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Count canonical nouns holding a REAL (non-empty) vector — the vectored-
|
||||||
|
* noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence
|
||||||
|
* cannot be decided from the id-directory listing alone: a deferred-embed
|
||||||
|
* noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`)
|
||||||
|
* until its embed LANDS, so this walk reads every noun's `vectors.json`
|
||||||
|
* CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time
|
||||||
|
* legacy-counts.json derivation or a lost/corrupted counts.json recovery;
|
||||||
|
* the result is persisted so this scan never repeats.
|
||||||
|
*/
|
||||||
|
private async scanVectoredNounCount(): Promise<number> {
|
||||||
|
const base = path.join(this.rootDir, 'entities', 'nouns')
|
||||||
|
let vectored = 0
|
||||||
|
try {
|
||||||
|
const shards = await fs.promises.readdir(base, { withFileTypes: true })
|
||||||
|
for (const shard of shards) {
|
||||||
|
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue
|
||||||
|
const shardPath = path.join(base, shard.name)
|
||||||
|
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
|
||||||
|
for (const entry of ids) {
|
||||||
|
if (!entry.isDirectory()) continue
|
||||||
|
const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name))
|
||||||
|
if (record && Array.isArray(record.vector) && record.vector.length > 0) {
|
||||||
|
vectored++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error: any) {
|
||||||
|
if (error?.code !== 'ENOENT') throw error
|
||||||
|
}
|
||||||
|
return vectored
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Persist counts to filesystem storage
|
* Persist counts to filesystem storage
|
||||||
*/
|
*/
|
||||||
|
|
@ -2744,6 +2829,10 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
// written before the ledger existed; initializeCounts() derives them once.
|
// written before the ledger existed; initializeCounts() derives them once.
|
||||||
totalNounCountAll: this.totalNounCountAll,
|
totalNounCountAll: this.totalNounCountAll,
|
||||||
totalVerbCountAll: this.totalVerbCountAll,
|
totalVerbCountAll: this.totalVerbCountAll,
|
||||||
|
// Vectored-noun ledger scalar — absent in files written before it
|
||||||
|
// existed; initializeCounts() derives it once (a content scan, see
|
||||||
|
// scanVectoredNounCount()'s JSDoc).
|
||||||
|
totalVectoredNounCount: this.totalVectoredNounCount,
|
||||||
allCountsSuspect: this.allCountsSuspect,
|
allCountsSuspect: this.allCountsSuspect,
|
||||||
lastUpdated: new Date().toISOString()
|
lastUpdated: new Date().toISOString()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -520,6 +520,12 @@ export class MemoryStorage extends BaseStorage {
|
||||||
|
|
||||||
let totalNouns = 0
|
let totalNouns = 0
|
||||||
let totalVerbs = 0
|
let totalVerbs = 0
|
||||||
|
// Vectored-noun scalar: unlike the bare presence check above, this needs
|
||||||
|
// the vectors.json RECORD'S content — a deferred-embed noun's record
|
||||||
|
// exists with an empty `vector: []` until its embed lands. In-memory this
|
||||||
|
// is a free field access (no I/O), unlike the filesystem adapter's
|
||||||
|
// per-noun disk read.
|
||||||
|
let totalVectoredNouns = 0
|
||||||
|
|
||||||
// Scan all paths in objectStore
|
// Scan all paths in objectStore
|
||||||
for (const path of this.objectStore.keys()) {
|
for (const path of this.objectStore.keys()) {
|
||||||
|
|
@ -528,6 +534,10 @@ export class MemoryStorage extends BaseStorage {
|
||||||
if (nounMatch) {
|
if (nounMatch) {
|
||||||
// Type is in metadata, not path - just count total
|
// Type is in metadata, not path - just count total
|
||||||
totalNouns++
|
totalNouns++
|
||||||
|
const record = this.objectStore.get(path) as { vector?: unknown } | undefined
|
||||||
|
if (Array.isArray(record?.vector) && record.vector.length > 0) {
|
||||||
|
totalVectoredNouns++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
|
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
|
||||||
|
|
@ -543,6 +553,7 @@ export class MemoryStorage extends BaseStorage {
|
||||||
// A scan of every canonical record IS the ALL-visibility count.
|
// A scan of every canonical record IS the ALL-visibility count.
|
||||||
this.totalNounCountAll = totalNouns
|
this.totalNounCountAll = totalNouns
|
||||||
this.totalVerbCountAll = totalVerbs
|
this.totalVerbCountAll = totalVerbs
|
||||||
|
this.totalVectoredNounCount = totalVectoredNouns
|
||||||
this.allCountsSuspect = false
|
this.allCountsSuspect = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1762,8 +1762,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete a noun from storage
|
* Delete a noun from storage
|
||||||
|
* @param hadVector - OPTIONAL vectored-noun ledger hint, forwarded to
|
||||||
|
* {@link deleteNounMetadata} unchanged — see its JSDoc.
|
||||||
*/
|
*/
|
||||||
public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void> {
|
public async deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
|
// FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
|
||||||
|
|
@ -1780,7 +1782,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// LONGER wrapped in a blind catch that masked faults as "file didn't exist".
|
// LONGER wrapped in a blind catch that masked faults as "file didn't exist".
|
||||||
// `priorMetadata` (the caller's pre-delete read) keeps the decrement honest
|
// `priorMetadata` (the caller's pre-delete read) keeps the decrement honest
|
||||||
// even when the canonical read inside returns null (replace race / ghost).
|
// even when the canonical read inside returns null (replace race / ghost).
|
||||||
await this.deleteNounMetadata(id, priorMetadata)
|
await this.deleteNounMetadata(id, priorMetadata, hadVector)
|
||||||
|
|
||||||
// Remove the now-empty entity container (a no-op for key/prefix stores).
|
// Remove the now-empty entity container (a no-op for key/prefix stores).
|
||||||
await this.removeCanonicalContainer(getNounVectorPath(id))
|
await this.removeCanonicalContainer(getNounVectorPath(id))
|
||||||
|
|
@ -3392,11 +3394,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
/**
|
/**
|
||||||
* Save noun metadata to storage (now typed)
|
* Save noun metadata to storage (now typed)
|
||||||
* Routes to correct sharded location based on UUID
|
* Routes to correct sharded location based on UUID
|
||||||
|
* @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc.
|
||||||
*/
|
*/
|
||||||
public async saveNounMetadata(id: string, metadata: NounMetadata): Promise<void> {
|
public async saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void> {
|
||||||
// Validate noun type in metadata - storage boundary protection
|
// Validate noun type in metadata - storage boundary protection
|
||||||
validateNounType(metadata.noun)
|
validateNounType(metadata.noun)
|
||||||
return this.saveNounMetadata_internal(id, metadata)
|
return this.saveNounMetadata_internal(id, metadata, hasVector)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3407,9 +3410,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
* This ensures counts are updated AFTER metadata exists, fixing the race condition
|
* This ensures counts are updated AFTER metadata exists, fixing the race condition
|
||||||
* where storage adapters tried to read metadata before it was saved.
|
* where storage adapters tried to read metadata before it was saved.
|
||||||
*
|
*
|
||||||
|
* @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc.
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise<void> {
|
protected async saveNounMetadata_internal(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// ID-first path - no type needed!
|
// ID-first path - no type needed!
|
||||||
|
|
@ -3465,6 +3469,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// record persists here so the ALL scalar never lags the tree.
|
// record persists here so the ALL scalar never lags the tree.
|
||||||
if (isNew) {
|
if (isNew) {
|
||||||
this.totalNounCountAll++
|
this.totalNounCountAll++
|
||||||
|
// Vectored-noun ledger: rides the SAME isNew gate (once per id, at
|
||||||
|
// creation) — this seam is metadata-write-driven and never re-runs on
|
||||||
|
// the HNSW neighbor-link re-saves that hit saveNoun_internal, so it
|
||||||
|
// cannot double-count. A deferred-embed insert passes hasVector=false
|
||||||
|
// (or omits it); its vector lands later via noteVectorLanded().
|
||||||
|
if (hasVector) {
|
||||||
|
this.totalVectoredNounCount++
|
||||||
|
}
|
||||||
if (!(metadata.noun && isCounted)) {
|
if (!(metadata.noun && isCounted)) {
|
||||||
this.scheduleCountPersist().catch(() => {
|
this.scheduleCountPersist().catch(() => {
|
||||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||||
|
|
@ -3860,8 +3872,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
* the skip permanently inflated the persisted totals (adds counted, paired
|
* the skip permanently inflated the persisted totals (adds counted, paired
|
||||||
* removals not decremented), and `Math.max(totalNounCount, scanned)` made
|
* removals not decremented), and `Math.max(totalNounCount, scanned)` made
|
||||||
* the inflation unfixable by any disk cleanup.
|
* the inflation unfixable by any disk cleanup.
|
||||||
|
* @param hadVector - OPTIONAL vectored-noun ledger hint — see
|
||||||
|
* {@link StorageAdapter.deleteNounMetadata}'s JSDoc. This method never
|
||||||
|
* reads `vectors.json` to answer the question itself (a canonical read
|
||||||
|
* the delete path must never add); a caller that cannot supply the fact
|
||||||
|
* for free leaves it `undefined`, and the ledger goes SUSPECT rather
|
||||||
|
* than guessing.
|
||||||
*/
|
*/
|
||||||
public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise<void> {
|
public async deleteNounMetadata(
|
||||||
|
id: string,
|
||||||
|
priorRecord?: NounMetadata | null,
|
||||||
|
hadVector?: boolean
|
||||||
|
): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// Direct O(1) delete with ID-first path. Read the canonical record BEFORE
|
// Direct O(1) delete with ID-first path. Read the canonical record BEFORE
|
||||||
|
|
@ -3885,6 +3907,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
} else {
|
} else {
|
||||||
this.markAllCountsSuspect('noun', id)
|
this.markAllCountsSuspect('noun', id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Vectored-noun ledger: a KNOWN vector fact decrements (or no-ops);
|
||||||
|
// an UNKNOWN one goes suspect rather than guessing — see @param hadVector.
|
||||||
|
if (hadVector === true) {
|
||||||
|
if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount--
|
||||||
|
else this.markAllCountsSuspect('noun-vector', id)
|
||||||
|
} else if (hadVector === undefined) {
|
||||||
|
this.markAllCountsSuspect('noun-vector', id)
|
||||||
|
}
|
||||||
this.scheduleCountPersist().catch(() => {
|
this.scheduleCountPersist().catch(() => {
|
||||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||||
})
|
})
|
||||||
|
|
@ -4549,6 +4580,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// construction.
|
// construction.
|
||||||
let allNouns = 0
|
let allNouns = 0
|
||||||
let allVerbs = 0
|
let allVerbs = 0
|
||||||
|
// Vectored-noun scalar: unlike `allNouns` (decided from the metadata.json
|
||||||
|
// LISTING alone), presence cannot be decided from the vectors.json
|
||||||
|
// listing alone — a deferred-embed noun's vectors.json EXISTS with an
|
||||||
|
// empty `vector: []` until its embed lands, so the file's CONTENT must be
|
||||||
|
// read. This walk already lists every path per shard (including
|
||||||
|
// vectors.json entries — `listCanonicalObjects` yields both legs), so
|
||||||
|
// reading them here costs one EXTRA read per noun beyond the metadata.json
|
||||||
|
// read above (doubling this walk's per-noun I/O) — honest cost, paid only
|
||||||
|
// by this diagnostic/repair recount, never on the hot path.
|
||||||
|
let allVectoredNouns = 0
|
||||||
|
|
||||||
// Scan noun shards
|
// Scan noun shards
|
||||||
for (let shard = 0; shard < 256; shard++) {
|
for (let shard = 0; shard < 256; shard++) {
|
||||||
|
|
@ -4559,6 +4600,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const paths = await this.listCanonicalObjects(shardDir)
|
const paths = await this.listCanonicalObjects(shardDir)
|
||||||
|
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
|
if (path.includes('/vectors.json')) {
|
||||||
|
try {
|
||||||
|
const vectorRecord = await this.readCanonicalObject(path)
|
||||||
|
if (vectorRecord && Array.isArray(vectorRecord.vector) && vectorRecord.vector.length > 0) {
|
||||||
|
allVectoredNouns++
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
// Skip vector records that fail to load — best-effort ground truth,
|
||||||
|
// same as the metadata read below.
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
if (!path.includes('/metadata.json')) continue
|
if (!path.includes('/metadata.json')) continue
|
||||||
allNouns++
|
allNouns++
|
||||||
|
|
||||||
|
|
@ -4634,17 +4687,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// IS the proof an unprovable delete could not give.
|
// IS the proof an unprovable delete could not give.
|
||||||
const nounsAllBefore = this.totalNounCountAll
|
const nounsAllBefore = this.totalNounCountAll
|
||||||
const verbsAllBefore = this.totalVerbCountAll
|
const verbsAllBefore = this.totalVerbCountAll
|
||||||
|
const vectoredBefore = this.totalVectoredNounCount
|
||||||
this.totalNounCountAll = allNouns
|
this.totalNounCountAll = allNouns
|
||||||
this.totalVerbCountAll = allVerbs
|
this.totalVerbCountAll = allVerbs
|
||||||
|
this.totalVectoredNounCount = allVectoredNouns
|
||||||
this.allCountsSuspect = false
|
this.allCountsSuspect = false
|
||||||
this.countCache.clear()
|
this.countCache.clear()
|
||||||
await this.persistCounts()
|
await this.persistCounts()
|
||||||
|
|
||||||
prodLog.info(
|
prodLog.info(
|
||||||
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
||||||
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
|
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs / ${allVectoredNouns} vectored nouns` +
|
||||||
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
|
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs || vectoredBefore !== allVectoredNouns
|
||||||
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
|
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore} / ${vectoredBefore})`
|
||||||
: ' (unchanged)') +
|
: ' (unchanged)') +
|
||||||
` — scalar + per-type persisted`
|
` — scalar + per-type persisted`
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -52,7 +52,15 @@ export class SaveNounMetadataOperation implements Operation {
|
||||||
private readonly storage: StorageAdapter,
|
private readonly storage: StorageAdapter,
|
||||||
private readonly id: string,
|
private readonly id: string,
|
||||||
private readonly metadata: NounMetadata,
|
private readonly metadata: NounMetadata,
|
||||||
private readonly isNew: boolean = false
|
private readonly isNew: boolean = false,
|
||||||
|
/**
|
||||||
|
* OPTIONAL vectored-noun ledger hint: `true` when this write's paired
|
||||||
|
* vector (the SAME insert's `vector` local) is real/non-empty — see
|
||||||
|
* {@link StorageAdapter.saveNounMetadata}'s JSDoc for the isNew-gated,
|
||||||
|
* double-count-proof seam this rides. Default `false`: a deferred-embed
|
||||||
|
* insert (or any caller that doesn't know) never counts here.
|
||||||
|
*/
|
||||||
|
private readonly hasVector: boolean = false
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async execute(): Promise<RollbackAction> {
|
async execute(): Promise<RollbackAction> {
|
||||||
|
|
@ -62,7 +70,7 @@ export class SaveNounMetadataOperation implements Operation {
|
||||||
: await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')
|
: await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')
|
||||||
|
|
||||||
// Save new metadata
|
// Save new metadata
|
||||||
await this.storage.saveNounMetadata(this.id, this.metadata)
|
await this.storage.saveNounMetadata(this.id, this.metadata, this.hasVector)
|
||||||
|
|
||||||
// Return rollback action
|
// Return rollback action
|
||||||
return async () => {
|
return async () => {
|
||||||
|
|
@ -70,8 +78,10 @@ export class SaveNounMetadataOperation implements Operation {
|
||||||
// Restore previous metadata
|
// Restore previous metadata
|
||||||
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
||||||
} else {
|
} else {
|
||||||
// Delete newly created metadata
|
// Delete newly created metadata. `this.hasVector` is the SAME fact
|
||||||
await this.storage.deleteNounMetadata(this.id)
|
// this operation's own execute() used to (maybe) count the vectored
|
||||||
|
// ledger — reversing with it on rollback needs no new read.
|
||||||
|
await this.storage.deleteNounMetadata(this.id, undefined, this.hasVector)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -140,7 +150,9 @@ export class SaveNounOperation implements Operation {
|
||||||
// Note: Not all adapters implement deleteNoun
|
// Note: Not all adapters implement deleteNoun
|
||||||
// This is acceptable - metadata deletion makes entity invisible
|
// This is acceptable - metadata deletion makes entity invisible
|
||||||
if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') {
|
if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') {
|
||||||
await this.storage.deleteNoun(this.noun.id)
|
// `this.noun.vector` is the SAME record just written — the
|
||||||
|
// vectored-noun ledger fact is free (no added read) and exact.
|
||||||
|
await this.storage.deleteNoun(this.noun.id, undefined, this.noun.vector.length > 0)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -198,14 +210,21 @@ export class DeleteNounMetadataOperation implements Operation {
|
||||||
return async () => {}
|
return async () => {}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Vectored-noun ledger fact: `previousNoun` is already read above for the
|
||||||
|
// before-image capture — no added read. `undefined` (noun genuinely
|
||||||
|
// absent, metadata-only ghost) is passed through honestly; the storage
|
||||||
|
// layer marks the ledger suspect rather than guessing.
|
||||||
|
const hadVector = previousNoun ? previousNoun.vector.length > 0 : undefined
|
||||||
|
|
||||||
// Full removal: both canonical legs + the entity container + count decrement
|
// Full removal: both canonical legs + the entity container + count decrement
|
||||||
// (the prior record keeps the decrement honest on a null canonical read).
|
// (the prior record keeps the decrement honest on a null canonical read).
|
||||||
await this.storage.deleteNoun(this.id, previousMetadata)
|
await this.storage.deleteNoun(this.id, previousMetadata, hadVector)
|
||||||
|
|
||||||
// Return rollback action
|
// Return rollback action
|
||||||
return async () => {
|
return async () => {
|
||||||
// Restore the vector leg, then the metadata leg through the count-aware
|
// Restore the vector leg, then the metadata leg through the count-aware
|
||||||
// save so deleteNoun()'s decrement is reversed.
|
// save so deleteNoun()'s decrement is reversed (hadVector's mirror:
|
||||||
|
// re-increments the vectored ledger iff the restored vector is real).
|
||||||
if (previousNoun) {
|
if (previousNoun) {
|
||||||
await this.storage.saveNoun({
|
await this.storage.saveNoun({
|
||||||
id: previousNoun.id,
|
id: previousNoun.id,
|
||||||
|
|
@ -215,7 +234,7 @@ export class DeleteNounMetadataOperation implements Operation {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (previousMetadata) {
|
if (previousMetadata) {
|
||||||
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
await this.storage.saveNounMetadata(this.id, previousMetadata, hadVector === true)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -17,10 +17,11 @@
|
||||||
* (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is
|
* (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is
|
||||||
* upgraded from the canonical id tree at open, then persisted.
|
* upgraded from the canonical id tree at open, then persisted.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
import * as fs from 'node:fs'
|
import * as fs from 'node:fs'
|
||||||
import * as os from 'node:os'
|
import * as os from 'node:os'
|
||||||
import * as path from 'node:path'
|
import * as path from 'node:path'
|
||||||
|
import * as zlib from 'node:zlib'
|
||||||
import { Brainy } from '../../src/index.js'
|
import { Brainy } from '../../src/index.js'
|
||||||
|
|
||||||
/** Count canonical `<root>/entities/<kind>/<shard>/<id>` directories — every tier. */
|
/** Count canonical `<root>/entities/<kind>/<shard>/<id>` directories — every tier. */
|
||||||
|
|
@ -180,3 +181,176 @@ describe('canonical count ledger — ALL-visibility scalars, unclamped totals, r
|
||||||
expect(ledger.nouns.all).toBe(truth)
|
expect(ledger.nouns.all).toBe(truth)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
/** Count `<root>/entities/nouns/<shard>/<id>/vectors.json[.gz]` files holding a non-empty `vector`. */
|
||||||
|
function countVectoredNouns(root: string): number {
|
||||||
|
const base = path.join(root, 'entities', 'nouns')
|
||||||
|
if (!fs.existsSync(base)) return 0
|
||||||
|
let n = 0
|
||||||
|
for (const shard of fs.readdirSync(base)) {
|
||||||
|
const shardDir = path.join(base, shard)
|
||||||
|
if (!fs.statSync(shardDir).isDirectory()) continue
|
||||||
|
for (const id of fs.readdirSync(shardDir)) {
|
||||||
|
const idDir = path.join(shardDir, id)
|
||||||
|
if (!fs.statSync(idDir).isDirectory()) continue
|
||||||
|
const plainPath = path.join(idDir, 'vectors.json')
|
||||||
|
const gzPath = `${plainPath}.gz`
|
||||||
|
let record: any = null
|
||||||
|
if (fs.existsSync(plainPath)) {
|
||||||
|
record = JSON.parse(fs.readFileSync(plainPath, 'utf-8'))
|
||||||
|
} else if (fs.existsSync(gzPath)) {
|
||||||
|
record = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf-8'))
|
||||||
|
} else {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (Array.isArray(record.vector) && record.vector.length > 0) n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s coverage denominator)', () => {
|
||||||
|
let dir: string
|
||||||
|
let brain: any
|
||||||
|
|
||||||
|
const open = async () => {
|
||||||
|
const b: any = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
silent: true,
|
||||||
|
dimensions: 384
|
||||||
|
})
|
||||||
|
await b.init()
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Baseline vectored count right after a fresh open() — init() creates a
|
||||||
|
* hidden system VFS-root noun that itself carries a real vector, so a
|
||||||
|
* brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off
|
||||||
|
* this baseline rather than hardcoding it away. */
|
||||||
|
let baseline: number
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||||
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-'))
|
||||||
|
brain = await open()
|
||||||
|
baseline = (await brain.storage.getCanonicalCounts()).vectors.all
|
||||||
|
})
|
||||||
|
afterEach(async () => {
|
||||||
|
vi.restoreAllMocks()
|
||||||
|
await brain.close?.().catch(() => {})
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an explicit-vector add counts immediately; the ledger matches the on-disk vectors.json content', async () => {
|
||||||
|
await brain.add({ data: 'a', type: 'document', vector: Array(384).fill(0).map((_, i) => Math.sin(i)) })
|
||||||
|
await brain.add({ data: 'b', type: 'document' }) // embedded (non-deferred) — also a real vector
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
const ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.vectors.all).toBe(baseline + 2)
|
||||||
|
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||||
|
expect(ledger.suspect).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a deferred-embed add does NOT count until its embed LANDS', async () => {
|
||||||
|
// Hold the background worker's embed call open under manual control — a
|
||||||
|
// deterministic embedder is fast enough that the landing could otherwise
|
||||||
|
// race ahead of the "still unlanded" assertion below.
|
||||||
|
let resolveEmbed: ((v: number[]) => void) | undefined
|
||||||
|
vi.spyOn(brain, 'embed').mockImplementation(
|
||||||
|
() => new Promise<number[]>((resolve) => { resolveEmbed = resolve })
|
||||||
|
)
|
||||||
|
|
||||||
|
const id = await brain.add({ data: 'deferred content', type: 'document', deferEmbedding: true })
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
// Landed nothing yet — the ledger must not count the stub.
|
||||||
|
let ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.vectors.all).toBe(baseline)
|
||||||
|
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||||
|
|
||||||
|
// Release the held embed, then cross the barrier: the vector lands
|
||||||
|
// (system:embed-landing).
|
||||||
|
resolveEmbed!(Array(384).fill(0).map((_, i) => Math.cos(i)))
|
||||||
|
await brain.awaitPendingEmbeds()
|
||||||
|
const landed = await brain.get(id, { includeVectors: true })
|
||||||
|
expect((landed!.vector as number[]).length).toBeGreaterThan(0)
|
||||||
|
|
||||||
|
ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.vectors.all).toBe(baseline + 1)
|
||||||
|
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||||
|
expect(ledger.suspect).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a proven delete of a vectored noun decrements; a non-vectored (unlanded) delete does not', async () => {
|
||||||
|
const vectoredId = await brain.add({ data: 'v', type: 'document' }) // real embed, unmocked
|
||||||
|
// Block the embed worker AFTER the real add above — a deterministic
|
||||||
|
// embedder is fast enough that the deferred noun below could otherwise
|
||||||
|
// land before this test observes its "still unlanded" state.
|
||||||
|
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
||||||
|
const deferredId = await brain.add({ data: 'd', type: 'document', deferEmbedding: true })
|
||||||
|
await brain.flush()
|
||||||
|
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(baseline + 1)
|
||||||
|
|
||||||
|
await brain.remove(vectoredId)
|
||||||
|
await brain.flush()
|
||||||
|
let ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.vectors.all).toBe(baseline)
|
||||||
|
expect(ledger.suspect).toBe(false)
|
||||||
|
|
||||||
|
await brain.remove(deferredId) // never had a real vector — no decrement, still unsuspect
|
||||||
|
await brain.flush()
|
||||||
|
ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.vectors.all).toBe(baseline)
|
||||||
|
expect(ledger.suspect).toBe(false)
|
||||||
|
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the recount corrects a tampered vectors.all scalar, surviving reopen', async () => {
|
||||||
|
await brain.add({ data: 'real 1', type: 'document' })
|
||||||
|
await brain.add({ data: 'real 2', type: 'document' })
|
||||||
|
await brain.flush()
|
||||||
|
const truth = countVectoredNouns(dir)
|
||||||
|
expect(truth).toBe(baseline + 2)
|
||||||
|
|
||||||
|
;(brain.storage as any).totalVectoredNounCount = truth + 40
|
||||||
|
await (brain.storage as any).persistCounts()
|
||||||
|
await brain.close()
|
||||||
|
brain = await open()
|
||||||
|
|
||||||
|
// The lie survives reopen (never clamped).
|
||||||
|
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth + 40)
|
||||||
|
|
||||||
|
await brain.repairIndex()
|
||||||
|
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth)
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
brain = await open()
|
||||||
|
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a legacy counts.json without totalVectoredNounCount is derived once from vectors.json content and persisted', async () => {
|
||||||
|
await brain.add({ data: 'one', type: 'document' }) // real embed, unmocked
|
||||||
|
// Block the embed worker AFTER the real add above — a deterministic
|
||||||
|
// embedder is fast enough that the deferred noun below could otherwise
|
||||||
|
// land before close(), which would inflate this test's expected count.
|
||||||
|
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
||||||
|
await brain.add({ data: 'two deferred', type: 'document', deferEmbedding: true })
|
||||||
|
await brain.flush()
|
||||||
|
await brain.close()
|
||||||
|
|
||||||
|
const countsPath = path.join(dir, '_system', 'counts.json')
|
||||||
|
const raw = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
||||||
|
expect(typeof raw.totalVectoredNounCount).toBe('number')
|
||||||
|
delete raw.totalVectoredNounCount
|
||||||
|
fs.writeFileSync(countsPath, JSON.stringify(raw, null, 2))
|
||||||
|
|
||||||
|
brain = await open()
|
||||||
|
const ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun
|
||||||
|
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
||||||
|
const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
||||||
|
expect(persisted.totalVectoredNounCount).toBe(baseline + 1)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
|
||||||
247
tests/integration/vector-leg-open-build.test.ts
Normal file
247
tests/integration/vector-leg-open-build.test.ts
Normal file
|
|
@ -0,0 +1,247 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/vector-leg-open-build
|
||||||
|
* @description THE LAST RED of the two-engine release gate: a migrated
|
||||||
|
* store can hold canonical vectored nouns with NO derived vector index
|
||||||
|
* built. `open()` owns building the derived indexes (reads never build —
|
||||||
|
* see `rebuildIndexesIfNeeded`'s JSDoc); the defect this pins is the vector
|
||||||
|
* leg's decision silently skipping that build, so `find`/search served `[]`
|
||||||
|
* with no error and no narration.
|
||||||
|
*
|
||||||
|
* A downstream deployment measures this through a native vector provider
|
||||||
|
* whose own health report can legitimately say `serving: true` even while
|
||||||
|
* vector COVERAGE is honestly unledgered on its side (an unledgered
|
||||||
|
* invariant never flips `serving` — see `HealthReport`'s derivation laws).
|
||||||
|
* This repo ships only the JS engine, so the reproduction here uses the
|
||||||
|
* SAME plugin seam a native provider would (`brain.use({ activate: ctx =>
|
||||||
|
* ctx.registerProvider('vector', factory) })`, the pattern
|
||||||
|
* `tests/unit/cold-open-rebuild-gate.test.ts` already established for this
|
||||||
|
* exact class of gate-decision bug) with a stub that WRAPS the real
|
||||||
|
* `JsHnswVectorIndex` — every method delegates to a genuine engine (so a
|
||||||
|
* successful rebuild restores REAL, searchable vectors), except `size()`
|
||||||
|
* (fakes 0 until rebuild runs — the "never built" posture) and
|
||||||
|
* `healthReport()` (always reports `serving: true`, `unledgered:
|
||||||
|
* ['vector-coverage']` — the "I don't track this yet" posture). This is
|
||||||
|
* "as close as the JS engine allows": the gap is reproduced at the exact
|
||||||
|
* decision the fix changes, not approximated by deleting files the JS
|
||||||
|
* engine's own cold-start heuristic already recovers from unaided (see the
|
||||||
|
* inverse pin below and cold-open-rebuild-gate.test.ts's already-pinned
|
||||||
|
* "isReady()===true, size()===0" contract, which this fix deliberately does
|
||||||
|
* NOT touch — bare isReady() has no unledgered concept to hide behind, and
|
||||||
|
* overriding it would reopen the 48-seconds-per-restart regression pinned
|
||||||
|
* there).
|
||||||
|
*
|
||||||
|
* Pins:
|
||||||
|
* (1) COVERAGE GAP FORCES THE BUILD: N vectored nouns, a provider that
|
||||||
|
* claims `serving: true` at `size()===0` — open() builds anyway (the
|
||||||
|
* ledger proves there is something to cover), and search returns real
|
||||||
|
* results, never `[]`.
|
||||||
|
* (2) THE INVERSE, HONEST EMPTY: 0 vectored nouns (every embed still
|
||||||
|
* deferred/unlanded) — open() does NOT attempt a rebuild (nothing to
|
||||||
|
* load; the old blunt "always rebuild when size()===0" heuristic wasted
|
||||||
|
* a full canonical walk here for zero benefit), and search honestly
|
||||||
|
* returns `[]` — no error, no false coverage-gap narration.
|
||||||
|
*
|
||||||
|
* SEARCH VERIFICATION NOTE: pin (1) verifies "search returns real results"
|
||||||
|
* via `find({ query: <text> })` (semantic search — embeds the query, then
|
||||||
|
* searches), matching the pattern `tests/integration/hnsw-rebuild.test.ts`
|
||||||
|
* already uses for exactly this "post-rebuild search works" class of pin.
|
||||||
|
* A raw `find({ vector: <array> })` / `index.search(vector, k)` call was
|
||||||
|
* tried first and found to reproducibly return only 1 hit after a
|
||||||
|
* FROM-CANONICAL rebuild (never the full requested `limit`, sometimes not
|
||||||
|
* even a real neighbor) — REGARDLESS of this task's changes: it reproduces
|
||||||
|
* identically on a plain, unwrapped, un-stubbed reopen with the stock JS
|
||||||
|
* engine (verified against `hnsw-rebuild.test.ts`'s own construction) and
|
||||||
|
* is therefore a PRE-EXISTING, orthogonal defect in the JS HNSW engine's
|
||||||
|
* rebuilt-graph connectivity — outside this task's two deliverables (the
|
||||||
|
* count ledger and the open-gate REBUILD DECISION, not rebuild()'s internal
|
||||||
|
* search quality). Left for a separate investigation; not touched here.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, afterEach, vi } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { Brainy } from '../../src/index.js'
|
||||||
|
import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js'
|
||||||
|
|
||||||
|
const tmpDirs: string[] = []
|
||||||
|
function mkTmp(): string {
|
||||||
|
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vector-leg-open-'))
|
||||||
|
tmpDirs.push(d)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks()
|
||||||
|
for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
const V = (seed: number) =>
|
||||||
|
Array.from({ length: 384 }, (_, i) => Math.sin((seed + 1) * 7919 + i * 131) * 0.5 + 0.5)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a store with N explicit-vector (non-deferred) nouns, flush, close.
|
||||||
|
* Each noun also carries embeddable text (`technology`/`science`, matching
|
||||||
|
* the query used below) so the semantic-search verification exercises real
|
||||||
|
* retrieval, not a coincidental match. The default JS engine builds a fully
|
||||||
|
* current store — the epoch marker is stamped current at this open's
|
||||||
|
* completion, so a later reopen's `_indexEpochStale` is honestly false and
|
||||||
|
* cannot mask the ledger-gap decision under test (nothing here manufactures
|
||||||
|
* epoch drift).
|
||||||
|
*/
|
||||||
|
async function buildVectoredStore(dir: string, n: number): Promise<string[]> {
|
||||||
|
const brain: any = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
plugins: [],
|
||||||
|
silent: true,
|
||||||
|
dimensions: 384
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
const ids: string[] = []
|
||||||
|
for (let i = 0; i < n; i++) {
|
||||||
|
ids.push(
|
||||||
|
await brain.add({
|
||||||
|
data: `doc ${i} about ${i % 2 === 0 ? 'technology' : 'science'}`,
|
||||||
|
type: 'document',
|
||||||
|
vector: V(i)
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
await brain.flush()
|
||||||
|
await brain.close()
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('vector-leg open-build (two-engine gate, last red)', () => {
|
||||||
|
it('coverage gap: a provider reporting serving:true at size()===0 is overridden by the vectored-noun ledger — open() builds, search returns real results', async () => {
|
||||||
|
const dir = mkTmp()
|
||||||
|
const ids = await buildVectoredStore(dir, 12)
|
||||||
|
|
||||||
|
// Wrap the REAL JS engine so a successful rebuild restores genuine,
|
||||||
|
// searchable vectors — only `size()` and `healthReport()` are faked,
|
||||||
|
// simulating a native provider that has never built its own coverage of
|
||||||
|
// an unledgered invariant.
|
||||||
|
const calls = { rebuild: 0 }
|
||||||
|
const brain: any = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
plugins: [],
|
||||||
|
silent: true,
|
||||||
|
dimensions: 384
|
||||||
|
})
|
||||||
|
brain.use({
|
||||||
|
name: 'fake-native-vector-unledgered-coverage',
|
||||||
|
activate: async (ctx: any) => {
|
||||||
|
ctx.registerProvider('vector', (config: any, distance: any, options: any) => {
|
||||||
|
const real = new JsHnswVectorIndex(config, distance, options)
|
||||||
|
let rebuilt = false
|
||||||
|
const originalRebuild = real.rebuild.bind(real)
|
||||||
|
;(real as any).rebuild = async (...args: any[]) => {
|
||||||
|
const r = await originalRebuild(...args)
|
||||||
|
calls.rebuild++
|
||||||
|
rebuilt = true
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
const originalSize = real.size.bind(real)
|
||||||
|
;(real as any).size = () => (rebuilt ? originalSize() : 0)
|
||||||
|
;(real as any).healthReport = () => ({
|
||||||
|
provider: 'vector',
|
||||||
|
healthy: true,
|
||||||
|
serving: true,
|
||||||
|
invariants: [],
|
||||||
|
checkedAt: Date.now(),
|
||||||
|
durationMs: 0,
|
||||||
|
generation: 1,
|
||||||
|
unledgered: ['vector-coverage']
|
||||||
|
})
|
||||||
|
return real
|
||||||
|
})
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
await brain.init()
|
||||||
|
|
||||||
|
// WITHOUT any find() first: open() itself must have built the leg.
|
||||||
|
expect(calls.rebuild, 'open() forced the rebuild despite serving:true').toBe(1)
|
||||||
|
const status = await brain.getIndexStatus()
|
||||||
|
expect(status.hnswIndex.size).toBeGreaterThanOrEqual(ids.length)
|
||||||
|
|
||||||
|
// Real, searchable results — never [] (see the module doc's SEARCH
|
||||||
|
// VERIFICATION NOTE for why this is a semantic `query`, not a raw
|
||||||
|
// `vector`, call).
|
||||||
|
const results = await brain.find({ query: 'technology document', limit: 5 })
|
||||||
|
expect(results.length).toBeGreaterThan(0)
|
||||||
|
expect(results.length).not.toBe(0)
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => {
|
||||||
|
// ARCHITECTURAL NOTE (found while building this pin): every brainy store
|
||||||
|
// carries ONE permanent, always-vectored noun beyond user data — the VFS
|
||||||
|
// root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`,
|
||||||
|
// src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero
|
||||||
|
// (but non-empty, length-384) vector on EVERY store's first open — never
|
||||||
|
// deferred (a deliberate WASM-cold-compile-avoidance fix, see that
|
||||||
|
// file's comment) — and VFS init unconditionally re-creates it if
|
||||||
|
// missing, before the rebuild gate ever runs. A literal "0 vectored
|
||||||
|
// nouns" store is therefore unreachable through the public API; a
|
||||||
|
// brand-new store's `vectors.all` floor is 1, not 0. This pin verifies
|
||||||
|
// the law the task names in the ACHIEVABLE form: nouns whose embed is
|
||||||
|
// still deferred/unlanded contribute NOTHING to the vectored-noun ledger
|
||||||
|
// — the coverage-gap comparison sees exactly the root (1), never
|
||||||
|
// root+deferred — and semantic search over deferred-only user content
|
||||||
|
// honestly returns `[]` (no error, no false "coverage restored" claim).
|
||||||
|
const dir = mkTmp()
|
||||||
|
|
||||||
|
const build: any = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
plugins: [],
|
||||||
|
silent: true,
|
||||||
|
dimensions: 384
|
||||||
|
})
|
||||||
|
await build.init()
|
||||||
|
const rootOnlyLedger = await build.storage.getCanonicalCounts()
|
||||||
|
// Block the embedder permanently so every add below stays deferred and
|
||||||
|
// unlanded for the rest of this test (a fast deterministic embedder
|
||||||
|
// could otherwise land it before we ever observe the "still 0 extra"
|
||||||
|
// state).
|
||||||
|
vi.spyOn(build, 'embed').mockImplementation(() => new Promise(() => {}))
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
await build.add({ data: `deferred ${i}`, type: 'document', deferEmbedding: true })
|
||||||
|
}
|
||||||
|
await build.flush()
|
||||||
|
const ledgerWithDeferred = await build.storage.getCanonicalCounts()
|
||||||
|
// The five deferred adds contributed ZERO to the vectored-noun ledger.
|
||||||
|
expect(ledgerWithDeferred.vectors.all).toBe(rootOnlyLedger.vectors.all)
|
||||||
|
await build.close()
|
||||||
|
|
||||||
|
// Reopen (default JS engine — no stub needed): the root is the ONLY
|
||||||
|
// thing the vector leg has to load; the deferred nouns are correctly
|
||||||
|
// invisible to it. Block the embedder again BEFORE init() — reopen
|
||||||
|
// recovers the durable pending-embed markers and kicks the worker as
|
||||||
|
// part of init() itself, and an unblocked deterministic embedder could
|
||||||
|
// land all five before this test observes the open-time ledger.
|
||||||
|
const brain: any = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
plugins: [],
|
||||||
|
silent: true,
|
||||||
|
dimensions: 384
|
||||||
|
})
|
||||||
|
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
||||||
|
await brain.init()
|
||||||
|
// The ledger is exactly the root — the five deferred, still-unlanded
|
||||||
|
// nouns (which the rebuild above DOES insert into the graph, each with
|
||||||
|
// its stub empty vector — `index.size()` counts EVERY canonical noun's
|
||||||
|
// graph node, deferred or not, so it is not the coverage metric) never
|
||||||
|
// inflate the VECTORED count.
|
||||||
|
const ledgerAfterReopen = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledgerAfterReopen.vectors.all).toBe(rootOnlyLedger.vectors.all)
|
||||||
|
|
||||||
|
const results = await brain.find({ vector: V(3), limit: 5 })
|
||||||
|
expect(results).toEqual([])
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
@ -380,6 +380,12 @@ describe.sequential('lifecycle — the working store', () => {
|
||||||
counted: aliveVerbs + model.vfsContainsVerbs,
|
counted: aliveVerbs + model.vfsContainsVerbs,
|
||||||
all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs
|
all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs
|
||||||
},
|
},
|
||||||
|
// Every noun this biography ever adds carries an explicit/computed
|
||||||
|
// vector (the harness never defers an embed), so the vectored-noun
|
||||||
|
// scalar tracks nouns.all exactly.
|
||||||
|
vectors: {
|
||||||
|
all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns
|
||||||
|
},
|
||||||
suspect: false
|
suspect: false
|
||||||
})
|
})
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue