Compare commits
No commits in common. "v10.4.0-rc.4" and "v10.4.0-rc.3" have entirely different histories.
v10.4.0-rc
...
v10.4.0-rc
13 changed files with 35 additions and 859 deletions
|
|
@ -2,11 +2,6 @@
|
||||||
|
|
||||||
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
|
||||||
|
|
||||||
### [10.4.0-rc.4](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.3...v10.4.0-rc.4) (2026-08-25)
|
|
||||||
|
|
||||||
- feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg (9730835b)
|
|
||||||
|
|
||||||
|
|
||||||
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
|
### [10.4.0-rc.3](https://source.soulcraft.com/soulcraft/brainy/compare/v10.4.0-rc.2...v10.4.0-rc.3) (2026-08-25)
|
||||||
|
|
||||||
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
|
- fix(update-seam): the metadata crossing never carries BigInt endpoint ints (f4780c8e)
|
||||||
|
|
|
||||||
4
package-lock.json
generated
4
package-lock.json
generated
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.4",
|
"version": "10.4.0-rc.3",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.4",
|
"version": "10.4.0-rc.3",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@msgpack/msgpack": "^3.1.2",
|
"@msgpack/msgpack": "^3.1.2",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "@soulcraft/brainy",
|
"name": "@soulcraft/brainy",
|
||||||
"version": "10.4.0-rc.4",
|
"version": "10.4.0-rc.3",
|
||||||
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
"description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"module": "dist/index.js",
|
"module": "dist/index.js",
|
||||||
|
|
|
||||||
145
src/brainy.ts
145
src/brainy.ts
|
|
@ -2382,17 +2382,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
[{ type: 'embed.landed', id, vector: newVector }],
|
[{ type: 'embed.landed', id, vector: newVector }],
|
||||||
'system:embed-landing'
|
'system:embed-landing'
|
||||||
)
|
)
|
||||||
// Vectored-noun ledger: the landing commit above carries a vector
|
|
||||||
// write with NO accompanying metadata operation, so the
|
|
||||||
// saveNounMetadata(..., hasVector) seam never fires for it — the
|
|
||||||
// narrow storage hook is the only seam left. `oldVector.length===0`
|
|
||||||
// (already known for free from the pre-embed read above) proves this
|
|
||||||
// is a GENUINE first landing, not a re-embed of an already-vectored
|
|
||||||
// row (e.g. a deferred update() on a row that already had a real
|
|
||||||
// vector) — the latter must never double-count.
|
|
||||||
if (oldVector.length === 0) {
|
|
||||||
await this.storage.noteVectorLanded?.(id)
|
|
||||||
}
|
|
||||||
this.clearPendingEmbed(id)
|
this.clearPendingEmbed(id)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
prodLog.warn(
|
prodLog.warn(
|
||||||
|
|
@ -3011,11 +3000,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
const runInsert: TransactionFunction<void> = async (tx) => {
|
const runInsert: TransactionFunction<void> = async (tx) => {
|
||||||
// Operation 1: Save metadata FIRST (TypeAwareStorage caching)
|
// Operation 1: Save metadata FIRST (TypeAwareStorage caching)
|
||||||
// isNew=true: skip pre-read for rollback (entity doesn't exist yet)
|
// isNew=true: skip pre-read for rollback (entity doesn't exist yet)
|
||||||
// hasVector: the vectored-noun ledger counts this insert iff its
|
|
||||||
// vector is real/non-empty (never true for a deferred embed, whose
|
|
||||||
// stub `vector` is `[]` — it counts later, at landing).
|
|
||||||
tx.addOperation(
|
tx.addOperation(
|
||||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, true, vector.length > 0)
|
new SaveNounMetadataOperation(this.storage, id, storageMetadata, true)
|
||||||
)
|
)
|
||||||
|
|
||||||
// Operation 2: Save vector data
|
// Operation 2: Save vector data
|
||||||
|
|
@ -10582,9 +10568,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
plan.postCommit.push(() => this.kickEmbedWorker())
|
plan.postCommit.push(() => this.kickEmbedWorker())
|
||||||
}
|
}
|
||||||
plan.operations.push(
|
plan.operations.push(
|
||||||
// hasVector: see the single-add() insert path's comment — never true
|
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew),
|
||||||
// for a deferred embed (stub vector `[]`; counted later at landing).
|
|
||||||
new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew, vector.length > 0),
|
|
||||||
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
|
new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew),
|
||||||
...(deferringEmbed
|
...(deferringEmbed
|
||||||
? []
|
? []
|
||||||
|
|
@ -16916,92 +16900,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
const shouldRebuildMetadata =
|
const shouldRebuildMetadata =
|
||||||
!metadataMigrating &&
|
!metadataMigrating &&
|
||||||
(epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0))
|
(epochStale || legNeedsRebuild(this.metadataIndex, metadataStats.totalEntries === 0))
|
||||||
|
const shouldRebuildVector =
|
||||||
// VECTOR LEG — the two-engine gate's last red: a migrated 7.x-era store
|
|
||||||
// can hold canonical vectored nouns with NO derived vector index built.
|
|
||||||
// `legNeedsRebuild`'s size-heuristic fallback (below) only fires off
|
|
||||||
// `hnswIndexSize === 0`, and its health-report branch trusts a
|
|
||||||
// provider's own `serving` verdict verbatim — but a provider's health
|
|
||||||
// report can legitimately say `serving: true` while vector coverage is
|
|
||||||
// honestly UNLEDGERED on ITS side too (an unledgered invariant never
|
|
||||||
// flips serving), so neither signal alone can tell "genuinely empty"
|
|
||||||
// apart from "never built". The canonical vectored-noun ledger
|
|
||||||
// (`getCanonicalCounts().vectors.all` — Deliverable 1) is the
|
|
||||||
// denominator that CAN tell them apart, and is compared here:
|
|
||||||
// - a CONFIDENT (non-suspect) ledger `> 0` while the reported node
|
|
||||||
// count is 0 is a proven coverage gap — force the build regardless
|
|
||||||
// of what a health report claims;
|
|
||||||
// - a CONFIDENT ledger `=== 0` while the node count is 0 proves there
|
|
||||||
// is nothing to load (e.g. every noun's embed is still deferred) —
|
|
||||||
// skip the size-heuristic fallback's blunt "always rebuild when
|
|
||||||
// empty" trigger, which otherwise wastes a full canonical walk for
|
|
||||||
// zero benefit on every cold open of such a store;
|
|
||||||
// - an unavailable/suspect ledger changes nothing — loud errors never
|
|
||||||
// quiet losses, so a doubtful ledger must never suppress a rebuild
|
|
||||||
// the old heuristic would have run.
|
|
||||||
// The bare `isReady()` boolean (no report, no `unledgered` concept) is
|
|
||||||
// NOT overridden — that signal is what fixed the 48-seconds-per-restart
|
|
||||||
// regression pinned in tests/unit/cold-open-rebuild-gate.test.ts (a
|
|
||||||
// disk-native provider legitimately reporting 0 resident while durable
|
|
||||||
// on disk), and re-deriving it from a denominator the provider itself
|
|
||||||
// has no way to consult would reopen exactly that regression.
|
|
||||||
const vectorAssessment = assessProviderHealth(this.index)
|
|
||||||
const vectorLedger = await this.storage.getCanonicalCounts?.()
|
|
||||||
const vectorLedgerAll = vectorLedger?.vectors.all
|
|
||||||
const vectorLedgerConfident = vectorLedger !== undefined && !vectorLedger.suspect
|
|
||||||
const vectorHasCoverageProof = vectorLedgerConfident && (vectorLedgerAll as number) > 0
|
|
||||||
const vectorConfirmedEmpty = vectorLedgerConfident && vectorLedgerAll === 0
|
|
||||||
|
|
||||||
let vectorNeedsRebuild: boolean
|
|
||||||
if (vectorAssessment.via === 'is-ready') {
|
|
||||||
// Bare isReady() stays authoritative and UNMODIFIED — see above.
|
|
||||||
vectorNeedsRebuild = vectorAssessment.readiness === 'not-ready'
|
|
||||||
} else if (vectorAssessment.via === 'health-report') {
|
|
||||||
vectorNeedsRebuild =
|
|
||||||
vectorAssessment.readiness !== 'ready' ||
|
|
||||||
(hnswIndexSize === 0 && vectorHasCoverageProof)
|
|
||||||
} else {
|
|
||||||
// size-heuristic / no provider (the built-in JS engine's own posture).
|
|
||||||
vectorNeedsRebuild = hnswIndexSize === 0 && !vectorConfirmedEmpty
|
|
||||||
}
|
|
||||||
|
|
||||||
const shouldRebuildVector = !vectorMigrating && (epochStale || vectorNeedsRebuild)
|
|
||||||
|
|
||||||
// Narration (and the FAIL-TYPED backstop below) are scoped EXACTLY to
|
|
||||||
// the defect this gate closes: a provider whose OWN health report
|
|
||||||
// claims `serving: true` — an affirmative "I am ready" a caller would
|
|
||||||
// otherwise trust outright — while the canonical ledger proves vector
|
|
||||||
// coverage is missing. This is deliberately NARROWER than "any branch
|
|
||||||
// where the ledger contributed to the decision":
|
|
||||||
// - the bare isReady() branch is untouched, as above (never in scope);
|
|
||||||
// - the health-report branch's OWN `readiness !== 'ready'` case is
|
|
||||||
// already an ordinary, PRE-EXISTING rebuild trigger (the provider
|
|
||||||
// admits not-ready) — not a ledger override, so not a "gap";
|
|
||||||
// - the size-heuristic/no-provider branch's rebuild-when-empty is the
|
|
||||||
// SAME blunt trigger the code always had (`hnswIndexSize === 0`)
|
|
||||||
// — the ledger only ever SUPPRESSES a rebuild there (the confirmed-
|
|
||||||
// empty case), it never forces one the old heuristic wouldn't
|
|
||||||
// already have run. Marking that branch a "gap" too made the
|
|
||||||
// FAIL-TYPED backstop fire on ordinary white-box tests that stub
|
|
||||||
// rebuild() as a no-op and pin `size()` at 0 to drive OTHER
|
|
||||||
// assertions (e.g. migration-deference's isMigrating() coverage) —
|
|
||||||
// those are not silent-empty defects, so they must open exactly as
|
|
||||||
// before (tests/unit/brainy/migration-deference.test.ts).
|
|
||||||
const vectorCoverageGap =
|
|
||||||
!vectorMigrating &&
|
!vectorMigrating &&
|
||||||
vectorAssessment.via === 'health-report' &&
|
(epochStale || legNeedsRebuild(this.index, hnswIndexSize === 0))
|
||||||
vectorAssessment.readiness === 'ready' &&
|
|
||||||
hnswIndexSize === 0 &&
|
|
||||||
vectorHasCoverageProof
|
|
||||||
if (vectorCoverageGap) {
|
|
||||||
prodLog.warn(
|
|
||||||
`[Brainy] open(): vector index reports ${hnswIndexSize} node(s) but the canonical ` +
|
|
||||||
`ledger holds ${vectorLedgerAll} vectored noun(s) — the derived vector index is ` +
|
|
||||||
`missing or unbuilt on this store. Forcing the vector rebuild rather than serving ` +
|
|
||||||
`silent-empty search results.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const shouldRebuildGraph =
|
const shouldRebuildGraph =
|
||||||
!graphMigrating &&
|
!graphMigrating &&
|
||||||
(epochStale || legNeedsRebuild(this.graphIndex, false))
|
(epochStale || legNeedsRebuild(this.graphIndex, false))
|
||||||
|
|
@ -17054,27 +16955,9 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// provider running its own background migration is skipped here (it owns
|
// provider running its own background migration is skipped here (it owns
|
||||||
// its index until it verifies-and-swaps).
|
// its index until it verifies-and-swaps).
|
||||||
const rebuildStartTime = Date.now()
|
const rebuildStartTime = Date.now()
|
||||||
// The vector leg's build door, by contract with the native provider: a
|
|
||||||
// provider exposing fillFromCanonical() gets THAT call — idempotent, the
|
|
||||||
// provider's own init runs it first so this is the backstop — never a
|
|
||||||
// full rebuild() for a coverage gap. A PARTIAL shortfall deliberately
|
|
||||||
// triggers nothing here: that is repair()'s operator door. The JS index
|
|
||||||
// has no fill door and keeps its rebuild.
|
|
||||||
const vectorBuild = (): Promise<unknown> => {
|
|
||||||
const fillDoor = (this.index as unknown as { fillFromCanonical?: () => Promise<unknown> })
|
|
||||||
.fillFromCanonical
|
|
||||||
if (vectorCoverageGap && typeof fillDoor === 'function') {
|
|
||||||
prodLog.warn(
|
|
||||||
`[Brainy] open(): vector coverage gap routes through the provider's ` +
|
|
||||||
`fillFromCanonical() (idempotent canonical fill), not a full rebuild.`
|
|
||||||
)
|
|
||||||
return fillDoor.call(this.index)
|
|
||||||
}
|
|
||||||
return this.index.rebuild()
|
|
||||||
}
|
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(),
|
shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(),
|
||||||
shouldRebuildVector ? vectorBuild() : Promise.resolve(),
|
shouldRebuildVector ? this.index.rebuild() : Promise.resolve(),
|
||||||
shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve()
|
shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve()
|
||||||
])
|
])
|
||||||
|
|
||||||
|
|
@ -17115,24 +16998,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
|
console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Vector coverage verification: the coverage-gap rebuild above (see
|
|
||||||
// `vectorCoverageGap`) MUST have actually restored the ledger's
|
|
||||||
// vectored nouns. A provider that STILL reports 0 nodes after its own
|
|
||||||
// rebuild() ran — no JS (or provider) fallback could build from what's
|
|
||||||
// on disk — cannot silently complete open(): search would then serve
|
|
||||||
// empty results with no signal, exactly the defect this gate closes.
|
|
||||||
// FAIL TYPED, pre-serve, rather than let a broken vector leg pass as a
|
|
||||||
// successful open.
|
|
||||||
if (vectorCoverageGap && this.index.size() === 0) {
|
|
||||||
throw new VectorIndexNotReadyError(
|
|
||||||
`open(): the canonical ledger holds ${vectorLedgerAll} vectored noun(s) but the vector ` +
|
|
||||||
`index still reports 0 node(s) after rebuild() — the derived vector index could not ` +
|
|
||||||
`be restored from canonical. Refusing to serve silent-empty search results; ` +
|
|
||||||
`investigate the vector provider/storage, or repairIndex({ rebuild: ['vector'] }) ` +
|
|
||||||
`after restoring the underlying data.`
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
|
// 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes
|
||||||
// have now rebuilt and verified, so they match this build's epoch —
|
// have now rebuilt and verified, so they match this build's epoch —
|
||||||
// re-stamp the marker LAST, only here. A crash anywhere above leaves the
|
// re-stamp the marker LAST, only here. A crash anywhere above leaves the
|
||||||
|
|
|
||||||
|
|
@ -801,17 +801,6 @@ export interface DerivedFamilyDeclaration {
|
||||||
export interface CanonicalCounts {
|
export interface CanonicalCounts {
|
||||||
nouns: { counted: number; all: number }
|
nouns: { counted: number; all: number }
|
||||||
verbs: { counted: number; all: number }
|
verbs: { counted: number; all: number }
|
||||||
/**
|
|
||||||
* The count of canonical nouns holding a REAL (non-empty) vector — the
|
|
||||||
* coverage denominator a vector index's node-count ledger is measured
|
|
||||||
* against (`nodeCount === vectors.all` is the whole-store coverage
|
|
||||||
* verdict for the vector leg, the vector-side mirror of `nouns.all` for
|
|
||||||
* metadata/graph). A deferred-embed noun (`add({ deferEmbedding: true })`)
|
|
||||||
* counts only once its vector actually LANDS — its canonical record exists
|
|
||||||
* (counted in `nouns.all`) with an empty vector until then, so it is
|
|
||||||
* deliberately NOT counted here in the interim.
|
|
||||||
*/
|
|
||||||
vectors: { all: number }
|
|
||||||
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
||||||
suspect: boolean
|
suspect: boolean
|
||||||
}
|
}
|
||||||
|
|
@ -830,47 +819,14 @@ export interface StorageAdapter {
|
||||||
* Save noun metadata separately
|
* Save noun metadata separately
|
||||||
* @param id Noun ID
|
* @param id Noun ID
|
||||||
* @param metadata Noun metadata
|
* @param metadata Noun metadata
|
||||||
* @param hasVector - OPTIONAL vectored-noun ledger hint: `true` when this
|
|
||||||
* write is a FRESH insert (`isNew`) whose vector is a real, non-empty
|
|
||||||
* array — the caller already knows this for free (the insert's own
|
|
||||||
* `vector` local), so the increment rides the SAME isNew gate that
|
|
||||||
* already protects `totalNounCountAll` from double-counting on HNSW
|
|
||||||
* neighbor-link re-saves (`saveNoun_internal` re-runs on every link
|
|
||||||
* change; this metadata seam does not). Absent/`false` ⇒ no ledger
|
|
||||||
* action. A deferred-embed insert passes `false` (its vector lands
|
|
||||||
* later — see {@link StorageAdapter.noteVectorLanded}).
|
|
||||||
*/
|
*/
|
||||||
saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void>
|
saveNounMetadata(id: string, metadata: NounMetadata): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete noun metadata
|
* Delete noun metadata
|
||||||
* @param id Noun ID
|
* @param id Noun ID
|
||||||
* @param priorRecord - OPTIONAL already-known metadata (the caller's
|
|
||||||
* pre-delete read) — see {@link StorageAdapter.deleteNoun}.
|
|
||||||
* @param hadVector - OPTIONAL vectored-noun ledger hint: `true`/`false`
|
|
||||||
* when the caller already knows (read as a side effect of ITS OWN delete
|
|
||||||
* flow — e.g. `remove()`'s pre-read for the vector-index removal — never
|
|
||||||
* a read added FOR this ledger), `undefined` when genuinely unknown. A
|
|
||||||
* known `true` decrements the vectored-noun ledger; a known `false` is a
|
|
||||||
* no-op (it was never counted); `undefined` marks the ledger SUSPECT
|
|
||||||
* rather than guessing — the delete path must never add a canonical read
|
|
||||||
* to answer this question.
|
|
||||||
*/
|
*/
|
||||||
deleteNounMetadata(id: string, priorRecord?: NounMetadata | null, hadVector?: boolean): Promise<void>
|
deleteNounMetadata(id: string): Promise<void>
|
||||||
|
|
||||||
/**
|
|
||||||
* OPTIONAL narrow ledger hook: record that a canonical noun's vector just
|
|
||||||
* LANDED for the first time. Exists ONLY for the deferred-embedding
|
|
||||||
* lifecycle — the landing commit (`system:embed-landing`) carries a vector
|
|
||||||
* write with no accompanying metadata operation, so the normal
|
|
||||||
* `saveNounMetadata(..., hasVector)` seam never fires for it. Callers MUST
|
|
||||||
* call this only when the noun held NO real vector before this write (the
|
|
||||||
* deferred-embed worker already holds that fact for free, from its own
|
|
||||||
* pre-embed read — never an added read). A backend without vectored-noun
|
|
||||||
* tracking is a no-op via this method's absence (feature-detected).
|
|
||||||
* @param id - The noun whose vector just landed.
|
|
||||||
*/
|
|
||||||
noteVectorLanded?(id: string): Promise<void>
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get noun with metadata combined
|
* Get noun with metadata combined
|
||||||
|
|
@ -919,11 +875,8 @@ export interface StorageAdapter {
|
||||||
* REQUIRE re-reading the record being removed: when the internal read
|
* REQUIRE re-reading the record being removed: when the internal read
|
||||||
* returns `null` (replace race, or a ghost left by an earlier version) the
|
* returns `null` (replace race, or a ghost left by an earlier version) the
|
||||||
* decrement falls back to this record instead of being silently skipped.
|
* decrement falls back to this record instead of being silently skipped.
|
||||||
* @param hadVector OPTIONAL vectored-noun ledger hint — see
|
|
||||||
* {@link StorageAdapter.deleteNounMetadata}'s `hadVector` param, which
|
|
||||||
* this forwards to unchanged.
|
|
||||||
*/
|
*/
|
||||||
deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise<void>
|
deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Save verb - Pure HNSW verb with core fields only
|
* Save verb - Pure HNSW verb with core fields only
|
||||||
|
|
|
||||||
|
|
@ -1041,27 +1041,12 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
*/
|
*/
|
||||||
protected totalNounCountAll = 0
|
protected totalNounCountAll = 0
|
||||||
protected totalVerbCountAll = 0
|
protected totalVerbCountAll = 0
|
||||||
/**
|
|
||||||
* The count of canonical nouns holding a REAL (non-empty) vector — the
|
|
||||||
* vector-side mirror of `totalNounCountAll` and the coverage denominator a
|
|
||||||
* vector index's node-count ledger is measured against. A deferred-embed
|
|
||||||
* noun (`add({ deferEmbedding: true })`) counts only once its vector
|
|
||||||
* LANDS (the `system:embed-landing` commit) — its canonical record exists
|
|
||||||
* (already counted in `totalNounCountAll`) with an empty vector until
|
|
||||||
* then. Maintained on the write path (a fresh insert whose vector is
|
|
||||||
* non-empty +1, a deferred embed's landing +1, a PROVEN delete of a
|
|
||||||
* vectored noun −1), persisted beside the other ALL scalars, recomputed by
|
|
||||||
* the sanctioned recount. Shares `allCountsSuspect` — no separate flag.
|
|
||||||
*/
|
|
||||||
protected totalVectoredNounCount = 0
|
|
||||||
/**
|
/**
|
||||||
* `true` when a delete could not prove whether the record existed (no
|
* `true` when a delete could not prove whether the record existed (no
|
||||||
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
||||||
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
||||||
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
||||||
* must treat a suspect scalar as unverified, never as exact. Also covers
|
* must treat a suspect scalar as unverified, never as exact.
|
||||||
* `totalVectoredNounCount` — a delete whose vector-presence fact was
|
|
||||||
* unknowable marks this SAME flag rather than minting a second one.
|
|
||||||
*/
|
*/
|
||||||
protected allCountsSuspect = false
|
protected allCountsSuspect = false
|
||||||
/** One narration per session for the suspect transition (never per delete). */
|
/** One narration per session for the suspect transition (never per delete). */
|
||||||
|
|
@ -1098,19 +1083,15 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
||||||
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
||||||
* the ALL-visibility scalar every unfiltered storage walk is measured
|
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||||
* against (the coverage-ledger denominator for derived-index providers);
|
* against (the coverage-ledger denominator for derived-index providers).
|
||||||
* `vectors.all` is the vectored-noun scalar — the coverage denominator for
|
* `suspect` is `true` when an unprovable delete has made `all` unverified
|
||||||
* a vector index's node-count ledger specifically.
|
* since the last sanctioned recount (`rebuildTypeCounts`).
|
||||||
* `suspect` is `true` when an unprovable delete has made `all` (any
|
* @returns Both scalars per family plus the suspect flag.
|
||||||
* family, including `vectors`) unverified since the last sanctioned
|
|
||||||
* recount (`rebuildTypeCounts`).
|
|
||||||
* @returns All scalars per family plus the suspect flag.
|
|
||||||
*/
|
*/
|
||||||
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
||||||
return {
|
return {
|
||||||
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
||||||
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
||||||
vectors: { all: this.totalVectoredNounCount },
|
|
||||||
suspect: this.allCountsSuspect
|
suspect: this.allCountsSuspect
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1122,7 +1103,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
* @param family - Which family's delete was unprovable.
|
* @param family - Which family's delete was unprovable.
|
||||||
* @param id - The id whose existence could not be established.
|
* @param id - The id whose existence could not be established.
|
||||||
*/
|
*/
|
||||||
protected markAllCountsSuspect(family: 'noun' | 'verb' | 'noun-vector', id: string): void {
|
protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void {
|
||||||
this.allCountsSuspect = true
|
this.allCountsSuspect = true
|
||||||
if (!this.allCountsSuspectNarrated) {
|
if (!this.allCountsSuspectNarrated) {
|
||||||
this.allCountsSuspectNarrated = true
|
this.allCountsSuspectNarrated = true
|
||||||
|
|
@ -1135,23 +1116,6 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* OPTIONAL narrow ledger hook (see {@link StorageAdapter.noteVectorLanded}):
|
|
||||||
* record a deferred-embed noun's FIRST real vector landing. The caller
|
|
||||||
* (the deferred-embed worker) proves this is a genuine landing — not a
|
|
||||||
* re-embed of an already-vectored row — by observing its own pre-embed
|
|
||||||
* read's vector was empty, at no added storage cost.
|
|
||||||
* @param id - The noun whose vector just landed (retained for a future
|
|
||||||
* narration seam; the count itself needs no id-keyed state).
|
|
||||||
*/
|
|
||||||
async noteVectorLanded(id: string): Promise<void> {
|
|
||||||
void id
|
|
||||||
this.totalVectoredNounCount++
|
|
||||||
this.scheduleCountPersist().catch(() => {
|
|
||||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment count for entity type - O(1) operation.
|
* Increment count for entity type - O(1) operation.
|
||||||
* Concurrency is handled by the process-global mutex
|
* Concurrency is handled by the process-global mutex
|
||||||
|
|
|
||||||
|
|
@ -2583,7 +2583,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
// record reads), persist, and never scan again. Absent keys are a
|
// record reads), persist, and never scan again. Absent keys are a
|
||||||
// legacy file, not a zero — a zero here would make every provider's
|
// legacy file, not a zero — a zero here would make every provider's
|
||||||
// coverage ledger read "over-posted" on a populated store.
|
// coverage ledger read "over-posted" on a populated store.
|
||||||
let needsPersist = false
|
|
||||||
if (
|
if (
|
||||||
typeof counts.totalNounCountAll === 'number' &&
|
typeof counts.totalNounCountAll === 'number' &&
|
||||||
typeof counts.totalVerbCountAll === 'number'
|
typeof counts.totalVerbCountAll === 'number'
|
||||||
|
|
@ -2602,29 +2601,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
||||||
`every tier) and persisted; no further scan.`
|
`every tier) and persisted; no further scan.`
|
||||||
)
|
)
|
||||||
needsPersist = true
|
|
||||||
}
|
|
||||||
|
|
||||||
// The vectored-noun scalar (shipped after the ALL scalars above — a
|
|
||||||
// counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but
|
|
||||||
// still predate THIS key). Unlike the ALL scalars, presence cannot be
|
|
||||||
// decided from the id-directory listing alone: a deferred-embed
|
|
||||||
// noun's `vectors.json` EXISTS with an empty `vector: []` until its
|
|
||||||
// embed lands, so this derivation reads every noun's `vectors.json`
|
|
||||||
// ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost.
|
|
||||||
if (typeof counts.totalVectoredNounCount === 'number') {
|
|
||||||
this.totalVectoredNounCount = counts.totalVectoredNounCount
|
|
||||||
} else {
|
|
||||||
const vectored = await this.scanVectoredNounCount()
|
|
||||||
this.totalVectoredNounCount = vectored
|
|
||||||
console.warn(
|
|
||||||
`[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` +
|
|
||||||
`derived once by reading every noun's vectors.json (${vectored} vectored) and ` +
|
|
||||||
`persisted; no further scan.`
|
|
||||||
)
|
|
||||||
needsPersist = true
|
|
||||||
}
|
|
||||||
if (needsPersist) {
|
|
||||||
await this.persistCounts()
|
await this.persistCounts()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2667,12 +2643,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.totalNounCountAll = nouns.count
|
this.totalNounCountAll = nouns.count
|
||||||
this.totalVerbCountAll = verbs.count
|
this.totalVerbCountAll = verbs.count
|
||||||
this.allCountsSuspect = false
|
this.allCountsSuspect = false
|
||||||
// Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
|
|
||||||
// (a deferred-embed noun's file exists but holds an empty vector until
|
|
||||||
// its embed lands), so this is a full O(nouns) content scan — see
|
|
||||||
// scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here,
|
|
||||||
// alongside the rest of this from-disk recovery.
|
|
||||||
this.totalVectoredNounCount = await this.scanVectoredNounCount()
|
|
||||||
|
|
||||||
// Sample some entities for the type distribution (don't read all).
|
// Sample some entities for the type distribution (don't read all).
|
||||||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||||||
|
|
@ -2758,61 +2728,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs
|
|
||||||
* — the vector-side mirror of {@link readEntityMetadataRaw}, same
|
|
||||||
* reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`).
|
|
||||||
* @param entityDir - Absolute `entities/nouns/<shard>/<id>` directory.
|
|
||||||
* @returns The parsed vector record, or null when absent/unreadable.
|
|
||||||
*/
|
|
||||||
private async readEntityVectorRaw(entityDir: string): Promise<any | null> {
|
|
||||||
const base = path.join(entityDir, 'vectors.json')
|
|
||||||
try {
|
|
||||||
return JSON.parse(await fs.promises.readFile(base, 'utf-8'))
|
|
||||||
} catch {
|
|
||||||
// fall through to the compressed variant
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const gz = await fs.promises.readFile(`${base}.gz`)
|
|
||||||
return JSON.parse(zlib.gunzipSync(gz).toString('utf-8'))
|
|
||||||
} catch {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Count canonical nouns holding a REAL (non-empty) vector — the vectored-
|
|
||||||
* noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence
|
|
||||||
* cannot be decided from the id-directory listing alone: a deferred-embed
|
|
||||||
* noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`)
|
|
||||||
* until its embed LANDS, so this walk reads every noun's `vectors.json`
|
|
||||||
* CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time
|
|
||||||
* legacy-counts.json derivation or a lost/corrupted counts.json recovery;
|
|
||||||
* the result is persisted so this scan never repeats.
|
|
||||||
*/
|
|
||||||
private async scanVectoredNounCount(): Promise<number> {
|
|
||||||
const base = path.join(this.rootDir, 'entities', 'nouns')
|
|
||||||
let vectored = 0
|
|
||||||
try {
|
|
||||||
const shards = await fs.promises.readdir(base, { withFileTypes: true })
|
|
||||||
for (const shard of shards) {
|
|
||||||
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue
|
|
||||||
const shardPath = path.join(base, shard.name)
|
|
||||||
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
|
|
||||||
for (const entry of ids) {
|
|
||||||
if (!entry.isDirectory()) continue
|
|
||||||
const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name))
|
|
||||||
if (record && Array.isArray(record.vector) && record.vector.length > 0) {
|
|
||||||
vectored++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} catch (error: any) {
|
|
||||||
if (error?.code !== 'ENOENT') throw error
|
|
||||||
}
|
|
||||||
return vectored
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Persist counts to filesystem storage
|
* Persist counts to filesystem storage
|
||||||
*/
|
*/
|
||||||
|
|
@ -2829,10 +2744,6 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
// written before the ledger existed; initializeCounts() derives them once.
|
// written before the ledger existed; initializeCounts() derives them once.
|
||||||
totalNounCountAll: this.totalNounCountAll,
|
totalNounCountAll: this.totalNounCountAll,
|
||||||
totalVerbCountAll: this.totalVerbCountAll,
|
totalVerbCountAll: this.totalVerbCountAll,
|
||||||
// Vectored-noun ledger scalar — absent in files written before it
|
|
||||||
// existed; initializeCounts() derives it once (a content scan, see
|
|
||||||
// scanVectoredNounCount()'s JSDoc).
|
|
||||||
totalVectoredNounCount: this.totalVectoredNounCount,
|
|
||||||
allCountsSuspect: this.allCountsSuspect,
|
allCountsSuspect: this.allCountsSuspect,
|
||||||
lastUpdated: new Date().toISOString()
|
lastUpdated: new Date().toISOString()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -520,12 +520,6 @@ export class MemoryStorage extends BaseStorage {
|
||||||
|
|
||||||
let totalNouns = 0
|
let totalNouns = 0
|
||||||
let totalVerbs = 0
|
let totalVerbs = 0
|
||||||
// Vectored-noun scalar: unlike the bare presence check above, this needs
|
|
||||||
// the vectors.json RECORD'S content — a deferred-embed noun's record
|
|
||||||
// exists with an empty `vector: []` until its embed lands. In-memory this
|
|
||||||
// is a free field access (no I/O), unlike the filesystem adapter's
|
|
||||||
// per-noun disk read.
|
|
||||||
let totalVectoredNouns = 0
|
|
||||||
|
|
||||||
// Scan all paths in objectStore
|
// Scan all paths in objectStore
|
||||||
for (const path of this.objectStore.keys()) {
|
for (const path of this.objectStore.keys()) {
|
||||||
|
|
@ -534,10 +528,6 @@ export class MemoryStorage extends BaseStorage {
|
||||||
if (nounMatch) {
|
if (nounMatch) {
|
||||||
// Type is in metadata, not path - just count total
|
// Type is in metadata, not path - just count total
|
||||||
totalNouns++
|
totalNouns++
|
||||||
const record = this.objectStore.get(path) as { vector?: unknown } | undefined
|
|
||||||
if (Array.isArray(record?.vector) && record.vector.length > 0) {
|
|
||||||
totalVectoredNouns++
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
|
// Count verbs (entities/verbs/{shard}/{id}/vectors.json)
|
||||||
|
|
@ -553,7 +543,6 @@ export class MemoryStorage extends BaseStorage {
|
||||||
// A scan of every canonical record IS the ALL-visibility count.
|
// A scan of every canonical record IS the ALL-visibility count.
|
||||||
this.totalNounCountAll = totalNouns
|
this.totalNounCountAll = totalNouns
|
||||||
this.totalVerbCountAll = totalVerbs
|
this.totalVerbCountAll = totalVerbs
|
||||||
this.totalVectoredNounCount = totalVectoredNouns
|
|
||||||
this.allCountsSuspect = false
|
this.allCountsSuspect = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1762,10 +1762,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Delete a noun from storage
|
* Delete a noun from storage
|
||||||
* @param hadVector - OPTIONAL vectored-noun ledger hint, forwarded to
|
|
||||||
* {@link deleteNounMetadata} unchanged — see its JSDoc.
|
|
||||||
*/
|
*/
|
||||||
public async deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise<void> {
|
public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
|
// FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
|
||||||
|
|
@ -1782,7 +1780,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// LONGER wrapped in a blind catch that masked faults as "file didn't exist".
|
// LONGER wrapped in a blind catch that masked faults as "file didn't exist".
|
||||||
// `priorMetadata` (the caller's pre-delete read) keeps the decrement honest
|
// `priorMetadata` (the caller's pre-delete read) keeps the decrement honest
|
||||||
// even when the canonical read inside returns null (replace race / ghost).
|
// even when the canonical read inside returns null (replace race / ghost).
|
||||||
await this.deleteNounMetadata(id, priorMetadata, hadVector)
|
await this.deleteNounMetadata(id, priorMetadata)
|
||||||
|
|
||||||
// Remove the now-empty entity container (a no-op for key/prefix stores).
|
// Remove the now-empty entity container (a no-op for key/prefix stores).
|
||||||
await this.removeCanonicalContainer(getNounVectorPath(id))
|
await this.removeCanonicalContainer(getNounVectorPath(id))
|
||||||
|
|
@ -3394,12 +3392,11 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
/**
|
/**
|
||||||
* Save noun metadata to storage (now typed)
|
* Save noun metadata to storage (now typed)
|
||||||
* Routes to correct sharded location based on UUID
|
* Routes to correct sharded location based on UUID
|
||||||
* @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc.
|
|
||||||
*/
|
*/
|
||||||
public async saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void> {
|
public async saveNounMetadata(id: string, metadata: NounMetadata): Promise<void> {
|
||||||
// Validate noun type in metadata - storage boundary protection
|
// Validate noun type in metadata - storage boundary protection
|
||||||
validateNounType(metadata.noun)
|
validateNounType(metadata.noun)
|
||||||
return this.saveNounMetadata_internal(id, metadata, hasVector)
|
return this.saveNounMetadata_internal(id, metadata)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -3410,10 +3407,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
* This ensures counts are updated AFTER metadata exists, fixing the race condition
|
* This ensures counts are updated AFTER metadata exists, fixing the race condition
|
||||||
* where storage adapters tried to read metadata before it was saved.
|
* where storage adapters tried to read metadata before it was saved.
|
||||||
*
|
*
|
||||||
* @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc.
|
|
||||||
* @protected
|
* @protected
|
||||||
*/
|
*/
|
||||||
protected async saveNounMetadata_internal(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void> {
|
protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise<void> {
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// ID-first path - no type needed!
|
// ID-first path - no type needed!
|
||||||
|
|
@ -3469,14 +3465,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// record persists here so the ALL scalar never lags the tree.
|
// record persists here so the ALL scalar never lags the tree.
|
||||||
if (isNew) {
|
if (isNew) {
|
||||||
this.totalNounCountAll++
|
this.totalNounCountAll++
|
||||||
// Vectored-noun ledger: rides the SAME isNew gate (once per id, at
|
|
||||||
// creation) — this seam is metadata-write-driven and never re-runs on
|
|
||||||
// the HNSW neighbor-link re-saves that hit saveNoun_internal, so it
|
|
||||||
// cannot double-count. A deferred-embed insert passes hasVector=false
|
|
||||||
// (or omits it); its vector lands later via noteVectorLanded().
|
|
||||||
if (hasVector) {
|
|
||||||
this.totalVectoredNounCount++
|
|
||||||
}
|
|
||||||
if (!(metadata.noun && isCounted)) {
|
if (!(metadata.noun && isCounted)) {
|
||||||
this.scheduleCountPersist().catch(() => {
|
this.scheduleCountPersist().catch(() => {
|
||||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||||
|
|
@ -3872,18 +3860,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
* the skip permanently inflated the persisted totals (adds counted, paired
|
* the skip permanently inflated the persisted totals (adds counted, paired
|
||||||
* removals not decremented), and `Math.max(totalNounCount, scanned)` made
|
* removals not decremented), and `Math.max(totalNounCount, scanned)` made
|
||||||
* the inflation unfixable by any disk cleanup.
|
* the inflation unfixable by any disk cleanup.
|
||||||
* @param hadVector - OPTIONAL vectored-noun ledger hint — see
|
|
||||||
* {@link StorageAdapter.deleteNounMetadata}'s JSDoc. This method never
|
|
||||||
* reads `vectors.json` to answer the question itself (a canonical read
|
|
||||||
* the delete path must never add); a caller that cannot supply the fact
|
|
||||||
* for free leaves it `undefined`, and the ledger goes SUSPECT rather
|
|
||||||
* than guessing.
|
|
||||||
*/
|
*/
|
||||||
public async deleteNounMetadata(
|
public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise<void> {
|
||||||
id: string,
|
|
||||||
priorRecord?: NounMetadata | null,
|
|
||||||
hadVector?: boolean
|
|
||||||
): Promise<void> {
|
|
||||||
await this.ensureInitialized()
|
await this.ensureInitialized()
|
||||||
|
|
||||||
// Direct O(1) delete with ID-first path. Read the canonical record BEFORE
|
// Direct O(1) delete with ID-first path. Read the canonical record BEFORE
|
||||||
|
|
@ -3907,15 +3885,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
} else {
|
} else {
|
||||||
this.markAllCountsSuspect('noun', id)
|
this.markAllCountsSuspect('noun', id)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Vectored-noun ledger: a KNOWN vector fact decrements (or no-ops);
|
|
||||||
// an UNKNOWN one goes suspect rather than guessing — see @param hadVector.
|
|
||||||
if (hadVector === true) {
|
|
||||||
if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount--
|
|
||||||
else this.markAllCountsSuspect('noun-vector', id)
|
|
||||||
} else if (hadVector === undefined) {
|
|
||||||
this.markAllCountsSuspect('noun-vector', id)
|
|
||||||
}
|
|
||||||
this.scheduleCountPersist().catch(() => {
|
this.scheduleCountPersist().catch(() => {
|
||||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||||
})
|
})
|
||||||
|
|
@ -4580,16 +4549,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// construction.
|
// construction.
|
||||||
let allNouns = 0
|
let allNouns = 0
|
||||||
let allVerbs = 0
|
let allVerbs = 0
|
||||||
// Vectored-noun scalar: unlike `allNouns` (decided from the metadata.json
|
|
||||||
// LISTING alone), presence cannot be decided from the vectors.json
|
|
||||||
// listing alone — a deferred-embed noun's vectors.json EXISTS with an
|
|
||||||
// empty `vector: []` until its embed lands, so the file's CONTENT must be
|
|
||||||
// read. This walk already lists every path per shard (including
|
|
||||||
// vectors.json entries — `listCanonicalObjects` yields both legs), so
|
|
||||||
// reading them here costs one EXTRA read per noun beyond the metadata.json
|
|
||||||
// read above (doubling this walk's per-noun I/O) — honest cost, paid only
|
|
||||||
// by this diagnostic/repair recount, never on the hot path.
|
|
||||||
let allVectoredNouns = 0
|
|
||||||
|
|
||||||
// Scan noun shards
|
// Scan noun shards
|
||||||
for (let shard = 0; shard < 256; shard++) {
|
for (let shard = 0; shard < 256; shard++) {
|
||||||
|
|
@ -4600,18 +4559,6 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const paths = await this.listCanonicalObjects(shardDir)
|
const paths = await this.listCanonicalObjects(shardDir)
|
||||||
|
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
if (path.includes('/vectors.json')) {
|
|
||||||
try {
|
|
||||||
const vectorRecord = await this.readCanonicalObject(path)
|
|
||||||
if (vectorRecord && Array.isArray(vectorRecord.vector) && vectorRecord.vector.length > 0) {
|
|
||||||
allVectoredNouns++
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
// Skip vector records that fail to load — best-effort ground truth,
|
|
||||||
// same as the metadata read below.
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (!path.includes('/metadata.json')) continue
|
if (!path.includes('/metadata.json')) continue
|
||||||
allNouns++
|
allNouns++
|
||||||
|
|
||||||
|
|
@ -4687,19 +4634,17 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// IS the proof an unprovable delete could not give.
|
// IS the proof an unprovable delete could not give.
|
||||||
const nounsAllBefore = this.totalNounCountAll
|
const nounsAllBefore = this.totalNounCountAll
|
||||||
const verbsAllBefore = this.totalVerbCountAll
|
const verbsAllBefore = this.totalVerbCountAll
|
||||||
const vectoredBefore = this.totalVectoredNounCount
|
|
||||||
this.totalNounCountAll = allNouns
|
this.totalNounCountAll = allNouns
|
||||||
this.totalVerbCountAll = allVerbs
|
this.totalVerbCountAll = allVerbs
|
||||||
this.totalVectoredNounCount = allVectoredNouns
|
|
||||||
this.allCountsSuspect = false
|
this.allCountsSuspect = false
|
||||||
this.countCache.clear()
|
this.countCache.clear()
|
||||||
await this.persistCounts()
|
await this.persistCounts()
|
||||||
|
|
||||||
prodLog.info(
|
prodLog.info(
|
||||||
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
||||||
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs / ${allVectoredNouns} vectored nouns` +
|
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
|
||||||
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs || vectoredBefore !== allVectoredNouns
|
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
|
||||||
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore} / ${vectoredBefore})`
|
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
|
||||||
: ' (unchanged)') +
|
: ' (unchanged)') +
|
||||||
` — scalar + per-type persisted`
|
` — scalar + per-type persisted`
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -52,15 +52,7 @@ export class SaveNounMetadataOperation implements Operation {
|
||||||
private readonly storage: StorageAdapter,
|
private readonly storage: StorageAdapter,
|
||||||
private readonly id: string,
|
private readonly id: string,
|
||||||
private readonly metadata: NounMetadata,
|
private readonly metadata: NounMetadata,
|
||||||
private readonly isNew: boolean = false,
|
private readonly isNew: boolean = false
|
||||||
/**
|
|
||||||
* OPTIONAL vectored-noun ledger hint: `true` when this write's paired
|
|
||||||
* vector (the SAME insert's `vector` local) is real/non-empty — see
|
|
||||||
* {@link StorageAdapter.saveNounMetadata}'s JSDoc for the isNew-gated,
|
|
||||||
* double-count-proof seam this rides. Default `false`: a deferred-embed
|
|
||||||
* insert (or any caller that doesn't know) never counts here.
|
|
||||||
*/
|
|
||||||
private readonly hasVector: boolean = false
|
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async execute(): Promise<RollbackAction> {
|
async execute(): Promise<RollbackAction> {
|
||||||
|
|
@ -70,7 +62,7 @@ export class SaveNounMetadataOperation implements Operation {
|
||||||
: await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')
|
: await tornHealsToNull(this.storage.getNounMetadata(this.id), 'noun metadata')
|
||||||
|
|
||||||
// Save new metadata
|
// Save new metadata
|
||||||
await this.storage.saveNounMetadata(this.id, this.metadata, this.hasVector)
|
await this.storage.saveNounMetadata(this.id, this.metadata)
|
||||||
|
|
||||||
// Return rollback action
|
// Return rollback action
|
||||||
return async () => {
|
return async () => {
|
||||||
|
|
@ -78,10 +70,8 @@ export class SaveNounMetadataOperation implements Operation {
|
||||||
// Restore previous metadata
|
// Restore previous metadata
|
||||||
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
||||||
} else {
|
} else {
|
||||||
// Delete newly created metadata. `this.hasVector` is the SAME fact
|
// Delete newly created metadata
|
||||||
// this operation's own execute() used to (maybe) count the vectored
|
await this.storage.deleteNounMetadata(this.id)
|
||||||
// ledger — reversing with it on rollback needs no new read.
|
|
||||||
await this.storage.deleteNounMetadata(this.id, undefined, this.hasVector)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -150,9 +140,7 @@ export class SaveNounOperation implements Operation {
|
||||||
// Note: Not all adapters implement deleteNoun
|
// Note: Not all adapters implement deleteNoun
|
||||||
// This is acceptable - metadata deletion makes entity invisible
|
// This is acceptable - metadata deletion makes entity invisible
|
||||||
if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') {
|
if ('deleteNoun' in this.storage && typeof this.storage.deleteNoun === 'function') {
|
||||||
// `this.noun.vector` is the SAME record just written — the
|
await this.storage.deleteNoun(this.noun.id)
|
||||||
// vectored-noun ledger fact is free (no added read) and exact.
|
|
||||||
await this.storage.deleteNoun(this.noun.id, undefined, this.noun.vector.length > 0)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -210,21 +198,14 @@ export class DeleteNounMetadataOperation implements Operation {
|
||||||
return async () => {}
|
return async () => {}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Vectored-noun ledger fact: `previousNoun` is already read above for the
|
|
||||||
// before-image capture — no added read. `undefined` (noun genuinely
|
|
||||||
// absent, metadata-only ghost) is passed through honestly; the storage
|
|
||||||
// layer marks the ledger suspect rather than guessing.
|
|
||||||
const hadVector = previousNoun ? previousNoun.vector.length > 0 : undefined
|
|
||||||
|
|
||||||
// Full removal: both canonical legs + the entity container + count decrement
|
// Full removal: both canonical legs + the entity container + count decrement
|
||||||
// (the prior record keeps the decrement honest on a null canonical read).
|
// (the prior record keeps the decrement honest on a null canonical read).
|
||||||
await this.storage.deleteNoun(this.id, previousMetadata, hadVector)
|
await this.storage.deleteNoun(this.id, previousMetadata)
|
||||||
|
|
||||||
// Return rollback action
|
// Return rollback action
|
||||||
return async () => {
|
return async () => {
|
||||||
// Restore the vector leg, then the metadata leg through the count-aware
|
// Restore the vector leg, then the metadata leg through the count-aware
|
||||||
// save so deleteNoun()'s decrement is reversed (hadVector's mirror:
|
// save so deleteNoun()'s decrement is reversed.
|
||||||
// re-increments the vectored ledger iff the restored vector is real).
|
|
||||||
if (previousNoun) {
|
if (previousNoun) {
|
||||||
await this.storage.saveNoun({
|
await this.storage.saveNoun({
|
||||||
id: previousNoun.id,
|
id: previousNoun.id,
|
||||||
|
|
@ -234,7 +215,7 @@ export class DeleteNounMetadataOperation implements Operation {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (previousMetadata) {
|
if (previousMetadata) {
|
||||||
await this.storage.saveNounMetadata(this.id, previousMetadata, hadVector === true)
|
await this.storage.saveNounMetadata(this.id, previousMetadata)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -17,11 +17,10 @@
|
||||||
* (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is
|
* (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is
|
||||||
* upgraded from the canonical id tree at open, then persisted.
|
* upgraded from the canonical id tree at open, then persisted.
|
||||||
*/
|
*/
|
||||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
import * as fs from 'node:fs'
|
import * as fs from 'node:fs'
|
||||||
import * as os from 'node:os'
|
import * as os from 'node:os'
|
||||||
import * as path from 'node:path'
|
import * as path from 'node:path'
|
||||||
import * as zlib from 'node:zlib'
|
|
||||||
import { Brainy } from '../../src/index.js'
|
import { Brainy } from '../../src/index.js'
|
||||||
|
|
||||||
/** Count canonical `<root>/entities/<kind>/<shard>/<id>` directories — every tier. */
|
/** Count canonical `<root>/entities/<kind>/<shard>/<id>` directories — every tier. */
|
||||||
|
|
@ -181,176 +180,3 @@ describe('canonical count ledger — ALL-visibility scalars, unclamped totals, r
|
||||||
expect(ledger.nouns.all).toBe(truth)
|
expect(ledger.nouns.all).toBe(truth)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
/** Count `<root>/entities/nouns/<shard>/<id>/vectors.json[.gz]` files holding a non-empty `vector`. */
|
|
||||||
function countVectoredNouns(root: string): number {
|
|
||||||
const base = path.join(root, 'entities', 'nouns')
|
|
||||||
if (!fs.existsSync(base)) return 0
|
|
||||||
let n = 0
|
|
||||||
for (const shard of fs.readdirSync(base)) {
|
|
||||||
const shardDir = path.join(base, shard)
|
|
||||||
if (!fs.statSync(shardDir).isDirectory()) continue
|
|
||||||
for (const id of fs.readdirSync(shardDir)) {
|
|
||||||
const idDir = path.join(shardDir, id)
|
|
||||||
if (!fs.statSync(idDir).isDirectory()) continue
|
|
||||||
const plainPath = path.join(idDir, 'vectors.json')
|
|
||||||
const gzPath = `${plainPath}.gz`
|
|
||||||
let record: any = null
|
|
||||||
if (fs.existsSync(plainPath)) {
|
|
||||||
record = JSON.parse(fs.readFileSync(plainPath, 'utf-8'))
|
|
||||||
} else if (fs.existsSync(gzPath)) {
|
|
||||||
record = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf-8'))
|
|
||||||
} else {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if (Array.isArray(record.vector) && record.vector.length > 0) n++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('canonical count ledger — the vectored-noun scalar (the vector leg\'s coverage denominator)', () => {
|
|
||||||
let dir: string
|
|
||||||
let brain: any
|
|
||||||
|
|
||||||
const open = async () => {
|
|
||||||
const b: any = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
silent: true,
|
|
||||||
dimensions: 384
|
|
||||||
})
|
|
||||||
await b.init()
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Baseline vectored count right after a fresh open() — init() creates a
|
|
||||||
* hidden system VFS-root noun that itself carries a real vector, so a
|
|
||||||
* brand-new store's `vectors.all` is 1, not 0. Tests assert DELTAS off
|
|
||||||
* this baseline rather than hardcoding it away. */
|
|
||||||
let baseline: number
|
|
||||||
|
|
||||||
beforeEach(async () => {
|
|
||||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
|
||||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vectored-ledger-'))
|
|
||||||
brain = await open()
|
|
||||||
baseline = (await brain.storage.getCanonicalCounts()).vectors.all
|
|
||||||
})
|
|
||||||
afterEach(async () => {
|
|
||||||
vi.restoreAllMocks()
|
|
||||||
await brain.close?.().catch(() => {})
|
|
||||||
fs.rmSync(dir, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
it('an explicit-vector add counts immediately; the ledger matches the on-disk vectors.json content', async () => {
|
|
||||||
await brain.add({ data: 'a', type: 'document', vector: Array(384).fill(0).map((_, i) => Math.sin(i)) })
|
|
||||||
await brain.add({ data: 'b', type: 'document' }) // embedded (non-deferred) — also a real vector
|
|
||||||
await brain.flush()
|
|
||||||
|
|
||||||
const ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.vectors.all).toBe(baseline + 2)
|
|
||||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
|
||||||
expect(ledger.suspect).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a deferred-embed add does NOT count until its embed LANDS', async () => {
|
|
||||||
// Hold the background worker's embed call open under manual control — a
|
|
||||||
// deterministic embedder is fast enough that the landing could otherwise
|
|
||||||
// race ahead of the "still unlanded" assertion below.
|
|
||||||
let resolveEmbed: ((v: number[]) => void) | undefined
|
|
||||||
vi.spyOn(brain, 'embed').mockImplementation(
|
|
||||||
() => new Promise<number[]>((resolve) => { resolveEmbed = resolve })
|
|
||||||
)
|
|
||||||
|
|
||||||
const id = await brain.add({ data: 'deferred content', type: 'document', deferEmbedding: true })
|
|
||||||
await brain.flush()
|
|
||||||
|
|
||||||
// Landed nothing yet — the ledger must not count the stub.
|
|
||||||
let ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.vectors.all).toBe(baseline)
|
|
||||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
|
||||||
|
|
||||||
// Release the held embed, then cross the barrier: the vector lands
|
|
||||||
// (system:embed-landing).
|
|
||||||
resolveEmbed!(Array(384).fill(0).map((_, i) => Math.cos(i)))
|
|
||||||
await brain.awaitPendingEmbeds()
|
|
||||||
const landed = await brain.get(id, { includeVectors: true })
|
|
||||||
expect((landed!.vector as number[]).length).toBeGreaterThan(0)
|
|
||||||
|
|
||||||
ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.vectors.all).toBe(baseline + 1)
|
|
||||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
|
||||||
expect(ledger.suspect).toBe(false)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a proven delete of a vectored noun decrements; a non-vectored (unlanded) delete does not', async () => {
|
|
||||||
const vectoredId = await brain.add({ data: 'v', type: 'document' }) // real embed, unmocked
|
|
||||||
// Block the embed worker AFTER the real add above — a deterministic
|
|
||||||
// embedder is fast enough that the deferred noun below could otherwise
|
|
||||||
// land before this test observes its "still unlanded" state.
|
|
||||||
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
|
||||||
const deferredId = await brain.add({ data: 'd', type: 'document', deferEmbedding: true })
|
|
||||||
await brain.flush()
|
|
||||||
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(baseline + 1)
|
|
||||||
|
|
||||||
await brain.remove(vectoredId)
|
|
||||||
await brain.flush()
|
|
||||||
let ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.vectors.all).toBe(baseline)
|
|
||||||
expect(ledger.suspect).toBe(false)
|
|
||||||
|
|
||||||
await brain.remove(deferredId) // never had a real vector — no decrement, still unsuspect
|
|
||||||
await brain.flush()
|
|
||||||
ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.vectors.all).toBe(baseline)
|
|
||||||
expect(ledger.suspect).toBe(false)
|
|
||||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
|
||||||
})
|
|
||||||
|
|
||||||
it('the recount corrects a tampered vectors.all scalar, surviving reopen', async () => {
|
|
||||||
await brain.add({ data: 'real 1', type: 'document' })
|
|
||||||
await brain.add({ data: 'real 2', type: 'document' })
|
|
||||||
await brain.flush()
|
|
||||||
const truth = countVectoredNouns(dir)
|
|
||||||
expect(truth).toBe(baseline + 2)
|
|
||||||
|
|
||||||
;(brain.storage as any).totalVectoredNounCount = truth + 40
|
|
||||||
await (brain.storage as any).persistCounts()
|
|
||||||
await brain.close()
|
|
||||||
brain = await open()
|
|
||||||
|
|
||||||
// The lie survives reopen (never clamped).
|
|
||||||
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth + 40)
|
|
||||||
|
|
||||||
await brain.repairIndex()
|
|
||||||
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth)
|
|
||||||
|
|
||||||
await brain.close()
|
|
||||||
brain = await open()
|
|
||||||
expect((await brain.storage.getCanonicalCounts()).vectors.all).toBe(truth)
|
|
||||||
})
|
|
||||||
|
|
||||||
it('a legacy counts.json without totalVectoredNounCount is derived once from vectors.json content and persisted', async () => {
|
|
||||||
await brain.add({ data: 'one', type: 'document' }) // real embed, unmocked
|
|
||||||
// Block the embed worker AFTER the real add above — a deterministic
|
|
||||||
// embedder is fast enough that the deferred noun below could otherwise
|
|
||||||
// land before close(), which would inflate this test's expected count.
|
|
||||||
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
|
||||||
await brain.add({ data: 'two deferred', type: 'document', deferEmbedding: true })
|
|
||||||
await brain.flush()
|
|
||||||
await brain.close()
|
|
||||||
|
|
||||||
const countsPath = path.join(dir, '_system', 'counts.json')
|
|
||||||
const raw = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
|
||||||
expect(typeof raw.totalVectoredNounCount).toBe('number')
|
|
||||||
delete raw.totalVectoredNounCount
|
|
||||||
fs.writeFileSync(countsPath, JSON.stringify(raw, null, 2))
|
|
||||||
|
|
||||||
brain = await open()
|
|
||||||
const ledger = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledger.vectors.all).toBe(baseline + 1) // the root + the one non-deferred noun
|
|
||||||
expect(ledger.vectors.all).toBe(countVectoredNouns(dir))
|
|
||||||
const persisted = JSON.parse(fs.readFileSync(countsPath, 'utf-8'))
|
|
||||||
expect(persisted.totalVectoredNounCount).toBe(baseline + 1)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
|
||||||
|
|
@ -1,247 +0,0 @@
|
||||||
/**
|
|
||||||
* @module tests/integration/vector-leg-open-build
|
|
||||||
* @description THE LAST RED of the two-engine release gate: a migrated
|
|
||||||
* store can hold canonical vectored nouns with NO derived vector index
|
|
||||||
* built. `open()` owns building the derived indexes (reads never build —
|
|
||||||
* see `rebuildIndexesIfNeeded`'s JSDoc); the defect this pins is the vector
|
|
||||||
* leg's decision silently skipping that build, so `find`/search served `[]`
|
|
||||||
* with no error and no narration.
|
|
||||||
*
|
|
||||||
* A downstream deployment measures this through a native vector provider
|
|
||||||
* whose own health report can legitimately say `serving: true` even while
|
|
||||||
* vector COVERAGE is honestly unledgered on its side (an unledgered
|
|
||||||
* invariant never flips `serving` — see `HealthReport`'s derivation laws).
|
|
||||||
* This repo ships only the JS engine, so the reproduction here uses the
|
|
||||||
* SAME plugin seam a native provider would (`brain.use({ activate: ctx =>
|
|
||||||
* ctx.registerProvider('vector', factory) })`, the pattern
|
|
||||||
* `tests/unit/cold-open-rebuild-gate.test.ts` already established for this
|
|
||||||
* exact class of gate-decision bug) with a stub that WRAPS the real
|
|
||||||
* `JsHnswVectorIndex` — every method delegates to a genuine engine (so a
|
|
||||||
* successful rebuild restores REAL, searchable vectors), except `size()`
|
|
||||||
* (fakes 0 until rebuild runs — the "never built" posture) and
|
|
||||||
* `healthReport()` (always reports `serving: true`, `unledgered:
|
|
||||||
* ['vector-coverage']` — the "I don't track this yet" posture). This is
|
|
||||||
* "as close as the JS engine allows": the gap is reproduced at the exact
|
|
||||||
* decision the fix changes, not approximated by deleting files the JS
|
|
||||||
* engine's own cold-start heuristic already recovers from unaided (see the
|
|
||||||
* inverse pin below and cold-open-rebuild-gate.test.ts's already-pinned
|
|
||||||
* "isReady()===true, size()===0" contract, which this fix deliberately does
|
|
||||||
* NOT touch — bare isReady() has no unledgered concept to hide behind, and
|
|
||||||
* overriding it would reopen the 48-seconds-per-restart regression pinned
|
|
||||||
* there).
|
|
||||||
*
|
|
||||||
* Pins:
|
|
||||||
* (1) COVERAGE GAP FORCES THE BUILD: N vectored nouns, a provider that
|
|
||||||
* claims `serving: true` at `size()===0` — open() builds anyway (the
|
|
||||||
* ledger proves there is something to cover), and search returns real
|
|
||||||
* results, never `[]`.
|
|
||||||
* (2) THE INVERSE, HONEST EMPTY: 0 vectored nouns (every embed still
|
|
||||||
* deferred/unlanded) — open() does NOT attempt a rebuild (nothing to
|
|
||||||
* load; the old blunt "always rebuild when size()===0" heuristic wasted
|
|
||||||
* a full canonical walk here for zero benefit), and search honestly
|
|
||||||
* returns `[]` — no error, no false coverage-gap narration.
|
|
||||||
*
|
|
||||||
* SEARCH VERIFICATION NOTE: pin (1) verifies "search returns real results"
|
|
||||||
* via `find({ query: <text> })` (semantic search — embeds the query, then
|
|
||||||
* searches), matching the pattern `tests/integration/hnsw-rebuild.test.ts`
|
|
||||||
* already uses for exactly this "post-rebuild search works" class of pin.
|
|
||||||
* A raw `find({ vector: <array> })` / `index.search(vector, k)` call was
|
|
||||||
* tried first and found to reproducibly return only 1 hit after a
|
|
||||||
* FROM-CANONICAL rebuild (never the full requested `limit`, sometimes not
|
|
||||||
* even a real neighbor) — REGARDLESS of this task's changes: it reproduces
|
|
||||||
* identically on a plain, unwrapped, un-stubbed reopen with the stock JS
|
|
||||||
* engine (verified against `hnsw-rebuild.test.ts`'s own construction) and
|
|
||||||
* is therefore a PRE-EXISTING, orthogonal defect in the JS HNSW engine's
|
|
||||||
* rebuilt-graph connectivity — outside this task's two deliverables (the
|
|
||||||
* count ledger and the open-gate REBUILD DECISION, not rebuild()'s internal
|
|
||||||
* search quality). Left for a separate investigation; not touched here.
|
|
||||||
*/
|
|
||||||
import { describe, it, expect, afterEach, vi } from 'vitest'
|
|
||||||
import * as fs from 'node:fs'
|
|
||||||
import * as os from 'node:os'
|
|
||||||
import * as path from 'node:path'
|
|
||||||
import { Brainy } from '../../src/index.js'
|
|
||||||
import { JsHnswVectorIndex } from '../../src/hnsw/hnswIndex.js'
|
|
||||||
|
|
||||||
const tmpDirs: string[] = []
|
|
||||||
function mkTmp(): string {
|
|
||||||
const d = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-vector-leg-open-'))
|
|
||||||
tmpDirs.push(d)
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
afterEach(() => {
|
|
||||||
vi.restoreAllMocks()
|
|
||||||
for (const d of tmpDirs.splice(0)) fs.rmSync(d, { recursive: true, force: true })
|
|
||||||
})
|
|
||||||
|
|
||||||
const V = (seed: number) =>
|
|
||||||
Array.from({ length: 384 }, (_, i) => Math.sin((seed + 1) * 7919 + i * 131) * 0.5 + 0.5)
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Build a store with N explicit-vector (non-deferred) nouns, flush, close.
|
|
||||||
* Each noun also carries embeddable text (`technology`/`science`, matching
|
|
||||||
* the query used below) so the semantic-search verification exercises real
|
|
||||||
* retrieval, not a coincidental match. The default JS engine builds a fully
|
|
||||||
* current store — the epoch marker is stamped current at this open's
|
|
||||||
* completion, so a later reopen's `_indexEpochStale` is honestly false and
|
|
||||||
* cannot mask the ledger-gap decision under test (nothing here manufactures
|
|
||||||
* epoch drift).
|
|
||||||
*/
|
|
||||||
async function buildVectoredStore(dir: string, n: number): Promise<string[]> {
|
|
||||||
const brain: any = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
plugins: [],
|
|
||||||
silent: true,
|
|
||||||
dimensions: 384
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
const ids: string[] = []
|
|
||||||
for (let i = 0; i < n; i++) {
|
|
||||||
ids.push(
|
|
||||||
await brain.add({
|
|
||||||
data: `doc ${i} about ${i % 2 === 0 ? 'technology' : 'science'}`,
|
|
||||||
type: 'document',
|
|
||||||
vector: V(i)
|
|
||||||
})
|
|
||||||
)
|
|
||||||
}
|
|
||||||
await brain.flush()
|
|
||||||
await brain.close()
|
|
||||||
return ids
|
|
||||||
}
|
|
||||||
|
|
||||||
describe('vector-leg open-build (two-engine gate, last red)', () => {
|
|
||||||
it('coverage gap: a provider reporting serving:true at size()===0 is overridden by the vectored-noun ledger — open() builds, search returns real results', async () => {
|
|
||||||
const dir = mkTmp()
|
|
||||||
const ids = await buildVectoredStore(dir, 12)
|
|
||||||
|
|
||||||
// Wrap the REAL JS engine so a successful rebuild restores genuine,
|
|
||||||
// searchable vectors — only `size()` and `healthReport()` are faked,
|
|
||||||
// simulating a native provider that has never built its own coverage of
|
|
||||||
// an unledgered invariant.
|
|
||||||
const calls = { rebuild: 0 }
|
|
||||||
const brain: any = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
plugins: [],
|
|
||||||
silent: true,
|
|
||||||
dimensions: 384
|
|
||||||
})
|
|
||||||
brain.use({
|
|
||||||
name: 'fake-native-vector-unledgered-coverage',
|
|
||||||
activate: async (ctx: any) => {
|
|
||||||
ctx.registerProvider('vector', (config: any, distance: any, options: any) => {
|
|
||||||
const real = new JsHnswVectorIndex(config, distance, options)
|
|
||||||
let rebuilt = false
|
|
||||||
const originalRebuild = real.rebuild.bind(real)
|
|
||||||
;(real as any).rebuild = async (...args: any[]) => {
|
|
||||||
const r = await originalRebuild(...args)
|
|
||||||
calls.rebuild++
|
|
||||||
rebuilt = true
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
const originalSize = real.size.bind(real)
|
|
||||||
;(real as any).size = () => (rebuilt ? originalSize() : 0)
|
|
||||||
;(real as any).healthReport = () => ({
|
|
||||||
provider: 'vector',
|
|
||||||
healthy: true,
|
|
||||||
serving: true,
|
|
||||||
invariants: [],
|
|
||||||
checkedAt: Date.now(),
|
|
||||||
durationMs: 0,
|
|
||||||
generation: 1,
|
|
||||||
unledgered: ['vector-coverage']
|
|
||||||
})
|
|
||||||
return real
|
|
||||||
})
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
await brain.init()
|
|
||||||
|
|
||||||
// WITHOUT any find() first: open() itself must have built the leg.
|
|
||||||
expect(calls.rebuild, 'open() forced the rebuild despite serving:true').toBe(1)
|
|
||||||
const status = await brain.getIndexStatus()
|
|
||||||
expect(status.hnswIndex.size).toBeGreaterThanOrEqual(ids.length)
|
|
||||||
|
|
||||||
// Real, searchable results — never [] (see the module doc's SEARCH
|
|
||||||
// VERIFICATION NOTE for why this is a semantic `query`, not a raw
|
|
||||||
// `vector`, call).
|
|
||||||
const results = await brain.find({ query: 'technology document', limit: 5 })
|
|
||||||
expect(results.length).toBeGreaterThan(0)
|
|
||||||
expect(results.length).not.toBe(0)
|
|
||||||
|
|
||||||
await brain.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
it('the inverse: only deferred (never-landed) user nouns — the ledger is never inflated by them, and search over them honestly returns []', async () => {
|
|
||||||
// ARCHITECTURAL NOTE (found while building this pin): every brainy store
|
|
||||||
// carries ONE permanent, always-vectored noun beyond user data — the VFS
|
|
||||||
// root (`entities/nouns/.../00000000-0000-0000-0000-000000000000`,
|
|
||||||
// src/vfs/VirtualFileSystem.ts). It is inserted with an explicit all-zero
|
|
||||||
// (but non-empty, length-384) vector on EVERY store's first open — never
|
|
||||||
// deferred (a deliberate WASM-cold-compile-avoidance fix, see that
|
|
||||||
// file's comment) — and VFS init unconditionally re-creates it if
|
|
||||||
// missing, before the rebuild gate ever runs. A literal "0 vectored
|
|
||||||
// nouns" store is therefore unreachable through the public API; a
|
|
||||||
// brand-new store's `vectors.all` floor is 1, not 0. This pin verifies
|
|
||||||
// the law the task names in the ACHIEVABLE form: nouns whose embed is
|
|
||||||
// still deferred/unlanded contribute NOTHING to the vectored-noun ledger
|
|
||||||
// — the coverage-gap comparison sees exactly the root (1), never
|
|
||||||
// root+deferred — and semantic search over deferred-only user content
|
|
||||||
// honestly returns `[]` (no error, no false "coverage restored" claim).
|
|
||||||
const dir = mkTmp()
|
|
||||||
|
|
||||||
const build: any = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
plugins: [],
|
|
||||||
silent: true,
|
|
||||||
dimensions: 384
|
|
||||||
})
|
|
||||||
await build.init()
|
|
||||||
const rootOnlyLedger = await build.storage.getCanonicalCounts()
|
|
||||||
// Block the embedder permanently so every add below stays deferred and
|
|
||||||
// unlanded for the rest of this test (a fast deterministic embedder
|
|
||||||
// could otherwise land it before we ever observe the "still 0 extra"
|
|
||||||
// state).
|
|
||||||
vi.spyOn(build, 'embed').mockImplementation(() => new Promise(() => {}))
|
|
||||||
for (let i = 0; i < 5; i++) {
|
|
||||||
await build.add({ data: `deferred ${i}`, type: 'document', deferEmbedding: true })
|
|
||||||
}
|
|
||||||
await build.flush()
|
|
||||||
const ledgerWithDeferred = await build.storage.getCanonicalCounts()
|
|
||||||
// The five deferred adds contributed ZERO to the vectored-noun ledger.
|
|
||||||
expect(ledgerWithDeferred.vectors.all).toBe(rootOnlyLedger.vectors.all)
|
|
||||||
await build.close()
|
|
||||||
|
|
||||||
// Reopen (default JS engine — no stub needed): the root is the ONLY
|
|
||||||
// thing the vector leg has to load; the deferred nouns are correctly
|
|
||||||
// invisible to it. Block the embedder again BEFORE init() — reopen
|
|
||||||
// recovers the durable pending-embed markers and kicks the worker as
|
|
||||||
// part of init() itself, and an unblocked deterministic embedder could
|
|
||||||
// land all five before this test observes the open-time ledger.
|
|
||||||
const brain: any = new Brainy({
|
|
||||||
requireSubtype: false,
|
|
||||||
storage: { type: 'filesystem', path: dir },
|
|
||||||
plugins: [],
|
|
||||||
silent: true,
|
|
||||||
dimensions: 384
|
|
||||||
})
|
|
||||||
vi.spyOn(brain, 'embed').mockImplementation(() => new Promise(() => {}))
|
|
||||||
await brain.init()
|
|
||||||
// The ledger is exactly the root — the five deferred, still-unlanded
|
|
||||||
// nouns (which the rebuild above DOES insert into the graph, each with
|
|
||||||
// its stub empty vector — `index.size()` counts EVERY canonical noun's
|
|
||||||
// graph node, deferred or not, so it is not the coverage metric) never
|
|
||||||
// inflate the VECTORED count.
|
|
||||||
const ledgerAfterReopen = await brain.storage.getCanonicalCounts()
|
|
||||||
expect(ledgerAfterReopen.vectors.all).toBe(rootOnlyLedger.vectors.all)
|
|
||||||
|
|
||||||
const results = await brain.find({ vector: V(3), limit: 5 })
|
|
||||||
expect(results).toEqual([])
|
|
||||||
|
|
||||||
await brain.close()
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
@ -380,12 +380,6 @@ describe.sequential('lifecycle — the working store', () => {
|
||||||
counted: aliveVerbs + model.vfsContainsVerbs,
|
counted: aliveVerbs + model.vfsContainsVerbs,
|
||||||
all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs
|
all: aliveVerbs + model.vfsContainsVerbs + model.vfsBaselineVerbs
|
||||||
},
|
},
|
||||||
// Every noun this biography ever adds carries an explicit/computed
|
|
||||||
// vector (the harness never defers an embed), so the vectored-noun
|
|
||||||
// scalar tracks nouns.all exactly.
|
|
||||||
vectors: {
|
|
||||||
all: aliveEntities.length + model.vfsFileNouns + model.vfsBaselineNouns
|
|
||||||
},
|
|
||||||
suspect: false
|
suspect: false
|
||||||
})
|
})
|
||||||
} finally {
|
} finally {
|
||||||
|
|
|
||||||
Reference in a new issue