feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg
The coverage denominator the health-by-accounting ratification named for
the vector family — never built until now, and its absence was measured as
the exact outage class it existed to prevent: a migrated store with
canonical vectors and no derived index opened with the vector leg EMPTY,
served [] from vector search with no error, and the report-driven read gate
had nothing to refuse on (the provider's coverage invariant was honestly
unledgered — the denominator was ours to supply).
- getCanonicalCounts() gains vectors: { all } — the count of canonical
nouns holding a REAL vector. Incremented where a vector lands (the
isNew-gated metadata seam for explicit vectors — the same discipline that
keeps HNSW neighbor-link re-saves from inflating counts; a narrow
noteVectorLanded hook for the deferred-embed landing, gated on the
worker's own pre-embed read). Decremented on a proven delete of a
vectored noun; a vector-uncertain delete marks the ledger suspect rather
than guessing (no new reads on the delete path). Recounted by the
sanctioned recount; legacy counts.json derives it once (a deferred noun's
vector file exists with an empty vector, so presence requires one
content read at derivation — never on the hot path).
- The open gate's vector leg: when a health-reporting provider claims
serving while the index holds zero nodes and the ledger proves vectored
canonical rows exist, open BUILDS (narrated) — routed through the
provider's idempotent fillFromCanonical() when exposed (the joint door;
a partial shortfall stays repair()'s operator business), the JS rebuild
otherwise — or fails typed pre-serve. Scoped exactly: bare isReady()
providers, migrating providers, and white-box size stubs open as before.
Pinned end-to-end from the partner gate's probe shape (store with vectored
canonical rows, no derived index, reopen → search serves N, never []),
red-proved against the pre-fix path; the inverse (zero vectored rows) opens
without building and serves [] honestly.
This commit is contained in:
parent
bce2593e24
commit
9730835bdf
10 changed files with 851 additions and 32 deletions
|
|
@ -1762,8 +1762,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
/**
|
||||
* Delete a noun from storage
|
||||
* @param hadVector - OPTIONAL vectored-noun ledger hint, forwarded to
|
||||
* {@link deleteNounMetadata} unchanged — see its JSDoc.
|
||||
*/
|
||||
public async deleteNoun(id: string, priorMetadata?: NounMetadata | null): Promise<void> {
|
||||
public async deleteNoun(id: string, priorMetadata?: NounMetadata | null, hadVector?: boolean): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// FULL removal (live-HEAD hygiene): remove BOTH canonical legs AND the
|
||||
|
|
@ -1780,7 +1782,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// LONGER wrapped in a blind catch that masked faults as "file didn't exist".
|
||||
// `priorMetadata` (the caller's pre-delete read) keeps the decrement honest
|
||||
// even when the canonical read inside returns null (replace race / ghost).
|
||||
await this.deleteNounMetadata(id, priorMetadata)
|
||||
await this.deleteNounMetadata(id, priorMetadata, hadVector)
|
||||
|
||||
// Remove the now-empty entity container (a no-op for key/prefix stores).
|
||||
await this.removeCanonicalContainer(getNounVectorPath(id))
|
||||
|
|
@ -3392,11 +3394,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
/**
|
||||
* Save noun metadata to storage (now typed)
|
||||
* Routes to correct sharded location based on UUID
|
||||
* @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc.
|
||||
*/
|
||||
public async saveNounMetadata(id: string, metadata: NounMetadata): Promise<void> {
|
||||
public async saveNounMetadata(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void> {
|
||||
// Validate noun type in metadata - storage boundary protection
|
||||
validateNounType(metadata.noun)
|
||||
return this.saveNounMetadata_internal(id, metadata)
|
||||
return this.saveNounMetadata_internal(id, metadata, hasVector)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -3407,9 +3410,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* This ensures counts are updated AFTER metadata exists, fixing the race condition
|
||||
* where storage adapters tried to read metadata before it was saved.
|
||||
*
|
||||
* @param hasVector - See {@link StorageAdapter.saveNounMetadata}'s JSDoc.
|
||||
* @protected
|
||||
*/
|
||||
protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise<void> {
|
||||
protected async saveNounMetadata_internal(id: string, metadata: NounMetadata, hasVector?: boolean): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// ID-first path - no type needed!
|
||||
|
|
@ -3465,6 +3469,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// record persists here so the ALL scalar never lags the tree.
|
||||
if (isNew) {
|
||||
this.totalNounCountAll++
|
||||
// Vectored-noun ledger: rides the SAME isNew gate (once per id, at
|
||||
// creation) — this seam is metadata-write-driven and never re-runs on
|
||||
// the HNSW neighbor-link re-saves that hit saveNoun_internal, so it
|
||||
// cannot double-count. A deferred-embed insert passes hasVector=false
|
||||
// (or omits it); its vector lands later via noteVectorLanded().
|
||||
if (hasVector) {
|
||||
this.totalVectoredNounCount++
|
||||
}
|
||||
if (!(metadata.noun && isCounted)) {
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
|
|
@ -3860,8 +3872,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
* the skip permanently inflated the persisted totals (adds counted, paired
|
||||
* removals not decremented), and `Math.max(totalNounCount, scanned)` made
|
||||
* the inflation unfixable by any disk cleanup.
|
||||
* @param hadVector - OPTIONAL vectored-noun ledger hint — see
|
||||
* {@link StorageAdapter.deleteNounMetadata}'s JSDoc. This method never
|
||||
* reads `vectors.json` to answer the question itself (a canonical read
|
||||
* the delete path must never add); a caller that cannot supply the fact
|
||||
* for free leaves it `undefined`, and the ledger goes SUSPECT rather
|
||||
* than guessing.
|
||||
*/
|
||||
public async deleteNounMetadata(id: string, priorRecord?: NounMetadata | null): Promise<void> {
|
||||
public async deleteNounMetadata(
|
||||
id: string,
|
||||
priorRecord?: NounMetadata | null,
|
||||
hadVector?: boolean
|
||||
): Promise<void> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
// Direct O(1) delete with ID-first path. Read the canonical record BEFORE
|
||||
|
|
@ -3885,6 +3907,15 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
} else {
|
||||
this.markAllCountsSuspect('noun', id)
|
||||
}
|
||||
|
||||
// Vectored-noun ledger: a KNOWN vector fact decrements (or no-ops);
|
||||
// an UNKNOWN one goes suspect rather than guessing — see @param hadVector.
|
||||
if (hadVector === true) {
|
||||
if (this.totalVectoredNounCount > 0) this.totalVectoredNounCount--
|
||||
else this.markAllCountsSuspect('noun-vector', id)
|
||||
} else if (hadVector === undefined) {
|
||||
this.markAllCountsSuspect('noun-vector', id)
|
||||
}
|
||||
this.scheduleCountPersist().catch(() => {
|
||||
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||
})
|
||||
|
|
@ -4549,6 +4580,16 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// construction.
|
||||
let allNouns = 0
|
||||
let allVerbs = 0
|
||||
// Vectored-noun scalar: unlike `allNouns` (decided from the metadata.json
|
||||
// LISTING alone), presence cannot be decided from the vectors.json
|
||||
// listing alone — a deferred-embed noun's vectors.json EXISTS with an
|
||||
// empty `vector: []` until its embed lands, so the file's CONTENT must be
|
||||
// read. This walk already lists every path per shard (including
|
||||
// vectors.json entries — `listCanonicalObjects` yields both legs), so
|
||||
// reading them here costs one EXTRA read per noun beyond the metadata.json
|
||||
// read above (doubling this walk's per-noun I/O) — honest cost, paid only
|
||||
// by this diagnostic/repair recount, never on the hot path.
|
||||
let allVectoredNouns = 0
|
||||
|
||||
// Scan noun shards
|
||||
for (let shard = 0; shard < 256; shard++) {
|
||||
|
|
@ -4559,6 +4600,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
const paths = await this.listCanonicalObjects(shardDir)
|
||||
|
||||
for (const path of paths) {
|
||||
if (path.includes('/vectors.json')) {
|
||||
try {
|
||||
const vectorRecord = await this.readCanonicalObject(path)
|
||||
if (vectorRecord && Array.isArray(vectorRecord.vector) && vectorRecord.vector.length > 0) {
|
||||
allVectoredNouns++
|
||||
}
|
||||
} catch (error) {
|
||||
// Skip vector records that fail to load — best-effort ground truth,
|
||||
// same as the metadata read below.
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (!path.includes('/metadata.json')) continue
|
||||
allNouns++
|
||||
|
||||
|
|
@ -4634,17 +4687,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// IS the proof an unprovable delete could not give.
|
||||
const nounsAllBefore = this.totalNounCountAll
|
||||
const verbsAllBefore = this.totalVerbCountAll
|
||||
const vectoredBefore = this.totalVectoredNounCount
|
||||
this.totalNounCountAll = allNouns
|
||||
this.totalVerbCountAll = allVerbs
|
||||
this.totalVectoredNounCount = allVectoredNouns
|
||||
this.allCountsSuspect = false
|
||||
this.countCache.clear()
|
||||
await this.persistCounts()
|
||||
|
||||
prodLog.info(
|
||||
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
||||
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
|
||||
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
|
||||
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
|
||||
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs / ${allVectoredNouns} vectored nouns` +
|
||||
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs || vectoredBefore !== allVectoredNouns
|
||||
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore} / ${vectoredBefore})`
|
||||
: ' (unchanged)') +
|
||||
` — scalar + per-type persisted`
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue