feat(vector): the vectored-noun scalar joins the count ledger; the open gate closes the vector leg
The coverage denominator the health-by-accounting ratification named for
the vector family — never built until now, and its absence was measured as
the exact outage class it existed to prevent: a migrated store with
canonical vectors and no derived index opened with the vector leg EMPTY,
served [] from vector search with no error, and the report-driven read gate
had nothing to refuse on (the provider's coverage invariant was honestly
unledgered — the denominator was ours to supply).
- getCanonicalCounts() gains vectors: { all } — the count of canonical
nouns holding a REAL vector. Incremented where a vector lands (the
isNew-gated metadata seam for explicit vectors — the same discipline that
keeps HNSW neighbor-link re-saves from inflating counts; a narrow
noteVectorLanded hook for the deferred-embed landing, gated on the
worker's own pre-embed read). Decremented on a proven delete of a
vectored noun; a vector-uncertain delete marks the ledger suspect rather
than guessing (no new reads on the delete path). Recounted by the
sanctioned recount; legacy counts.json derives it once (a deferred noun's
vector file exists with an empty vector, so presence requires one
content read at derivation — never on the hot path).
- The open gate's vector leg: when a health-reporting provider claims
serving while the index holds zero nodes and the ledger proves vectored
canonical rows exist, open BUILDS (narrated) — routed through the
provider's idempotent fillFromCanonical() when exposed (the joint door;
a partial shortfall stays repair()'s operator business), the JS rebuild
otherwise — or fails typed pre-serve. Scoped exactly: bare isReady()
providers, migrating providers, and white-box size stubs open as before.
Pinned end-to-end from the partner gate's probe shape (store with vectored
canonical rows, no derived index, reopen → search serves N, never []),
red-proved against the pre-fix path; the inverse (zero vectored rows) opens
without building and serves [] honestly.
This commit is contained in:
parent
bce2593e24
commit
9730835bdf
10 changed files with 851 additions and 32 deletions
|
|
@ -2583,6 +2583,7 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// record reads), persist, and never scan again. Absent keys are a
|
||||
// legacy file, not a zero — a zero here would make every provider's
|
||||
// coverage ledger read "over-posted" on a populated store.
|
||||
let needsPersist = false
|
||||
if (
|
||||
typeof counts.totalNounCountAll === 'number' &&
|
||||
typeof counts.totalVerbCountAll === 'number'
|
||||
|
|
@ -2601,6 +2602,29 @@ export class FileSystemStorage extends BaseStorage {
|
|||
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
||||
`every tier) and persisted; no further scan.`
|
||||
)
|
||||
needsPersist = true
|
||||
}
|
||||
|
||||
// The vectored-noun scalar (shipped after the ALL scalars above — a
|
||||
// counts.json can carry `totalNounCountAll`/`totalVerbCountAll` but
|
||||
// still predate THIS key). Unlike the ALL scalars, presence cannot be
|
||||
// decided from the id-directory listing alone: a deferred-embed
|
||||
// noun's `vectors.json` EXISTS with an empty `vector: []` until its
|
||||
// embed lands, so this derivation reads every noun's `vectors.json`
|
||||
// ONCE (O(nouns) reads, not O(ids) listing) — honest, one-time cost.
|
||||
if (typeof counts.totalVectoredNounCount === 'number') {
|
||||
this.totalVectoredNounCount = counts.totalVectoredNounCount
|
||||
} else {
|
||||
const vectored = await this.scanVectoredNounCount()
|
||||
this.totalVectoredNounCount = vectored
|
||||
console.warn(
|
||||
`[FileSystemStorage] counts.json predates the vectored-noun count ledger — ` +
|
||||
`derived once by reading every noun's vectors.json (${vectored} vectored) and ` +
|
||||
`persisted; no further scan.`
|
||||
)
|
||||
needsPersist = true
|
||||
}
|
||||
if (needsPersist) {
|
||||
await this.persistCounts()
|
||||
}
|
||||
|
||||
|
|
@ -2643,6 +2667,12 @@ export class FileSystemStorage extends BaseStorage {
|
|||
this.totalNounCountAll = nouns.count
|
||||
this.totalVerbCountAll = verbs.count
|
||||
this.allCountsSuspect = false
|
||||
// Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
|
||||
// (a deferred-embed noun's file exists but holds an empty vector until
|
||||
// its embed lands), so this is a full O(nouns) content scan — see
|
||||
// scanVectoredNounCount()'s JSDoc for the cost note. Paid once, here,
|
||||
// alongside the rest of this from-disk recovery.
|
||||
this.totalVectoredNounCount = await this.scanVectoredNounCount()
|
||||
|
||||
// Sample some entities for the type distribution (don't read all).
|
||||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||||
|
|
@ -2728,6 +2758,61 @@ export class FileSystemStorage extends BaseStorage {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one canonical noun's `vectors.json` (or `.json.gz`) directly with fs
|
||||
* — the vector-side mirror of {@link readEntityMetadataRaw}, same
|
||||
* reentrancy reason (bypasses `getNoun()`'s `ensureInitialized()`).
|
||||
* @param entityDir - Absolute `entities/nouns/<shard>/<id>` directory.
|
||||
* @returns The parsed vector record, or null when absent/unreadable.
|
||||
*/
|
||||
private async readEntityVectorRaw(entityDir: string): Promise<any | null> {
|
||||
const base = path.join(entityDir, 'vectors.json')
|
||||
try {
|
||||
return JSON.parse(await fs.promises.readFile(base, 'utf-8'))
|
||||
} catch {
|
||||
// fall through to the compressed variant
|
||||
}
|
||||
try {
|
||||
const gz = await fs.promises.readFile(`${base}.gz`)
|
||||
return JSON.parse(zlib.gunzipSync(gz).toString('utf-8'))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Count canonical nouns holding a REAL (non-empty) vector — the vectored-
|
||||
* noun ledger scalar. UNLIKE {@link scanCanonicalEntities}, presence
|
||||
* cannot be decided from the id-directory listing alone: a deferred-embed
|
||||
* noun's `vectors.json` EXISTS (written at `add()` time with `vector: []`)
|
||||
* until its embed LANDS, so this walk reads every noun's `vectors.json`
|
||||
* CONTENT — O(nouns) reads, not O(ids) listing. Used ONLY for a one-time
|
||||
* legacy-counts.json derivation or a lost/corrupted counts.json recovery;
|
||||
* the result is persisted so this scan never repeats.
|
||||
*/
|
||||
private async scanVectoredNounCount(): Promise<number> {
|
||||
const base = path.join(this.rootDir, 'entities', 'nouns')
|
||||
let vectored = 0
|
||||
try {
|
||||
const shards = await fs.promises.readdir(base, { withFileTypes: true })
|
||||
for (const shard of shards) {
|
||||
if (!shard.isDirectory() || !/^[0-9a-f]{2}$/i.test(shard.name)) continue
|
||||
const shardPath = path.join(base, shard.name)
|
||||
const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
|
||||
for (const entry of ids) {
|
||||
if (!entry.isDirectory()) continue
|
||||
const record = await this.readEntityVectorRaw(path.join(shardPath, entry.name))
|
||||
if (record && Array.isArray(record.vector) && record.vector.length > 0) {
|
||||
vectored++
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
}
|
||||
return vectored
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist counts to filesystem storage
|
||||
*/
|
||||
|
|
@ -2744,6 +2829,10 @@ export class FileSystemStorage extends BaseStorage {
|
|||
// written before the ledger existed; initializeCounts() derives them once.
|
||||
totalNounCountAll: this.totalNounCountAll,
|
||||
totalVerbCountAll: this.totalVerbCountAll,
|
||||
// Vectored-noun ledger scalar — absent in files written before it
|
||||
// existed; initializeCounts() derives it once (a content scan, see
|
||||
// scanVectoredNounCount()'s JSDoc).
|
||||
totalVectoredNounCount: this.totalVectoredNounCount,
|
||||
allCountsSuspect: this.allCountsSuspect,
|
||||
lastUpdated: new Date().toISOString()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue