fix: honest index readiness — no silently-empty queries on a cold index

Closes the "dishonest readiness proxy" anti-pattern (Pattern A): size()>0 /
isInitialized were treated as "this index serves queries", but a cold native
index can load its COUNT before its SERVING structure, so a query returned a
silent [] indistinguishable from "no such data". A shared assessIndexReadiness()
now reads only the provider's honest isReady() signal (never size()), applied at
every site:

- Vector: a one-shot verifyVectorLive() guard on the semantic/proximity search
  path (a pure semantic find({query}) has no filter, so nothing guarded it). It
  prefers isReady(), else a known-vector self-match probe; self-heals via rebuild
  or throws the new VectorIndexNotReadyError instead of a silent [].
- Graph: getVerbsBySource/ByTarget skip the fast path when the provider reports
  not-ready (falling to the canonical shard scan), plus a one-shot probe that
  self-heals a no-isReady provider whose adjacency did not cold-load.
- getIndexStatus(): folds in per-index honest `ready` (making `populated`
  honest) + rebuildFailed/rebuildError/degradedIds, so a readiness probe never
  200s a brain that is still warming up or degraded.

Unblocked by the native providers now reporting serving-truth (graph via
SSTable-residency readiness, vector via durableBaseLoadFailed). New export:
VectorIndexNotReadyError. getIndexStatus gains additive fields. No breaking API.
13 new tests; existing readiness guards green.
This commit is contained in:
David Snelling 2026-07-13 13:08:52 -07:00
parent 36d4e80ba2
commit d0f69c731f
9 changed files with 654 additions and 9 deletions

View file

@ -5,6 +5,7 @@
import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js'
import type { GraphEntityIdResolver } from '../graph/graphAdjacencyIndex.js'
import { assessIndexReadiness } from '../utils/indexReadiness.js'
import {
GraphVerb,
@ -248,6 +249,8 @@ function isCountedVisibility(visibility: unknown): boolean {
*/
export abstract class BaseStorage extends BaseStorageAdapter {
protected isInitialized = false
/** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */
private _graphFastPathProbed = false
protected graphIndex?: GraphAdjacencyIndex
protected graphIndexPromise?: Promise<GraphAdjacencyIndex>
/**
@ -2681,6 +2684,61 @@ export abstract class BaseStorage extends BaseStorageAdapter {
return this.graphIndex
}
/**
* @description One-shot cold-load self-heal for a graph provider that does NOT
* expose the honest `isReady()` signal. A native provider wired via
* {@link setGraphIndex} bypasses `_initializeGraphIndex`'s `size()===0`
* self-heal, so a cold-open that loaded the COUNT but not the sourcetarget
* adjacency would let the fast path return a silent `[]`. This probe (run once,
* before the fast path) samples ONE known persisted edge: if the index claims
* relationships (`size() > 0`) yet that edge's source resolves to no verb ints,
* the adjacency did not load rebuild once from storage. Providers that expose
* `isReady()` are covered by the honest gate in
* getVerbsBy{Source,Target}_internal and skip this probe; the JS index (which
* self-heals in `_initializeGraphIndex`) resolves its known edge and no-ops.
*/
private async ensureGraphFastPathProbed(): Promise<void> {
if (this._graphFastPathProbed) return
const index = this.graphIndex
const resolver = this.graphEntityIdResolver
if (!index || !index.isInitialized || !resolver) return
// isReady()-capable providers report serving honestly — the gate handles them.
if (assessIndexReadiness(index) !== 'unknown') {
this._graphFastPathProbed = true
return
}
if (index.size() <= 0) {
this._graphFastPathProbed = true
return // no edges claimed — nothing to verify
}
try {
const sample = await this.getVerbs({ pagination: { limit: 1 } })
const verb = sample.items?.[0]
if (!verb || !verb.sourceId) {
this._graphFastPathProbed = true
return // no edges in storage — stale count, harmless
}
const sourceInt = resolver.getInt(verb.sourceId)
if (sourceInt === undefined) {
this._graphFastPathProbed = true
return // foreign / unmapped sample — inconclusive, never a false rebuild
}
const verbInts = await index.getVerbIdsBySource(BigInt(sourceInt))
if (verbInts.length === 0) {
prodLog.warn(
`[BaseStorage] Graph fast path: index reports ${index.size()} relationship(s) but a ` +
`known persisted edge resolves to none — the persisted adjacency did not load. ` +
`Rebuilding once from storage.`
)
await index.rebuild()
}
this._graphFastPathProbed = true
} catch (error) {
// Transient probe/rebuild failure must not break the read; re-arm for next call.
prodLog.debug(`[BaseStorage] Graph fast-path probe skipped (transient): ${error}`)
}
}
/**
* Clear all data from storage
* This method should be implemented by each specific adapter
@ -4199,13 +4257,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
sourceId: string
): Promise<HNSWVerbWithMetadata[]> {
await this.ensureInitialized()
await this.ensureGraphFastPathProbed()
prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`)
// Fast path - use GraphAdjacencyIndex if available (lazy-loaded).
// 8.0 BigInt boundary: convert the UUID to an entity int up front and
// resolve returned verb ints back to verb-id strings.
if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) {
// Honest gate: a provider that exposes isReady() and reports not-ready
// (count/manifest loaded but source→target edges NOT) is SKIPPED so we fall
// to the correct-but-slower canonical shard scan below instead of a silent [].
if (
this.graphIndex &&
this.graphIndex.isInitialized &&
this.graphEntityIdResolver &&
assessIndexReadiness(this.graphIndex) !== 'not-ready'
) {
try {
const sourceInt = this.graphEntityIdResolver.getInt(sourceId)
if (sourceInt === undefined) {
@ -4424,11 +4491,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
targetId: string
): Promise<HNSWVerbWithMetadata[]> {
await this.ensureInitialized()
await this.ensureGraphFastPathProbed()
// Fast path - use GraphAdjacencyIndex if available (lazy-loaded).
// 8.0 BigInt boundary: convert the UUID to an entity int up front and
// resolve returned verb ints back to verb-id strings.
if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) {
// Honest gate: a not-ready provider (count loaded but edges NOT) is SKIPPED so
// we fall to the correct-but-slower canonical shard scan instead of a silent [].
if (
this.graphIndex &&
this.graphIndex.isInitialized &&
this.graphEntityIdResolver &&
assessIndexReadiness(this.graphIndex) !== 'not-ready'
) {
try {
const targetInt = this.graphEntityIdResolver.getInt(targetId)
if (targetInt === undefined) {