fix: honest index readiness — no silently-empty queries on a cold index
Closes the "dishonest readiness proxy" anti-pattern (Pattern A): size()>0 /
isInitialized were treated as "this index serves queries", but a cold native
index can load its COUNT before its SERVING structure, so a query returned a
silent [] indistinguishable from "no such data". A shared assessIndexReadiness()
now reads only the provider's honest isReady() signal (never size()), applied at
every site:
- Vector: a one-shot verifyVectorLive() guard on the semantic/proximity search
path (a pure semantic find({query}) has no filter, so nothing guarded it). It
prefers isReady(), else a known-vector self-match probe; self-heals via rebuild
or throws the new VectorIndexNotReadyError instead of a silent [].
- Graph: getVerbsBySource/ByTarget skip the fast path when the provider reports
not-ready (falling to the canonical shard scan), plus a one-shot probe that
self-heals a no-isReady provider whose adjacency did not cold-load.
- getIndexStatus(): folds in per-index honest `ready` (making `populated`
honest) + rebuildFailed/rebuildError/degradedIds, so a readiness probe never
200s a brain that is still warming up or degraded.
Unblocked by the native providers now reporting serving-truth (graph via
SSTable-residency readiness, vector via durableBaseLoadFailed). New export:
VectorIndexNotReadyError. getIndexStatus gains additive fields. No breaking API.
13 new tests; existing readiness guards green.
This commit is contained in:
parent
36d4e80ba2
commit
d0f69c731f
9 changed files with 654 additions and 9 deletions
|
|
@ -5,6 +5,7 @@
|
|||
|
||||
import { GraphAdjacencyIndex } from '../graph/graphAdjacencyIndex.js'
|
||||
import type { GraphEntityIdResolver } from '../graph/graphAdjacencyIndex.js'
|
||||
import { assessIndexReadiness } from '../utils/indexReadiness.js'
|
||||
|
||||
import {
|
||||
GraphVerb,
|
||||
|
|
@ -248,6 +249,8 @@ function isCountedVisibility(visibility: unknown): boolean {
|
|||
*/
|
||||
export abstract class BaseStorage extends BaseStorageAdapter {
|
||||
protected isInitialized = false
|
||||
/** One-shot guard so the graph fast-path cold-load probe runs once per adapter. */
|
||||
private _graphFastPathProbed = false
|
||||
protected graphIndex?: GraphAdjacencyIndex
|
||||
protected graphIndexPromise?: Promise<GraphAdjacencyIndex>
|
||||
/**
|
||||
|
|
@ -2681,6 +2684,61 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
|
||||
return this.graphIndex
|
||||
}
|
||||
|
||||
/**
|
||||
* @description One-shot cold-load self-heal for a graph provider that does NOT
|
||||
* expose the honest `isReady()` signal. A native provider wired via
|
||||
* {@link setGraphIndex} bypasses `_initializeGraphIndex`'s `size()===0`
|
||||
* self-heal, so a cold-open that loaded the COUNT but not the source→target
|
||||
* adjacency would let the fast path return a silent `[]`. This probe (run once,
|
||||
* before the fast path) samples ONE known persisted edge: if the index claims
|
||||
* relationships (`size() > 0`) yet that edge's source resolves to no verb ints,
|
||||
* the adjacency did not load → rebuild once from storage. Providers that expose
|
||||
* `isReady()` are covered by the honest gate in
|
||||
* getVerbsBy{Source,Target}_internal and skip this probe; the JS index (which
|
||||
* self-heals in `_initializeGraphIndex`) resolves its known edge and no-ops.
|
||||
*/
|
||||
private async ensureGraphFastPathProbed(): Promise<void> {
|
||||
if (this._graphFastPathProbed) return
|
||||
const index = this.graphIndex
|
||||
const resolver = this.graphEntityIdResolver
|
||||
if (!index || !index.isInitialized || !resolver) return
|
||||
// isReady()-capable providers report serving honestly — the gate handles them.
|
||||
if (assessIndexReadiness(index) !== 'unknown') {
|
||||
this._graphFastPathProbed = true
|
||||
return
|
||||
}
|
||||
if (index.size() <= 0) {
|
||||
this._graphFastPathProbed = true
|
||||
return // no edges claimed — nothing to verify
|
||||
}
|
||||
try {
|
||||
const sample = await this.getVerbs({ pagination: { limit: 1 } })
|
||||
const verb = sample.items?.[0]
|
||||
if (!verb || !verb.sourceId) {
|
||||
this._graphFastPathProbed = true
|
||||
return // no edges in storage — stale count, harmless
|
||||
}
|
||||
const sourceInt = resolver.getInt(verb.sourceId)
|
||||
if (sourceInt === undefined) {
|
||||
this._graphFastPathProbed = true
|
||||
return // foreign / unmapped sample — inconclusive, never a false rebuild
|
||||
}
|
||||
const verbInts = await index.getVerbIdsBySource(BigInt(sourceInt))
|
||||
if (verbInts.length === 0) {
|
||||
prodLog.warn(
|
||||
`[BaseStorage] Graph fast path: index reports ${index.size()} relationship(s) but a ` +
|
||||
`known persisted edge resolves to none — the persisted adjacency did not load. ` +
|
||||
`Rebuilding once from storage.`
|
||||
)
|
||||
await index.rebuild()
|
||||
}
|
||||
this._graphFastPathProbed = true
|
||||
} catch (error) {
|
||||
// Transient probe/rebuild failure must not break the read; re-arm for next call.
|
||||
prodLog.debug(`[BaseStorage] Graph fast-path probe skipped (transient): ${error}`)
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Clear all data from storage
|
||||
* This method should be implemented by each specific adapter
|
||||
|
|
@ -4199,13 +4257,22 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
sourceId: string
|
||||
): Promise<HNSWVerbWithMetadata[]> {
|
||||
await this.ensureInitialized()
|
||||
await this.ensureGraphFastPathProbed()
|
||||
|
||||
prodLog.debug(`[BaseStorage] getVerbsBySource_internal: sourceId=${sourceId}, graphIndex=${!!this.graphIndex}, isInitialized=${this.graphIndex?.isInitialized}`)
|
||||
|
||||
// Fast path - use GraphAdjacencyIndex if available (lazy-loaded).
|
||||
// 8.0 BigInt boundary: convert the UUID to an entity int up front and
|
||||
// resolve returned verb ints back to verb-id strings.
|
||||
if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) {
|
||||
// Honest gate: a provider that exposes isReady() and reports not-ready
|
||||
// (count/manifest loaded but source→target edges NOT) is SKIPPED so we fall
|
||||
// to the correct-but-slower canonical shard scan below instead of a silent [].
|
||||
if (
|
||||
this.graphIndex &&
|
||||
this.graphIndex.isInitialized &&
|
||||
this.graphEntityIdResolver &&
|
||||
assessIndexReadiness(this.graphIndex) !== 'not-ready'
|
||||
) {
|
||||
try {
|
||||
const sourceInt = this.graphEntityIdResolver.getInt(sourceId)
|
||||
if (sourceInt === undefined) {
|
||||
|
|
@ -4424,11 +4491,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
targetId: string
|
||||
): Promise<HNSWVerbWithMetadata[]> {
|
||||
await this.ensureInitialized()
|
||||
await this.ensureGraphFastPathProbed()
|
||||
|
||||
// Fast path - use GraphAdjacencyIndex if available (lazy-loaded).
|
||||
// 8.0 BigInt boundary: convert the UUID to an entity int up front and
|
||||
// resolve returned verb ints back to verb-id strings.
|
||||
if (this.graphIndex && this.graphIndex.isInitialized && this.graphEntityIdResolver) {
|
||||
// Honest gate: a not-ready provider (count loaded but edges NOT) is SKIPPED so
|
||||
// we fall to the correct-but-slower canonical shard scan instead of a silent [].
|
||||
if (
|
||||
this.graphIndex &&
|
||||
this.graphIndex.isInitialized &&
|
||||
this.graphEntityIdResolver &&
|
||||
assessIndexReadiness(this.graphIndex) !== 'not-ready'
|
||||
) {
|
||||
try {
|
||||
const targetInt = this.graphEntityIdResolver.getInt(targetId)
|
||||
if (targetInt === undefined) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue