feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data
Every persisted projection artifact (metadata field indexes + column segments, HNSW node records, graph adjacency LSM trees) now carries a stamp asserting 'this state reflects every committed generation ≤ W, atomically' — written LAST in each owner's flush (stamp-after-data: a crash between data and stamp = unstamped = rescan, never trust). At load, each owner computes the three-way verdict: stamped==committed → adopt (zero work) · behind → catchup (gap reported) · above/unstamped → RESCAN, loudly. Legacy artifacts re-derive once, then are stamped forever. Shared law in projectionWatermark.ts (the aggregation verdict machinery, generalized); vector artifacts carry model dimensions. Verdicts are computed and exposed (watermark()/watermarkVerdict()/watermarkGap()); rebuild triggers unchanged — acting on 'catchup' is the fold train. Pins: 22 unit (7 metadata · 8 hnsw · 7 graph, incl. spy-order stamp-after-data) + the end-to-end reopen-adopts pin.
This commit is contained in:
parent
26c6025158
commit
b35d87a7ab
8 changed files with 1259 additions and 1 deletions
|
|
@ -16,6 +16,22 @@ import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
|
|||
import { prodLog } from '../utils/logger.js'
|
||||
import type { VectorIndexProvider, OpaqueIdSet, AtGenerationVectors } from '../plugin.js'
|
||||
import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js'
|
||||
import {
|
||||
computeWatermarkVerdict,
|
||||
makeProjectionStamp,
|
||||
readStampedWatermark,
|
||||
type WatermarkVerdict,
|
||||
type WatermarkVerdictResult
|
||||
} from '../utils/projectionWatermark.js'
|
||||
|
||||
/**
|
||||
* Storage key for the JS HNSW projection's watermark stamp — a sidecar
|
||||
* record beside the artifact (per-node vector-index records + connection
|
||||
* blobs + the entryPoint/maxLevel system record). Written LAST in
|
||||
* {@link JsHnswVectorIndex.flush} so stamp-after-data ordering holds for
|
||||
* every byte the stamp certifies.
|
||||
*/
|
||||
export const HNSW_INDEX_STAMP_KEY = '__index_hnsw_watermark__'
|
||||
|
||||
// Default HNSW parameters
|
||||
const DEFAULT_CONFIG: HNSWConfig = {
|
||||
|
|
@ -99,6 +115,14 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
private dirtyNodes: Set<string> = new Set() // Nodes with unpersisted HNSW data
|
||||
private dirtySystem: boolean = false // Whether system data (entryPoint, maxLevel) needs persist
|
||||
|
||||
// --- Watermark stamp state (see utils/projectionWatermark for the law) ---
|
||||
/** Generation handed in via {@link stampWatermark}, awaiting the next flush. */
|
||||
private pendingWatermark: number | null = null
|
||||
/** Last watermark durably stamped by this instance or loaded on rebuild. */
|
||||
private stampedWatermark: number | null = null
|
||||
/** The three-way verdict computed at load; null until rebuild() runs. */
|
||||
private loadVerdict: WatermarkVerdictResult | null = null
|
||||
|
||||
// Lazy vector storage (B2 optimization): evict the float32 vector to
|
||||
// storage after insert; reload on demand via getVectorSafe() + UnifiedCache.
|
||||
private vectorStorageMode: 'memory' | 'lazy' = 'memory'
|
||||
|
|
@ -170,6 +194,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
}
|
||||
|
||||
if (this.dirtyNodes.size === 0 && !this.dirtySystem) {
|
||||
// Nothing dirty — but a pending watermark still stamps: every byte it
|
||||
// certifies is already durable, so stamp-after-data holds trivially.
|
||||
await this.writePendingStamp()
|
||||
return 0
|
||||
}
|
||||
|
||||
|
|
@ -239,6 +266,13 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
throw new HnswFlushError(failedNodes.size, systemFailed, firstError ?? undefined)
|
||||
}
|
||||
|
||||
// STAMP-AFTER-DATA: the watermark stamp is the LAST write of the flush —
|
||||
// it lands only after every dirty node and the system record persisted
|
||||
// (the throw above guarantees it). A crash anywhere earlier leaves the
|
||||
// artifact behind-stamped or unstamped, which verdicts as catchup/rescan
|
||||
// on the next open — never a wrong adopt.
|
||||
await this.writePendingStamp()
|
||||
|
||||
if (nodeCount > 0) {
|
||||
prodLog.info(`[HNSW] Flushed ${nodeCount} dirty nodes in ${duration}ms`)
|
||||
}
|
||||
|
|
@ -246,6 +280,126 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
return nodeCount
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Record the committed generation this projection reflects.
|
||||
* The stamp is NOT written here — it is written as the final storage write
|
||||
* of the next {@link flush} (stamp-after-data ordering is a module
|
||||
* guarantee, not a caller obligation). The coordinator calls this with the
|
||||
* store's committed generation right before flushing.
|
||||
* @param generation - The committed generation every flushed byte reflects.
|
||||
*/
|
||||
public stampWatermark(generation: number): void {
|
||||
this.pendingWatermark = generation
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The projection's current watermark: the stamp loaded at
|
||||
* rebuild (or the last stamp durably written by this instance). Null =
|
||||
* unstamped (legacy artifact, first boot, or stamping never wired).
|
||||
*/
|
||||
public watermark(): number | null {
|
||||
return this.stampedWatermark
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The three-way adoption verdict computed at load —
|
||||
* `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped <
|
||||
* committed; the gap from {@link watermarkGap} awaits an incremental
|
||||
* fold), `'rescan'` (unstamped or stamped above committed — never
|
||||
* trusted). Null until rebuild() has run. Computed and exposed only; no
|
||||
* load behavior changes ride on it yet — today's rebuild triggers are
|
||||
* unchanged.
|
||||
*/
|
||||
public watermarkVerdict(): WatermarkVerdict | null {
|
||||
return this.loadVerdict?.verdict ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The catch-up window `(from, to]` when the load verdict was
|
||||
* `'catchup'`; null otherwise.
|
||||
*/
|
||||
public watermarkGap(): { from: number; to: number } | null {
|
||||
return this.loadVerdict?.gap ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Write the pending watermark stamp as a sidecar record —
|
||||
* always called AFTER the data it certifies is durable. The stamp carries
|
||||
* the vector-space identity this module can honestly assert: dimensions
|
||||
* only (no embedding-model id is reachable from the index — it never sees
|
||||
* the embedder). A stamp-write failure is fail-safe (unstamped/behind →
|
||||
* rescan/catchup on next open, never a wrong adopt) but is said out loud
|
||||
* and the pending stamp is retained for the next flush.
|
||||
*/
|
||||
private async writePendingStamp(): Promise<void> {
|
||||
if (this.pendingWatermark === null || !this.storage) return
|
||||
const watermark = this.pendingWatermark
|
||||
try {
|
||||
await this.storage.saveMetadata(HNSW_INDEX_STAMP_KEY, {
|
||||
noun: 'IndexWatermark',
|
||||
...makeProjectionStamp(watermark, { dimensions: this.dimension })
|
||||
})
|
||||
this.stampedWatermark = watermark
|
||||
this.pendingWatermark = null
|
||||
} catch (error) {
|
||||
prodLog.error(
|
||||
`[HNSW] failed to write watermark stamp (generation ${watermark}) — ` +
|
||||
`artifact stays behind-stamped (safe: verdicts catchup/rescan, never wrong-adopt); ` +
|
||||
`retrying on next flush:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the artifact's stamp and compute the three-way verdict
|
||||
* against the store's committed generation. Unstamped state on a stamped
|
||||
* store verdicts `'rescan'` LOUDLY — never a silent adopt.
|
||||
*
|
||||
* MIGRATION COST: existing pre-stamp brains verdict `'rescan'` exactly
|
||||
* once (that open re-derives via the rebuild it is already running); the
|
||||
* next flush stamps them, and every later open adopts.
|
||||
*
|
||||
* @param artifactPresent - Whether a persisted artifact exists at all (a
|
||||
* system record was found); gates loud-vs-quiet on the rescan verdict so
|
||||
* first boots don't scream.
|
||||
*/
|
||||
private async loadWatermarkVerdict(artifactPresent: boolean): Promise<void> {
|
||||
if (!this.storage) return
|
||||
const committed = this.storage.committedGeneration?.() ?? null
|
||||
let stamped: number | null = null
|
||||
try {
|
||||
const record = await this.storage.getMetadata(HNSW_INDEX_STAMP_KEY)
|
||||
stamped = readStampedWatermark(record)
|
||||
} catch {
|
||||
// An unreadable stamp is unstamped — the fail-safe direction.
|
||||
stamped = null
|
||||
}
|
||||
const result = computeWatermarkVerdict(stamped, committed)
|
||||
this.loadVerdict = result
|
||||
this.stampedWatermark = stamped
|
||||
|
||||
if (result.verdict === 'rescan') {
|
||||
if (artifactPresent || stamped !== null) {
|
||||
prodLog.warn(
|
||||
`[HNSW] watermark verdict: RESCAN — persisted index is ` +
|
||||
(stamped === null
|
||||
? 'unstamped (legacy pre-stamp artifact, or a crash between data and stamp)'
|
||||
: `stamped at generation ${stamped}, ABOVE the store's committed generation ${committed}`) +
|
||||
` — never adopting unverifiable state`
|
||||
)
|
||||
} else {
|
||||
prodLog.debug('[HNSW] watermark verdict: rescan (no persisted artifact — first boot)')
|
||||
}
|
||||
} else if (result.verdict === 'catchup') {
|
||||
prodLog.info(
|
||||
`[HNSW] watermark verdict: catchup — index stamped at generation ${stamped}, ` +
|
||||
`store committed at ${committed}; the (${stamped}, ${committed}] window awaits ` +
|
||||
`an incremental fold (verdict exposed; the fold lands with the coordinator's wiring)`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Persist one node's connections. When the connections codec is
|
||||
* wired AND the storage adapter exposes `saveBinaryBlob`, the per-level
|
||||
|
|
@ -1563,6 +1717,11 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
this.maxLevel = systemData.maxLevel
|
||||
}
|
||||
|
||||
// Step 2b: Watermark verdict for the persisted artifact — computed and
|
||||
// exposed only (today's rebuild flow is unchanged; this rebuild IS the
|
||||
// re-derive a 'rescan' verdict asks for).
|
||||
await this.loadWatermarkVerdict(systemData !== null)
|
||||
|
||||
// Step 3: Determine preloading strategy (adaptive caching)
|
||||
// Check if vectors should be preloaded at init or loaded on-demand
|
||||
const stats = await this.storage.getStatistics()
|
||||
|
|
|
|||
Reference in a new issue