feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data
Every persisted projection artifact (metadata field indexes + column segments, HNSW node records, graph adjacency LSM trees) now carries a stamp asserting 'this state reflects every committed generation ≤ W, atomically' — written LAST in each owner's flush (stamp-after-data: a crash between data and stamp = unstamped = rescan, never trust). At load, each owner computes the three-way verdict: stamped==committed → adopt (zero work) · behind → catchup (gap reported) · above/unstamped → RESCAN, loudly. Legacy artifacts re-derive once, then are stamped forever. Shared law in projectionWatermark.ts (the aggregation verdict machinery, generalized); vector artifacts carry model dimensions. Verdicts are computed and exposed (watermark()/watermarkVerdict()/watermarkGap()); rebuild triggers unchanged — acting on 'catchup' is the fold train. Pins: 22 unit (7 metadata · 8 hnsw · 7 graph, incl. spy-order stamp-after-data) + the end-to-end reopen-adopts pin.
This commit is contained in:
parent
26c6025158
commit
b35d87a7ab
8 changed files with 1259 additions and 1 deletions
|
|
@ -27,6 +27,22 @@ import { UnifiedCache, getGlobalCache } from '../utils/unifiedCache.js'
|
|||
import { prodLog } from '../utils/logger.js'
|
||||
import { LSMTree } from './lsm/LSMTree.js'
|
||||
import type { GraphIndexProvider } from '../plugin.js'
|
||||
import {
|
||||
computeWatermarkVerdict,
|
||||
makeProjectionStamp,
|
||||
readStampedWatermark,
|
||||
type WatermarkVerdict,
|
||||
type WatermarkVerdictResult
|
||||
} from '../utils/projectionWatermark.js'
|
||||
|
||||
/**
|
||||
* Storage key for the graph-adjacency projection's watermark stamp — a
|
||||
* sidecar record beside the artifact (the two verb-id LSM trees' persisted
|
||||
* SSTables + manifests). Written LAST in
|
||||
* {@link GraphAdjacencyIndex.flush} / {@link GraphAdjacencyIndex.close} so
|
||||
* stamp-after-data ordering holds for every byte the stamp certifies.
|
||||
*/
|
||||
export const GRAPH_ADJACENCY_STAMP_KEY = '__index_graph_adjacency_watermark__'
|
||||
|
||||
export interface GraphIndexConfig {
|
||||
maxIndexSize?: number // Default: 100000
|
||||
|
|
@ -112,6 +128,14 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
// Initialization flag
|
||||
private initialized = false
|
||||
|
||||
// --- Watermark stamp state (see utils/projectionWatermark for the law) ---
|
||||
/** Generation handed in via {@link stampWatermark}, awaiting the next flush. */
|
||||
private pendingWatermark: number | null = null
|
||||
/** Last watermark durably stamped by this instance or loaded at init. */
|
||||
private stampedWatermark: number | null = null
|
||||
/** The three-way verdict computed at init; null until init runs. */
|
||||
private loadVerdict: WatermarkVerdictResult | null = null
|
||||
|
||||
/**
|
||||
* Check if index is initialized and ready for use
|
||||
*/
|
||||
|
|
@ -241,12 +265,135 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
await this.populateVerbIdSetFromStorage()
|
||||
}
|
||||
|
||||
// Watermark verdict for the persisted adjacency artifact (the LSM
|
||||
// SSTables just loaded) — computed and exposed only: today's rebuild /
|
||||
// recovery triggers are unchanged (acting on 'catchup' — the incremental
|
||||
// fold — lands with the coordinator's wiring).
|
||||
await this.loadWatermarkVerdict(lsmTreeSize > 0)
|
||||
|
||||
// Start auto-flush timer after initialization
|
||||
this.startAutoFlush()
|
||||
|
||||
this.initialized = true
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Record the committed generation this projection reflects.
|
||||
* The stamp is NOT written here — it is written as the final storage write
|
||||
* of the next {@link flush} (or {@link close}), so stamp-after-data
|
||||
* ordering is a module guarantee, not a caller obligation. The coordinator
|
||||
* calls this with the store's committed generation right before flushing.
|
||||
* @param generation - The committed generation every flushed byte reflects.
|
||||
*/
|
||||
stampWatermark(generation: number): void {
|
||||
this.pendingWatermark = generation
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The projection's current watermark: the stamp loaded at
|
||||
* init (or the last stamp durably written by this instance). Null =
|
||||
* unstamped (legacy artifact, first boot, or stamping never wired).
|
||||
*/
|
||||
watermark(): number | null {
|
||||
return this.stampedWatermark
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The three-way adoption verdict computed at init —
|
||||
* `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped <
|
||||
* committed; the gap from {@link watermarkGap} awaits an incremental
|
||||
* fold), `'rescan'` (unstamped or stamped above committed — never
|
||||
* trusted). Null until init() has run. Computed and exposed only; no
|
||||
* load behavior changes ride on it yet.
|
||||
*/
|
||||
watermarkVerdict(): WatermarkVerdict | null {
|
||||
return this.loadVerdict?.verdict ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The catch-up window `(from, to]` when the init verdict was
|
||||
* `'catchup'`; null otherwise.
|
||||
*/
|
||||
watermarkGap(): { from: number; to: number } | null {
|
||||
return this.loadVerdict?.gap ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Write the pending watermark stamp as a sidecar record —
|
||||
* always called AFTER the LSM flushes it certifies completed. A stamp-write
|
||||
* failure is fail-safe (unstamped/behind → rescan/catchup on next open,
|
||||
* never a wrong adopt) but is said out loud and the pending stamp is
|
||||
* retained for the next flush.
|
||||
*/
|
||||
private async writePendingStamp(): Promise<void> {
|
||||
if (this.pendingWatermark === null) return
|
||||
const watermark = this.pendingWatermark
|
||||
try {
|
||||
await this.storage.saveMetadata(GRAPH_ADJACENCY_STAMP_KEY, {
|
||||
noun: 'IndexWatermark',
|
||||
...makeProjectionStamp(watermark)
|
||||
})
|
||||
this.stampedWatermark = watermark
|
||||
this.pendingWatermark = null
|
||||
} catch (error) {
|
||||
prodLog.error(
|
||||
`[GraphAdjacencyIndex] failed to write watermark stamp (generation ${watermark}) — ` +
|
||||
`artifact stays behind-stamped (safe: verdicts catchup/rescan, never wrong-adopt); ` +
|
||||
`retrying on next flush:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the artifact's stamp and compute the three-way verdict
|
||||
* against the store's committed generation. Unstamped state on a stamped
|
||||
* store verdicts `'rescan'` LOUDLY — never a silent adopt.
|
||||
*
|
||||
* MIGRATION COST: existing pre-stamp brains verdict `'rescan'` exactly
|
||||
* once (that open re-derives via the recovery walk it already runs); the
|
||||
* next flush stamps them, and every later open adopts.
|
||||
*
|
||||
* @param artifactPresent - Whether persisted SSTables exist at all; gates
|
||||
* loud-vs-quiet on the rescan verdict so first boots don't scream.
|
||||
*/
|
||||
private async loadWatermarkVerdict(artifactPresent: boolean): Promise<void> {
|
||||
const committed = this.storage.committedGeneration?.() ?? null
|
||||
let stamped: number | null = null
|
||||
try {
|
||||
const record = await this.storage.getMetadata(GRAPH_ADJACENCY_STAMP_KEY)
|
||||
stamped = readStampedWatermark(record)
|
||||
} catch {
|
||||
// An unreadable stamp is unstamped — the fail-safe direction.
|
||||
stamped = null
|
||||
}
|
||||
const result = computeWatermarkVerdict(stamped, committed)
|
||||
this.loadVerdict = result
|
||||
this.stampedWatermark = stamped
|
||||
|
||||
if (result.verdict === 'rescan') {
|
||||
if (artifactPresent || stamped !== null) {
|
||||
prodLog.warn(
|
||||
`[GraphAdjacencyIndex] watermark verdict: RESCAN — persisted adjacency is ` +
|
||||
(stamped === null
|
||||
? 'unstamped (legacy pre-stamp artifact, or a crash between data and stamp)'
|
||||
: `stamped at generation ${stamped}, ABOVE the store's committed generation ${committed}`) +
|
||||
` — never adopting unverifiable state`
|
||||
)
|
||||
} else {
|
||||
prodLog.debug(
|
||||
'[GraphAdjacencyIndex] watermark verdict: rescan (no persisted artifact — first boot)'
|
||||
)
|
||||
}
|
||||
} else if (result.verdict === 'catchup') {
|
||||
prodLog.info(
|
||||
`[GraphAdjacencyIndex] watermark verdict: catchup — adjacency stamped at generation ` +
|
||||
`${stamped}, store committed at ${committed}; the (${stamped}, ${committed}] window ` +
|
||||
`awaits an incremental fold (verdict exposed; the fold lands with the coordinator's wiring)`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Populate verbIdSet from storage without full rebuild
|
||||
* Lighter weight than full rebuild - only loads verb IDs, not all verb data
|
||||
|
|
@ -935,6 +1082,12 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
}),
|
||||
])
|
||||
|
||||
// STAMP-AFTER-DATA: the watermark stamp is the LAST write of the flush —
|
||||
// both trees' SSTables are durable before the stamp lands. A crash
|
||||
// anywhere above leaves the artifact behind-stamped or unstamped, which
|
||||
// verdicts as catchup/rescan on the next open — never a wrong adopt.
|
||||
await this.writePendingStamp()
|
||||
|
||||
const elapsed = Date.now() - startTime
|
||||
|
||||
prodLog.debug(`GraphAdjacencyIndex: Flush completed in ${elapsed}ms`)
|
||||
|
|
@ -955,6 +1108,10 @@ export class GraphAdjacencyIndex implements GraphIndexProvider {
|
|||
this.lsmTreeVerbsBySource.close(),
|
||||
this.lsmTreeVerbsByTarget.close(),
|
||||
])
|
||||
|
||||
// Stamp-after-data on the shutdown path too: the trees' final flushes
|
||||
// completed above, so a pending watermark may land now.
|
||||
await this.writePendingStamp()
|
||||
}
|
||||
|
||||
prodLog.info('GraphAdjacencyIndex: Shutdown complete')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue