David Snelling
b35d87a7ab
Every persisted projection artifact (metadata field indexes + column segments, HNSW node records, graph adjacency LSM trees) now carries a stamp asserting 'this state reflects every committed generation ≤ W, atomically' — written LAST in each owner's flush (stamp-after-data: a crash between data and stamp = unstamped = rescan, never trust). At load, each owner computes the three-way verdict: stamped==committed → adopt (zero work) · behind → catchup (gap reported) · above/unstamped → RESCAN, loudly. Legacy artifacts re-derive once, then are stamped forever. Shared law in projectionWatermark.ts (the aggregation verdict machinery, generalized); vector artifacts carry model dimensions. Verdicts are computed and exposed (watermark()/watermarkVerdict()/watermarkGap()); rebuild triggers unchanged — acting on 'catchup' is the fold train. Pins: 22 unit (7 metadata · 8 hnsw · 7 graph, incl. spy-order stamp-after-data) + the end-to-end reopen-adopts pin.
150 lines
6.5 KiB
TypeScript
150 lines
6.5 KiB
TypeScript
/**
|
|
* @module utils/projectionWatermark
|
|
* @description The watermark-stamp contract shared by Brainy's persisted TS
|
|
* projections (metadata index, JS HNSW vector index, graph adjacency index).
|
|
*
|
|
* THE LAW: every persisted projection artifact carries a stamp asserting
|
|
* "this state reflects every committed generation ≤ watermark and nothing
|
|
* above it, atomically". STAMP-AFTER-DATA: the stamp is written only after
|
|
* every byte it certifies is durable — a crash between data and stamp leaves
|
|
* the artifact unstamped, which verdicts as a rescan, never a wrong adopt.
|
|
*
|
|
* At load, each owner computes a three-way verdict against the store's
|
|
* committed generation — the same rule and verdict names the aggregation
|
|
* machinery ships (see `AggregationIndex.stateAdoptionVerdict`):
|
|
*
|
|
* - `'adopt'` — stamped == committed (clean reopen, zero work), or the
|
|
* store exposes no committed generation at all (pre-stamp
|
|
* stores keep their pre-stamp behavior).
|
|
* - `'catchup'` — stamped < committed (an unclean exit after later writes,
|
|
* or a long-lived writer whose last stamp predates recent
|
|
* commits). The artifact is exact AS OF its stamp, so the
|
|
* missing window `(stamped, committed]` can be folded
|
|
* incrementally — at-least-once idempotent, bounded by
|
|
* writes since the stamp, never by store size.
|
|
* - `'rescan'` — unstamped (a legacy pre-stamp artifact, or a crash between
|
|
* data and stamp) or stamped ABOVE committed (e.g. a log
|
|
* truncation on a copied store pulled the watermark back):
|
|
* the state over-claims unverifiably — one exact rescan,
|
|
* said out loud, never a silent adopt.
|
|
*
|
|
* MIGRATION COST (stated once, honored by every owner): existing pre-stamp
|
|
* brains verdict `'rescan'` exactly once — they re-derive from source on
|
|
* that open, the next flush stamps them, and every later open adopts.
|
|
*
|
|
* The verdict is COMPUTED AND EXPOSED by each owner; acting on `'catchup'`
|
|
* (the incremental fold) lands with the owner's coordinator wiring.
|
|
*/
|
|
|
|
/** The three-way load verdict for a persisted projection artifact. */
|
|
export type WatermarkVerdict = 'adopt' | 'catchup' | 'rescan'
|
|
|
|
/**
|
|
* Format version written into every projection stamp. Bump when the stamp
|
|
* record's shape changes incompatibly; readers treat an unknown version as
|
|
* unstamped (→ rescan) rather than guessing.
|
|
*/
|
|
export const PROJECTION_STAMP_FORMAT_VERSION = 1
|
|
|
|
/**
|
|
* @description The stamp record a projection writes into (or beside) its
|
|
* persisted artifact, always AFTER the data it certifies is durable.
|
|
*/
|
|
export interface ProjectionStamp {
|
|
/** The committed generation this artifact reflects, exactly and entirely. */
|
|
watermark: number
|
|
/** {@link PROJECTION_STAMP_FORMAT_VERSION} at write time. */
|
|
formatVersion: number
|
|
/** Wall-clock ms at stamp write — diagnostic only, never load-bearing. */
|
|
stampedAt: number
|
|
/**
|
|
* Identity of the vector space for vector-bearing artifacts (the HNSW
|
|
* index). The JS index has no reachable embedding-model id in its module,
|
|
* so dimensions are the only identity it can honestly assert.
|
|
*/
|
|
modelIdentity?: { embedModelId?: string; dimensions: number | null }
|
|
}
|
|
|
|
/** The verdict plus everything the owner needs to report or act on it. */
|
|
export interface WatermarkVerdictResult {
|
|
verdict: WatermarkVerdict
|
|
/** Watermark read from the artifact's stamp; null = unstamped. */
|
|
stamped: number | null
|
|
/** The store's committed generation at load; null = no capability. */
|
|
committed: number | null
|
|
/** The catch-up window `(from, to]` when verdict is `'catchup'`, else null. */
|
|
gap: { from: number; to: number } | null
|
|
}
|
|
|
|
/**
|
|
* @description Build a stamp record for a projection artifact.
|
|
* @param watermark - The committed generation the artifact reflects.
|
|
* @param modelIdentity - Vector-space identity for vector-bearing artifacts.
|
|
* @returns The stamp record to persist (stamp-after-data).
|
|
*/
|
|
export function makeProjectionStamp(
|
|
watermark: number,
|
|
modelIdentity?: ProjectionStamp['modelIdentity']
|
|
): ProjectionStamp {
|
|
const stamp: ProjectionStamp = {
|
|
watermark,
|
|
formatVersion: PROJECTION_STAMP_FORMAT_VERSION,
|
|
stampedAt: Date.now()
|
|
}
|
|
if (modelIdentity !== undefined) stamp.modelIdentity = modelIdentity
|
|
return stamp
|
|
}
|
|
|
|
/**
|
|
* @description Read the stamped watermark out of a persisted record, treating
|
|
* anything malformed (missing, wrong type, non-finite, negative, or an
|
|
* unknown format version) as unstamped — the fail-safe direction is rescan,
|
|
* never a guessed adopt.
|
|
* @param record - The raw persisted record (or null/undefined).
|
|
* @returns The stamped watermark, or null if effectively unstamped.
|
|
*/
|
|
export function readStampedWatermark(record: unknown): number | null {
|
|
if (record === null || typeof record !== 'object') return null
|
|
const rec = record as Record<string, unknown>
|
|
const version = rec.formatVersion
|
|
if (typeof version !== 'number' || version > PROJECTION_STAMP_FORMAT_VERSION) {
|
|
return null
|
|
}
|
|
const raw = rec.watermark
|
|
if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) return null
|
|
return raw
|
|
}
|
|
|
|
/**
|
|
* @description The three-way adoption verdict — the single decision rule
|
|
* every stamped projection shares (mirrors the aggregation machinery's
|
|
* `stateAdoptionVerdict` exactly: same names, same directions).
|
|
* @param stamped - Watermark read from the artifact ({@link readStampedWatermark}).
|
|
* @param committed - The store's committed generation (null = no capability).
|
|
* @returns The verdict with the stamped/committed pair and the catch-up gap.
|
|
*/
|
|
export function computeWatermarkVerdict(
|
|
stamped: number | null,
|
|
committed: number | null
|
|
): WatermarkVerdictResult {
|
|
// No committed-generation capability: hash/shape checks are the only
|
|
// adoption gate, exactly the pre-stamp behavior. Never fail a store that
|
|
// cannot express the question.
|
|
if (committed === null) {
|
|
return { verdict: 'adopt', stamped, committed, gap: null }
|
|
}
|
|
if (stamped === committed) {
|
|
return { verdict: 'adopt', stamped, committed, gap: null }
|
|
}
|
|
if (stamped !== null && stamped < committed) {
|
|
return {
|
|
verdict: 'catchup',
|
|
stamped,
|
|
committed,
|
|
gap: { from: stamped, to: committed }
|
|
}
|
|
}
|
|
// Unstamped, or stamped above committed: unverifiable — rescan, loudly
|
|
// (the caller owns the loud log so it can name its projection).
|
|
return { verdict: 'rescan', stamped, committed, gap: null }
|
|
}
|