feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data
Every persisted projection artifact (metadata field indexes + column segments, HNSW node records, graph adjacency LSM trees) now carries a stamp asserting 'this state reflects every committed generation ≤ W, atomically' — written LAST in each owner's flush (stamp-after-data: a crash between data and stamp = unstamped = rescan, never trust). At load, each owner computes the three-way verdict: stamped==committed → adopt (zero work) · behind → catchup (gap reported) · above/unstamped → RESCAN, loudly. Legacy artifacts re-derive once, then are stamped forever. Shared law in projectionWatermark.ts (the aggregation verdict machinery, generalized); vector artifacts carry model dimensions. Verdicts are computed and exposed (watermark()/watermarkVerdict()/watermarkGap()); rebuild triggers unchanged — acting on 'catchup' is the fold train. Pins: 22 unit (7 metadata · 8 hnsw · 7 graph, incl. spy-order stamp-after-data) + the end-to-end reopen-adopts pin.
This commit is contained in:
parent
26c6025158
commit
b35d87a7ab
8 changed files with 1259 additions and 1 deletions
|
|
@ -13,6 +13,13 @@ import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCac
|
|||
import { compareCodePoints } from './collation.js'
|
||||
import { prodLog } from './logger.js'
|
||||
import { getGlobalCache, UnifiedCache } from './unifiedCache.js'
|
||||
import {
|
||||
computeWatermarkVerdict,
|
||||
makeProjectionStamp,
|
||||
readStampedWatermark,
|
||||
type WatermarkVerdict,
|
||||
type WatermarkVerdictResult
|
||||
} from './projectionWatermark.js'
|
||||
import {
|
||||
NounType,
|
||||
VerbType,
|
||||
|
|
@ -109,6 +116,15 @@ interface FieldStats {
|
|||
normalizationStrategy?: 'none' | 'precision' | 'bucket'
|
||||
}
|
||||
|
||||
/**
|
||||
* Storage key for the metadata projection's watermark stamp — a sidecar
|
||||
* record beside the artifact (field registry + field indexes + chunked
|
||||
* sparse indexes + column-store segments + id-mapper records). Written LAST
|
||||
* in {@link MetadataIndexManager.flush} so stamp-after-data ordering holds
|
||||
* for every byte the stamp certifies.
|
||||
*/
|
||||
export const METADATA_INDEX_STAMP_KEY = '__index_metadata_watermark__'
|
||||
|
||||
/**
|
||||
* Implements {@link MetadataIndexProvider}: the metadata-index surface Brainy
|
||||
* calls on whatever the `'metadataIndex'` provider resolves to (its own
|
||||
|
|
@ -124,6 +140,14 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
private lastFlushTime = Date.now()
|
||||
private autoFlushThreshold = 10 // Start with 10 for more frequent non-blocking flushes
|
||||
|
||||
// --- Watermark stamp state (see utils/projectionWatermark for the law) ---
|
||||
/** Generation handed in via {@link stampWatermark}, awaiting the next flush. */
|
||||
private pendingWatermark: number | null = null
|
||||
/** Last watermark durably stamped by this instance or loaded at init. */
|
||||
private stampedWatermark: number | null = null
|
||||
/** The three-way verdict computed at init; null until init runs. */
|
||||
private loadVerdict: WatermarkVerdictResult | null = null
|
||||
|
||||
// Cardinality and field statistics tracking
|
||||
private fieldStats = new Map<string, FieldStats>()
|
||||
private cardinalityUpdateInterval = 100 // Update cardinality every N operations
|
||||
|
|
@ -250,6 +274,13 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
// Must run first to populate fieldIndexes directory before warming cache
|
||||
await this.loadFieldRegistry()
|
||||
|
||||
// Compute the watermark verdict for the persisted artifact BEFORE any
|
||||
// early return below — the verdict is recorded for every open, whether
|
||||
// the workspace is empty, rebuilding, or warm. Computed and exposed
|
||||
// only: today's rebuild triggers are unchanged (acting on 'catchup' —
|
||||
// the incremental fold — lands with the coordinator's wiring).
|
||||
await this.loadWatermarkVerdict()
|
||||
|
||||
// Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage)
|
||||
await this.idMapper.init()
|
||||
|
||||
|
|
@ -2599,6 +2630,10 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
|
||||
// Check if we have anything else to flush
|
||||
if (this.dirtyFields.size === 0) {
|
||||
// Nothing dirty — but a pending watermark still stamps (the registry
|
||||
// + id-mapper writes above are the only bytes this pass touched, and
|
||||
// they are durable at this point). Stamp-after-data holds.
|
||||
await this.writePendingStamp()
|
||||
return // No dirty field indexes to flush
|
||||
}
|
||||
|
||||
|
|
@ -2638,8 +2673,131 @@ export class MetadataIndexManager implements MetadataIndexProvider {
|
|||
if (this.columnStore) {
|
||||
await this.columnStore.flush()
|
||||
}
|
||||
|
||||
// STAMP-AFTER-DATA: the watermark stamp is the LAST write of the flush —
|
||||
// every byte it certifies (field indexes, registry, id-mapper records,
|
||||
// column-store segments) is durable before the stamp lands. A crash
|
||||
// anywhere above leaves the artifact behind-stamped or unstamped, which
|
||||
// verdicts as catchup/rescan on the next open — never a wrong adopt.
|
||||
await this.writePendingStamp()
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @description Record the committed generation this projection reflects.
|
||||
* The stamp is NOT written here — it is written as the final storage write
|
||||
* of the next {@link flush} (stamp-after-data ordering is a module
|
||||
* guarantee, not a caller obligation). The coordinator calls this with the
|
||||
* store's committed generation right before flushing.
|
||||
* @param generation - The committed generation every flushed byte reflects.
|
||||
*/
|
||||
stampWatermark(generation: number): void {
|
||||
this.pendingWatermark = generation
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The projection's current watermark: the stamp loaded at
|
||||
* init (or the last stamp durably written by this instance). Null =
|
||||
* unstamped (legacy artifact, first boot, or stamping never wired).
|
||||
*/
|
||||
watermark(): number | null {
|
||||
return this.stampedWatermark
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The three-way adoption verdict computed at init —
|
||||
* `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped <
|
||||
* committed; the gap from {@link watermarkGap} awaits an incremental
|
||||
* fold), `'rescan'` (unstamped or stamped above committed — never
|
||||
* trusted). Null until init() has run. Computed and exposed only; no
|
||||
* load behavior changes ride on it yet.
|
||||
*/
|
||||
watermarkVerdict(): WatermarkVerdict | null {
|
||||
return this.loadVerdict?.verdict ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The catch-up window `(from, to]` when the init verdict was
|
||||
* `'catchup'`; null otherwise.
|
||||
*/
|
||||
watermarkGap(): { from: number; to: number } | null {
|
||||
return this.loadVerdict?.gap ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Write the pending watermark stamp as a sidecar record —
|
||||
* always called AFTER the data it certifies is durable. A stamp-write
|
||||
* failure is fail-safe (the artifact stays unstamped/behind → rescan or
|
||||
* catchup on next open, never a wrong adopt) but is said out loud and the
|
||||
* pending stamp is retained for the next flush.
|
||||
*/
|
||||
private async writePendingStamp(): Promise<void> {
|
||||
if (this.pendingWatermark === null) return
|
||||
const watermark = this.pendingWatermark
|
||||
try {
|
||||
await this.storage.saveMetadata(METADATA_INDEX_STAMP_KEY, {
|
||||
noun: 'IndexWatermark',
|
||||
...makeProjectionStamp(watermark)
|
||||
})
|
||||
this.stampedWatermark = watermark
|
||||
this.pendingWatermark = null
|
||||
} catch (error) {
|
||||
prodLog.error(
|
||||
`[MetadataIndex] failed to write watermark stamp (generation ${watermark}) — ` +
|
||||
`artifact stays behind-stamped (safe: verdicts catchup/rescan, never wrong-adopt); ` +
|
||||
`retrying on next flush:`,
|
||||
error
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the artifact's stamp and compute the three-way verdict
|
||||
* against the store's committed generation. Unstamped state on a stamped
|
||||
* store verdicts `'rescan'` LOUDLY — never a silent adopt.
|
||||
*
|
||||
* MIGRATION COST: existing pre-stamp brains verdict `'rescan'` exactly
|
||||
* once (this open re-derives from source as it already does today); the
|
||||
* next flush stamps them, and every later open adopts.
|
||||
*/
|
||||
private async loadWatermarkVerdict(): Promise<void> {
|
||||
const committed = this.storage.committedGeneration?.() ?? null
|
||||
let stamped: number | null = null
|
||||
try {
|
||||
const record = await this.storage.getMetadata(METADATA_INDEX_STAMP_KEY)
|
||||
stamped = readStampedWatermark(record)
|
||||
} catch {
|
||||
// An unreadable stamp is unstamped — the fail-safe direction.
|
||||
stamped = null
|
||||
}
|
||||
const result = computeWatermarkVerdict(stamped, committed)
|
||||
this.loadVerdict = result
|
||||
this.stampedWatermark = stamped
|
||||
|
||||
if (result.verdict === 'rescan') {
|
||||
const artifactPresent = this.fieldIndexes.size > 0 || stamped !== null
|
||||
if (artifactPresent) {
|
||||
prodLog.warn(
|
||||
`[MetadataIndex] watermark verdict: RESCAN — persisted index is ` +
|
||||
(stamped === null
|
||||
? 'unstamped (legacy pre-stamp artifact, or a crash between data and stamp)'
|
||||
: `stamped at generation ${stamped}, ABOVE the store's committed generation ${committed}`) +
|
||||
` — never adopting unverifiable state`
|
||||
)
|
||||
} else {
|
||||
prodLog.debug(
|
||||
'[MetadataIndex] watermark verdict: rescan (no persisted artifact — first boot)'
|
||||
)
|
||||
}
|
||||
} else if (result.verdict === 'catchup') {
|
||||
prodLog.info(
|
||||
`[MetadataIndex] watermark verdict: catchup — index stamped at generation ` +
|
||||
`${stamped}, store committed at ${committed}; the (${stamped}, ${committed}] ` +
|
||||
`window awaits an incremental fold (verdict exposed; the fold lands with the ` +
|
||||
`coordinator's wiring)`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Yield control back to the Node.js event loop
|
||||
* Prevents blocking during long-running operations
|
||||
|
|
|
|||
150
src/utils/projectionWatermark.ts
Normal file
150
src/utils/projectionWatermark.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
/**
|
||||
* @module utils/projectionWatermark
|
||||
* @description The watermark-stamp contract shared by Brainy's persisted TS
|
||||
* projections (metadata index, JS HNSW vector index, graph adjacency index).
|
||||
*
|
||||
* THE LAW: every persisted projection artifact carries a stamp asserting
|
||||
* "this state reflects every committed generation ≤ watermark and nothing
|
||||
* above it, atomically". STAMP-AFTER-DATA: the stamp is written only after
|
||||
* every byte it certifies is durable — a crash between data and stamp leaves
|
||||
* the artifact unstamped, which verdicts as a rescan, never a wrong adopt.
|
||||
*
|
||||
* At load, each owner computes a three-way verdict against the store's
|
||||
* committed generation — the same rule and verdict names the aggregation
|
||||
* machinery ships (see `AggregationIndex.stateAdoptionVerdict`):
|
||||
*
|
||||
* - `'adopt'` — stamped == committed (clean reopen, zero work), or the
|
||||
* store exposes no committed generation at all (pre-stamp
|
||||
* stores keep their pre-stamp behavior).
|
||||
* - `'catchup'` — stamped < committed (an unclean exit after later writes,
|
||||
* or a long-lived writer whose last stamp predates recent
|
||||
* commits). The artifact is exact AS OF its stamp, so the
|
||||
* missing window `(stamped, committed]` can be folded
|
||||
* incrementally — at-least-once idempotent, bounded by
|
||||
* writes since the stamp, never by store size.
|
||||
* - `'rescan'` — unstamped (a legacy pre-stamp artifact, or a crash between
|
||||
* data and stamp) or stamped ABOVE committed (e.g. a log
|
||||
* truncation on a copied store pulled the watermark back):
|
||||
* the state over-claims unverifiably — one exact rescan,
|
||||
* said out loud, never a silent adopt.
|
||||
*
|
||||
* MIGRATION COST (stated once, honored by every owner): existing pre-stamp
|
||||
* brains verdict `'rescan'` exactly once — they re-derive from source on
|
||||
* that open, the next flush stamps them, and every later open adopts.
|
||||
*
|
||||
* The verdict is COMPUTED AND EXPOSED by each owner; acting on `'catchup'`
|
||||
* (the incremental fold) lands with the owner's coordinator wiring.
|
||||
*/
|
||||
|
||||
/** The three-way load verdict for a persisted projection artifact. */
|
||||
export type WatermarkVerdict = 'adopt' | 'catchup' | 'rescan'
|
||||
|
||||
/**
|
||||
* Format version written into every projection stamp. Bump when the stamp
|
||||
* record's shape changes incompatibly; readers treat an unknown version as
|
||||
* unstamped (→ rescan) rather than guessing.
|
||||
*/
|
||||
export const PROJECTION_STAMP_FORMAT_VERSION = 1
|
||||
|
||||
/**
|
||||
* @description The stamp record a projection writes into (or beside) its
|
||||
* persisted artifact, always AFTER the data it certifies is durable.
|
||||
*/
|
||||
export interface ProjectionStamp {
|
||||
/** The committed generation this artifact reflects, exactly and entirely. */
|
||||
watermark: number
|
||||
/** {@link PROJECTION_STAMP_FORMAT_VERSION} at write time. */
|
||||
formatVersion: number
|
||||
/** Wall-clock ms at stamp write — diagnostic only, never load-bearing. */
|
||||
stampedAt: number
|
||||
/**
|
||||
* Identity of the vector space for vector-bearing artifacts (the HNSW
|
||||
* index). The JS index has no reachable embedding-model id in its module,
|
||||
* so dimensions are the only identity it can honestly assert.
|
||||
*/
|
||||
modelIdentity?: { embedModelId?: string; dimensions: number | null }
|
||||
}
|
||||
|
||||
/** The verdict plus everything the owner needs to report or act on it. */
|
||||
export interface WatermarkVerdictResult {
|
||||
verdict: WatermarkVerdict
|
||||
/** Watermark read from the artifact's stamp; null = unstamped. */
|
||||
stamped: number | null
|
||||
/** The store's committed generation at load; null = no capability. */
|
||||
committed: number | null
|
||||
/** The catch-up window `(from, to]` when verdict is `'catchup'`, else null. */
|
||||
gap: { from: number; to: number } | null
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Build a stamp record for a projection artifact.
|
||||
* @param watermark - The committed generation the artifact reflects.
|
||||
* @param modelIdentity - Vector-space identity for vector-bearing artifacts.
|
||||
* @returns The stamp record to persist (stamp-after-data).
|
||||
*/
|
||||
export function makeProjectionStamp(
|
||||
watermark: number,
|
||||
modelIdentity?: ProjectionStamp['modelIdentity']
|
||||
): ProjectionStamp {
|
||||
const stamp: ProjectionStamp = {
|
||||
watermark,
|
||||
formatVersion: PROJECTION_STAMP_FORMAT_VERSION,
|
||||
stampedAt: Date.now()
|
||||
}
|
||||
if (modelIdentity !== undefined) stamp.modelIdentity = modelIdentity
|
||||
return stamp
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Read the stamped watermark out of a persisted record, treating
|
||||
* anything malformed (missing, wrong type, non-finite, negative, or an
|
||||
* unknown format version) as unstamped — the fail-safe direction is rescan,
|
||||
* never a guessed adopt.
|
||||
* @param record - The raw persisted record (or null/undefined).
|
||||
* @returns The stamped watermark, or null if effectively unstamped.
|
||||
*/
|
||||
export function readStampedWatermark(record: unknown): number | null {
|
||||
if (record === null || typeof record !== 'object') return null
|
||||
const rec = record as Record<string, unknown>
|
||||
const version = rec.formatVersion
|
||||
if (typeof version !== 'number' || version > PROJECTION_STAMP_FORMAT_VERSION) {
|
||||
return null
|
||||
}
|
||||
const raw = rec.watermark
|
||||
if (typeof raw !== 'number' || !Number.isFinite(raw) || raw < 0) return null
|
||||
return raw
|
||||
}
|
||||
|
||||
/**
|
||||
* @description The three-way adoption verdict — the single decision rule
|
||||
* every stamped projection shares (mirrors the aggregation machinery's
|
||||
* `stateAdoptionVerdict` exactly: same names, same directions).
|
||||
* @param stamped - Watermark read from the artifact ({@link readStampedWatermark}).
|
||||
* @param committed - The store's committed generation (null = no capability).
|
||||
* @returns The verdict with the stamped/committed pair and the catch-up gap.
|
||||
*/
|
||||
export function computeWatermarkVerdict(
|
||||
stamped: number | null,
|
||||
committed: number | null
|
||||
): WatermarkVerdictResult {
|
||||
// No committed-generation capability: hash/shape checks are the only
|
||||
// adoption gate, exactly the pre-stamp behavior. Never fail a store that
|
||||
// cannot express the question.
|
||||
if (committed === null) {
|
||||
return { verdict: 'adopt', stamped, committed, gap: null }
|
||||
}
|
||||
if (stamped === committed) {
|
||||
return { verdict: 'adopt', stamped, committed, gap: null }
|
||||
}
|
||||
if (stamped !== null && stamped < committed) {
|
||||
return {
|
||||
verdict: 'catchup',
|
||||
stamped,
|
||||
committed,
|
||||
gap: { from: stamped, to: committed }
|
||||
}
|
||||
}
|
||||
// Unstamped, or stamped above committed: unverifiable — rescan, loudly
|
||||
// (the caller owns the loud log so it can name its projection).
|
||||
return { verdict: 'rescan', stamped, committed, gap: null }
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue