feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data

Every persisted projection artifact (metadata field indexes + column
segments, HNSW node records, graph adjacency LSM trees) now carries a
stamp asserting 'this state reflects every committed generation ≤ W,
atomically' — written LAST in each owner's flush (stamp-after-data: a
crash between data and stamp = unstamped = rescan, never trust). At load,
each owner computes the three-way verdict: stamped==committed → adopt
(zero work) · behind → catchup (gap reported) · above/unstamped → RESCAN,
loudly. Legacy artifacts re-derive once, then are stamped forever. Shared
law in projectionWatermark.ts (the aggregation verdict machinery,
generalized); vector artifacts carry model dimensions. Verdicts are
computed and exposed (watermark()/watermarkVerdict()/watermarkGap());
rebuild triggers unchanged — acting on 'catchup' is the fold train.

Pins: 22 unit (7 metadata · 8 hnsw · 7 graph, incl. spy-order
stamp-after-data) + the end-to-end reopen-adopts pin.

src/utils/metadataIndex.ts

@@ -13,6 +13,13 @@ import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCac
 import { compareCodePoints } from './collation.js'
 import { prodLog } from './logger.js'
 import { getGlobalCache, UnifiedCache } from './unifiedCache.js'
+import {
+  computeWatermarkVerdict,
+  makeProjectionStamp,
+  readStampedWatermark,
+  type WatermarkVerdict,
+  type WatermarkVerdictResult
+} from './projectionWatermark.js'
 import {
   NounType,
   VerbType,
@@ -109,6 +116,15 @@ interface FieldStats {
   normalizationStrategy?: 'none' | 'precision' | 'bucket'
 }
 
+/**
+ * Storage key for the metadata projection's watermark stamp — a sidecar
+ * record beside the artifact (field registry + field indexes + chunked
+ * sparse indexes + column-store segments + id-mapper records). Written LAST
+ * in {@link MetadataIndexManager.flush} so stamp-after-data ordering holds
+ * for every byte the stamp certifies.
+ */
+export const METADATA_INDEX_STAMP_KEY = '__index_metadata_watermark__'
+
 /**
  * Implements {@link MetadataIndexProvider}: the metadata-index surface Brainy
  * calls on whatever the `'metadataIndex'` provider resolves to (its own
@@ -124,6 +140,14 @@ export class MetadataIndexManager implements MetadataIndexProvider {
   private lastFlushTime = Date.now()
   private autoFlushThreshold = 10 // Start with 10 for more frequent non-blocking flushes
 
+  // --- Watermark stamp state (see utils/projectionWatermark for the law) ---
+  /** Generation handed in via {@link stampWatermark}, awaiting the next flush. */
+  private pendingWatermark: number | null = null
+  /** Last watermark durably stamped by this instance or loaded at init. */
+  private stampedWatermark: number | null = null
+  /** The three-way verdict computed at init; null until init runs. */
+  private loadVerdict: WatermarkVerdictResult | null = null
+
   // Cardinality and field statistics tracking
   private fieldStats = new Map<string, FieldStats>()
   private cardinalityUpdateInterval = 100 // Update cardinality every N operations
@@ -250,6 +274,13 @@ export class MetadataIndexManager implements MetadataIndexProvider {
     // Must run first to populate fieldIndexes directory before warming cache
     await this.loadFieldRegistry()
 
+    // Compute the watermark verdict for the persisted artifact BEFORE any
+    // early return below — the verdict is recorded for every open, whether
+    // the workspace is empty, rebuilding, or warm. Computed and exposed
+    // only: today's rebuild triggers are unchanged (acting on 'catchup' —
+    // the incremental fold — lands with the coordinator's wiring).
+    await this.loadWatermarkVerdict()
+
     // Initialize EntityIdMapper (loads UUID ↔ integer mappings from storage)
     await this.idMapper.init()
 
@@ -2599,6 +2630,10 @@ export class MetadataIndexManager implements MetadataIndexProvider {
 
     // Check if we have anything else to flush
     if (this.dirtyFields.size === 0) {
+      // Nothing dirty — but a pending watermark still stamps (the registry
+      // + id-mapper writes above are the only bytes this pass touched, and
+      // they are durable at this point). Stamp-after-data holds.
+      await this.writePendingStamp()
       return // No dirty field indexes to flush
     }
 
@@ -2638,8 +2673,131 @@ export class MetadataIndexManager implements MetadataIndexProvider {
     if (this.columnStore) {
       await this.columnStore.flush()
     }
+
+    // STAMP-AFTER-DATA: the watermark stamp is the LAST write of the flush —
+    // every byte it certifies (field indexes, registry, id-mapper records,
+    // column-store segments) is durable before the stamp lands. A crash
+    // anywhere above leaves the artifact behind-stamped or unstamped, which
+    // verdicts as catchup/rescan on the next open — never a wrong adopt.
+    await this.writePendingStamp()
   }
-  
+
+  /**
+   * @description Record the committed generation this projection reflects.
+   * The stamp is NOT written here — it is written as the final storage write
+   * of the next {@link flush} (stamp-after-data ordering is a module
+   * guarantee, not a caller obligation). The coordinator calls this with the
+   * store's committed generation right before flushing.
+   * @param generation - The committed generation every flushed byte reflects.
+   */
+  stampWatermark(generation: number): void {
+    this.pendingWatermark = generation
+  }
+
+  /**
+   * @description The projection's current watermark: the stamp loaded at
+   * init (or the last stamp durably written by this instance). Null =
+   * unstamped (legacy artifact, first boot, or stamping never wired).
+   */
+  watermark(): number | null {
+    return this.stampedWatermark
+  }
+
+  /**
+   * @description The three-way adoption verdict computed at init —
+   * `'adopt'` (stamped == committed, zero work), `'catchup'` (stamped <
+   * committed; the gap from {@link watermarkGap} awaits an incremental
+   * fold), `'rescan'` (unstamped or stamped above committed — never
+   * trusted). Null until init() has run. Computed and exposed only; no
+   * load behavior changes ride on it yet.
+   */
+  watermarkVerdict(): WatermarkVerdict | null {
+    return this.loadVerdict?.verdict ?? null
+  }
+
+  /**
+   * @description The catch-up window `(from, to]` when the init verdict was
+   * `'catchup'`; null otherwise.
+   */
+  watermarkGap(): { from: number; to: number } | null {
+    return this.loadVerdict?.gap ?? null
+  }
+
+  /**
+   * @description Write the pending watermark stamp as a sidecar record —
+   * always called AFTER the data it certifies is durable. A stamp-write
+   * failure is fail-safe (the artifact stays unstamped/behind → rescan or
+   * catchup on next open, never a wrong adopt) but is said out loud and the
+   * pending stamp is retained for the next flush.
+   */
+  private async writePendingStamp(): Promise<void> {
+    if (this.pendingWatermark === null) return
+    const watermark = this.pendingWatermark
+    try {
+      await this.storage.saveMetadata(METADATA_INDEX_STAMP_KEY, {
+        noun: 'IndexWatermark',
+        ...makeProjectionStamp(watermark)
+      })
+      this.stampedWatermark = watermark
+      this.pendingWatermark = null
+    } catch (error) {
+      prodLog.error(
+        `[MetadataIndex] failed to write watermark stamp (generation ${watermark}) — ` +
+          `artifact stays behind-stamped (safe: verdicts catchup/rescan, never wrong-adopt); ` +
+          `retrying on next flush:`,
+        error
+      )
+    }
+  }
+
+  /**
+   * @description Read the artifact's stamp and compute the three-way verdict
+   * against the store's committed generation. Unstamped state on a stamped
+   * store verdicts `'rescan'` LOUDLY — never a silent adopt.
+   *
+   * MIGRATION COST: existing pre-stamp brains verdict `'rescan'` exactly
+   * once (this open re-derives from source as it already does today); the
+   * next flush stamps them, and every later open adopts.
+   */
+  private async loadWatermarkVerdict(): Promise<void> {
+    const committed = this.storage.committedGeneration?.() ?? null
+    let stamped: number | null = null
+    try {
+      const record = await this.storage.getMetadata(METADATA_INDEX_STAMP_KEY)
+      stamped = readStampedWatermark(record)
+    } catch {
+      // An unreadable stamp is unstamped — the fail-safe direction.
+      stamped = null
+    }
+    const result = computeWatermarkVerdict(stamped, committed)
+    this.loadVerdict = result
+    this.stampedWatermark = stamped
+
+    if (result.verdict === 'rescan') {
+      const artifactPresent = this.fieldIndexes.size > 0 || stamped !== null
+      if (artifactPresent) {
+        prodLog.warn(
+          `[MetadataIndex] watermark verdict: RESCAN — persisted index is ` +
+            (stamped === null
+              ? 'unstamped (legacy pre-stamp artifact, or a crash between data and stamp)'
+              : `stamped at generation ${stamped}, ABOVE the store's committed generation ${committed}`) +
+            ` — never adopting unverifiable state`
+        )
+      } else {
+        prodLog.debug(
+          '[MetadataIndex] watermark verdict: rescan (no persisted artifact — first boot)'
+        )
+      }
+    } else if (result.verdict === 'catchup') {
+      prodLog.info(
+        `[MetadataIndex] watermark verdict: catchup — index stamped at generation ` +
+          `${stamped}, store committed at ${committed}; the (${stamped}, ${committed}] ` +
+          `window awaits an incremental fold (verdict exposed; the fold lands with the ` +
+          `coordinator's wiring)`
+      )
+    }
+  }
+
   /**
    * Yield control back to the Node.js event loop
    * Prevents blocking during long-running operations