fix(aggregation): the lifecycle cluster — flush stamps, behind-stamp catches up incrementally, the native rebuild finally gets invoked, deletes are never silently skipped

SELF-ENGINE-LIFECYCLE-SPRINT + BRAINY-PROD-LATENCY-TRIAD, the four asks:

(a) brain.flush() persists aggregation state stamped at the committed
    generation. The stamp used to advance only at close(), so a long-lived
    writer that flushes but never closes — the primary production shape —
    left every write window behind the stamp, and ANY unclean exit forced a
    whole-store backfill walk (per-entity work, measured >60s and
    door-starving on a 9k-row production brain) on the first stats call.

(b) BEHIND-stamp adoption becomes adopt + INCREMENTAL CATCH-UP: the exact
    missing window (stamp, committed] resolves its affected-id set from the
    fact log and reconciles each entity with time-travel before/after reads
    (asOf at both window bounds) through the same delta algebra the live
    hooks use — cost bounded by writes since the last flush, never store
    size, and exact under interleaving because reconciliation targets the
    FIXED window end while later writes chain through hooks. Oversized
    windows (>5000 affected) and unreadable windows demote to the announced
    rescan — never a silent partial serve.

(c) The native provider's parallel rebuildAggregate — on the contract since
    8.x but never invoked anywhere — is now the backfill walk's preferred
    door: one call per aggregate with source-matched entities, replacing
    the per-entity FFI stream.

(d) A delete whose before-image is unavailable can no longer SKIP the
    aggregation hook silently (counts drifted upward forever): both delete
    paths (remove() and transact) flag an exact rescan, loudly.

Pins: integration (flush stamp; unclean-exit reopen → exact counts through
an add + group-move + delete window with the walk spy proving ZERO
whole-store walks) + unit (provider rebuild invoked once with filtered
entities; flagAllForRescan; reconcile delta algebra). Gates: unit 1913/1913
· integration 760 · conformance 27/27.

src/brainy.ts

@@ -683,6 +683,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
   private _pendingMigrationRunner?: MigrationRunner  // Deferred migration runner for large datasets
   private _aggregationIndex?: AggregationIndex       // Incremental aggregation engine
   private _aggregationBackfillFlight: Promise<void> | null = null  // Single-flight backfill walk
+  private _aggregationCatchUpFlight: Promise<void> | null = null   // Single-flight behind-stamp catch-up
   // A failed walk latches its error: retries within the cooldown rethrow it
   // instantly instead of re-walking, so a tight caller-side retry loop costs
   // one loud error per query, never a full store walk per query.
@@ -3063,12 +3064,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       // Aggregation hook (outside transaction — derived data). The view must
       // carry EVERY reserved field top-level (not a subset): a groupBy on
       // subtype/visibility/etc. otherwise decrements a nonexistent group and
-      // the real count never comes down.
-      if (this._aggregationIndex && metadata) {
-        this._aggregationIndex.onEntityDeleted(
-          id,
-          this.entityForAggFromRawRecord(metadata as Record<string, unknown>)
-        )
+      // the real count never comes down. A delete whose before-image is
+      // unavailable can no longer SKIP the hook silently (the gated skip let
+      // counts drift upward forever) — it flags an exact rescan, loudly.
+      if (this._aggregationIndex) {
+        if (metadata) {
+          this._aggregationIndex.onEntityDeleted(
+            id,
+            this.entityForAggFromRawRecord(metadata as Record<string, unknown>)
+          )
+        } else {
+          this._aggregationIndex.flagAllForRescan(
+            `delete of ${id} carried no before-image metadata — contribution unknowable`
+          )
+        }
       }
   }
 
@@ -9426,6 +9435,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
           this._aggregationIndex.onEntityDeleted(id, entityForAgg)
         }
       })
+    } else {
+      // Un-gated (mirror of remove()): a before-image-less delete flags an
+      // exact rescan instead of silently skipping the decrement.
+      plan.postCommit.push(() => {
+        this._aggregationIndex?.flagAllForRescan(
+          `transact delete of ${id} carried no before-image metadata — contribution unknowable`
+        )
+      })
     }
 
     state.nouns.delete(id)
@@ -10403,7 +10420,22 @@ export class Brainy<T = any> implements BrainyInterface<T> {
 
       // 5. Persist the generation counter (8.0 MVCC — coalesced single-op
       //    bumps become durable on every explicit flush)
-      this.generationStore.persistCounterNow()
+      this.generationStore.persistCounterNow(),
+
+      // 6. Persist aggregation state, stamped at the committed generation
+      //    (BRAINY-PROD-LATENCY-TRIAD / SELF-ENGINE-LIFECYCLE-SPRINT ask (a)):
+      //    aggregation used to persist ONLY at close(), so a long-lived
+      //    writer that flushes but never closes — the primary production
+      //    shape — left its stamp behind after every write window, and any
+      //    unclean exit forced a WHOLE-STORE backfill walk on the next
+      //    first stats call (measured >60s and door-starving on a 9k-row
+      //    production brain). Flushing here keeps the stamp current, so a
+      //    reopen adopts (or incrementally catches up) instead of rescanning.
+      (async () => {
+        if (this._aggregationIndex) {
+          await this._aggregationIndex.flush()
+        }
+      })()
     ])
 
     // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY
@@ -16105,6 +16137,20 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     // persisted state is NOT listed — no walk at all on a clean reopen).
     await index.ready()
 
+    // Behind-stamp catch-up FIRST (SELF-ENGINE-LIFECYCLE-SPRINT ask (b)):
+    // adopted-but-behind state reconciles its exact missing window
+    // incrementally — bounded by that window's affected entities — instead
+    // of the whole-store rescan an unclean exit used to force. Single-flight
+    // like the walk below; a failed catch-up demotes to a LOUD rescan.
+    if (index.getPendingCatchUps().length > 0) {
+      if (!this._aggregationCatchUpFlight) {
+        this._aggregationCatchUpFlight = this.runAggregationCatchUp().finally(() => {
+          this._aggregationCatchUpFlight = null
+        })
+      }
+      await this._aggregationCatchUpFlight
+    }
+
     // Single-flight: concurrent queries share ONE walk instead of each wiping
     // the others' partial state and starting their own (the stampede that kept
     // a busy store from ever converging). The loop covers the rare case where
@@ -16133,6 +16179,128 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     }
   }
 
+  /**
+   * @description Build the aggregation view of a LIVE entity — top-level
+   * engine fields + the user bag, the same shape `entityForIndexing` and
+   * `entityForAggFromRawRecord` produce, so group keys and source filters
+   * resolve identically whichever door an entity arrives through.
+   */
+  private aggViewFromEntity(e: Entity<T>): Record<string, unknown> {
+    return {
+      type: e.type,
+      ...(e.subtype !== undefined && { subtype: e.subtype }),
+      ...((e as unknown as Record<string, unknown>).visibility !== undefined && {
+        visibility: (e as unknown as Record<string, unknown>).visibility
+      }),
+      ...(e.confidence !== undefined && { confidence: e.confidence }),
+      ...(e.weight !== undefined && { weight: e.weight }),
+      createdAt: e.createdAt,
+      updatedAt: e.updatedAt,
+      ...(e.service !== undefined && { service: e.service }),
+      ...(e.data !== undefined && { data: e.data }),
+      ...(e.createdBy !== undefined && { createdBy: e.createdBy }),
+      metadata: e.metadata ?? {}
+    }
+  }
+
+  /** Cap on a catch-up window's affected-entity count before demoting to a rescan. */
+  private static readonly AGGREGATION_CATCHUP_MAX_AFFECTED = 5000
+
+  /**
+   * Reconcile every behind-stamp aggregate's exact missing window
+   * `(from, to]` using the fact log for the AFFECTED ID SET and time-travel
+   * reads for exact before/after states — cost bounded by writes since the
+   * last flush, never store size. Reconciliation targets the FIXED window
+   * end (`to` = the committed generation at adoption), so live write hooks
+   * compose exactly: every application on both paths is a precise old/new
+   * delta pair, and interleaving cannot drift totals. Any failure or an
+   * oversized window demotes to the announced full rescan — never a silent
+   * partial serve.
+   */
+  private async runAggregationCatchUp(): Promise<void> {
+    const index = this._aggregationIndex!
+    const catchups = index.getPendingCatchUps()
+    if (catchups.length === 0) return
+
+    const startedAt = Date.now()
+    try {
+      // One fact scan covers every window (they share flush boundaries in
+      // practice); per-name windows filter per id below.
+      const from = Math.min(...catchups.map(c => c.from))
+      const to = Math.max(...catchups.map(c => c.to))
+      const scan = this.scanFacts({ fromGeneration: from + 1, toGeneration: to, kinds: ['noun'] })
+      if (!scan) {
+        for (const c of catchups) {
+          index.demoteCatchUpToBackfill(c.name, 'no fact log on this store — window unreadable')
+        }
+        return
+      }
+
+      // id → generations it changed at, inside the union window.
+      const affected = new Map<string, number[]>()
+      for await (const batch of scan.batches()) {
+        for (const fact of batch.facts) {
+          for (const op of fact.ops) {
+            if (op.kind !== 'noun') continue
+            const gens = affected.get(op.id)
+            if (gens) gens.push(fact.generation)
+            else affected.set(op.id, [fact.generation])
+          }
+        }
+        if (affected.size > Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED) break
+      }
+      if (affected.size > Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED) {
+        for (const c of catchups) {
+          index.demoteCatchUpToBackfill(
+            c.name,
+            `window touches >${Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED} entities — a rescan is cheaper`
+          )
+        }
+        return
+      }
+
+      // Exact before/after views per unique generation bound, via time travel.
+      const dbCache = new Map<number, Db<T>>()
+      const dbAt = async (gen: number): Promise<Db<T>> => {
+        let db = dbCache.get(gen)
+        if (!db) {
+          db = await this.asOf(gen)
+          dbCache.set(gen, db)
+        }
+        return db
+      }
+      try {
+        for (const c of catchups) {
+          const beforeDb = await dbAt(c.from)
+          const afterDb = await dbAt(c.to)
+          let reconciled = 0
+          for (const [id, gens] of affected) {
+            if (!gens.some(g => g > c.from && g <= c.to)) continue
+            const [before, after] = await Promise.all([beforeDb.get(id), afterDb.get(id)])
+            index.reconcileEntity(
+              c.name,
+              id,
+              before ? this.aggViewFromEntity(before) : null,
+              after ? this.aggViewFromEntity(after) : null
+            )
+            reconciled++
+          }
+          index.finishCatchUp(c.name)
+          prodLog.info(
+            `[Aggregation] '${c.name}': caught up generations ${c.from}→${c.to} — ` +
+              `${reconciled} entit${reconciled === 1 ? 'y' : 'ies'} reconciled in ${Date.now() - startedAt}ms (no store rescan)`
+          )
+        }
+      } finally {
+        await Promise.all(Array.from(dbCache.values(), db => db.release().catch(() => {})))
+      }
+    } catch (err) {
+      for (const c of index.getPendingCatchUps()) {
+        index.demoteCatchUpToBackfill(c.name, `catch-up failed: ${(err as Error).message}`)
+      }
+    }
+  }
+
   /**
    * One store walk fills EVERY aggregate currently pending backfill — M pending
    * aggregates cost one enumeration, not M. Only reached when an aggregate
@@ -16149,6 +16317,16 @@ export class Brainy<T = any> implements BrainyInterface<T> {
     const startedAt = Date.now()
     for (const n of names) index.beginBackfill(n)
 
+    // SELF-ENGINE-LIFECYCLE-SPRINT ask (c): when the native provider offers
+    // the parallel whole-rebuild (`rebuildAggregate` — on the contract since
+    // 8.x but never invoked), collect the walk's views and hand them over in
+    // ONE call per aggregate instead of a per-entity FFI stream. Memory note:
+    // the collected views are metadata-only records (no vectors); at the
+    // scales where this walk is even reached the array is the cheap part —
+    // the per-entity FFI round-trips were the measured cost.
+    const useProviderRebuild = index.hasProviderRebuild()
+    const collected: Array<Record<string, unknown>> = []
+
     let scanned = 0
     try {
       const PAGE = 500
@@ -16160,8 +16338,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
         })
         for (const noun of page.items) {
           const record = noun as unknown as Record<string, unknown>
-          for (const n of names) {
-            index.backfillEntity(n, record)
+          if (useProviderRebuild) {
+            collected.push(record)
+          } else {
+            for (const n of names) {
+              index.backfillEntity(n, record)
+            }
           }
         }
         scanned += page.items.length
@@ -16194,10 +16376,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
       throw err
     }
 
-    for (const n of names) index.finishBackfill(n)
+    if (useProviderRebuild) {
+      for (const n of names) {
+        if (!index.rebuildWithProvider(n, collected)) {
+          // Provider refused/absent for this one — stream it the JS way.
+          for (const record of collected) index.backfillEntity(n, record)
+          index.finishBackfill(n)
+        }
+      }
+    } else {
+      for (const n of names) index.finishBackfill(n)
+    }
     this._aggregationBackfillFailure = null
     prodLog.info(
-      `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) in ${Date.now() - startedAt}ms`
+      `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) ` +
+        `in ${Date.now() - startedAt}ms${useProviderRebuild ? ' (native parallel rebuild)' : ''}`
     )
   }