From 1dc861d299d3b39e05a43dc44cee41ceda900183 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 5 Aug 2026 15:49:12 -0700 Subject: [PATCH] =?UTF-8?q?fix(aggregation):=20the=20lifecycle=20cluster?= =?UTF-8?q?=20=E2=80=94=20flush=20stamps,=20behind-stamp=20catches=20up=20?= =?UTF-8?q?incrementally,=20the=20native=20rebuild=20finally=20gets=20invo?= =?UTF-8?q?ked,=20deletes=20are=20never=20silently=20skipped?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SELF-ENGINE-LIFECYCLE-SPRINT + BRAINY-PROD-LATENCY-TRIAD, the four asks: (a) brain.flush() persists aggregation state stamped at the committed generation. The stamp used to advance only at close(), so a long-lived writer that flushes but never closes — the primary production shape — left every write window behind the stamp, and ANY unclean exit forced a whole-store backfill walk (per-entity work, measured >60s and door-starving on a 9k-row production brain) on the first stats call. (b) BEHIND-stamp adoption becomes adopt + INCREMENTAL CATCH-UP: the exact missing window (stamp, committed] resolves its affected-id set from the fact log and reconciles each entity with time-travel before/after reads (asOf at both window bounds) through the same delta algebra the live hooks use — cost bounded by writes since the last flush, never store size, and exact under interleaving because reconciliation targets the FIXED window end while later writes chain through hooks. Oversized windows (>5000 affected) and unreadable windows demote to the announced rescan — never a silent partial serve. (c) The native provider's parallel rebuildAggregate — on the contract since 8.x but never invoked anywhere — is now the backfill walk's preferred door: one call per aggregate with source-matched entities, replacing the per-entity FFI stream. (d) A delete whose before-image is unavailable can no longer SKIP the aggregation hook silently (counts drifted upward forever): both delete paths (remove() and transact) flag an exact rescan, loudly. Pins: integration (flush stamp; unclean-exit reopen → exact counts through an add + group-move + delete window with the walk spy proving ZERO whole-store walks) + unit (provider rebuild invoked once with filtered entities; flagAllForRescan; reconcile delta algebra). Gates: unit 1913/1913 · integration 760 · conformance 27/27. --- src/aggregation/AggregationIndex.ts | 200 +++++++++++++--- src/brainy.ts | 215 +++++++++++++++++- .../aggregation-lifecycle-catchup.test.ts | 143 ++++++++++++ .../aggregation-provider-rebuild.test.ts | 134 +++++++++++ .../metadataIndex-nested-orderby.test.ts | 142 ++++++++++++ 5 files changed, 796 insertions(+), 38 deletions(-) create mode 100644 tests/integration/aggregation-lifecycle-catchup.test.ts create mode 100644 tests/unit/aggregation/aggregation-provider-rebuild.test.ts create mode 100644 tests/unit/utils/metadataIndex-nested-orderby.test.ts diff --git a/src/aggregation/AggregationIndex.ts b/src/aggregation/AggregationIndex.ts index ca44ac8b..9c221c84 100644 --- a/src/aggregation/AggregationIndex.ts +++ b/src/aggregation/AggregationIndex.ts @@ -371,6 +371,15 @@ export class AggregationIndex { */ private pendingAdopt = new Set() + /** + * Aggregates adopted with a BEHIND stamp: name → the exact generation + * window `(from, to]` whose writes the adopted state has not seen. The + * owner (Brainy) drains this via {@link getPendingCatchUps} + + * {@link reconcileEntity} + {@link finishCatchUp} BEFORE serving queries — + * cost bounded by the window's affected entities, never store size. + */ + private pendingCatchUp = new Map() + /** * In-flight rescan targets. While a name has a staging map, ALL * contributions (the walk's and concurrent write hooks') land there instead @@ -437,25 +446,47 @@ export class AggregationIndex { } /** - * May this persisted state be ADOPTED? When the store exposes its committed - * watermark, the state's `sourceGeneration` must EQUAL it: behind means - * later writes are missing from the state (unclean shutdown); ahead means - * it counts writes that no longer exist (e.g. a fact-log truncation on a - * copied store pulled the watermark back). Either way: one exact rescan, - * said out loud — never a silent adopt. Stores without the capability (and - * pre-stamp state on them) fall back to hash-only adoption. + * The adoption verdict for persisted state, against the store's committed + * watermark (SELF-ENGINE-LIFECYCLE-SPRINT ask (b) — behind-stamp is no + * longer a whole-store rescan): + * + * - `'adopt'` — stamp equals the watermark (clean), or the store has no + * watermark capability (hash-only adoption, the pre-stamp behavior). + * - `'catchup'` — stamp is BEHIND the watermark (an unclean exit after + * later writes, or a long-lived writer whose last flush predates recent + * writes). The state is exact AS OF its stamp, so it is adopted and the + * missing window `(stamp, committed]` is reconciled INCREMENTALLY per + * affected entity via time-travel reads — bounded by writes since the + * last flush, never by store size. The owner drains + * {@link getPendingCatchUps} before serving queries. + * - `'rescan'` — no stamp (pre-stamp state on a stamped store) or stamp + * AHEAD of the watermark (e.g. a fact-log truncation on a copied store + * pulled the watermark back): the state over-counts unverifiably; one + * exact rescan, said out loud. */ - private stateGenerationAdoptable(name: string, stateData: unknown): boolean { + private stateAdoptionVerdict( + name: string, + stateData: unknown + ): 'adopt' | 'catchup' | 'rescan' { const committed = this.storage.committedGeneration?.() ?? null - if (committed === null) return true + if (committed === null) return 'adopt' const raw = (stateData as Record).sourceGeneration const stamped = typeof raw === 'number' ? raw : null - if (stamped === committed) return true + if (stamped === committed) return 'adopt' + if (stamped !== null && stamped < committed) { + this.pendingCatchUp.set(name, { from: stamped, to: committed }) + prodLog.info( + `[Aggregation] '${name}': persisted state is at generation ${stamped}, store is at ` + + `${committed} — adopting and reconciling the ${committed - stamped}-generation window ` + + `incrementally (no store rescan)` + ) + return 'catchup' + } prodLog.warn( `[Aggregation] '${name}': persisted state is at generation ${stamped ?? 'unstamped'} ` + `but the store's committed generation is ${committed} — rescanning instead of adopting` ) - return false + return 'rescan' } private async loadPersisted(): Promise { @@ -476,20 +507,21 @@ export class AggregationIndex { const appHash = this.definitionHashes.get(def.name) || '' if (appHash === savedHash && this.pendingAdopt.has(def.name)) { const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) - if ( - stateData && - stateData.groups && - this.stateGenerationAdoptable(def.name, stateData) - ) { + const verdict = + stateData && stateData.groups + ? this.stateAdoptionVerdict(def.name, stateData) + : 'rescan' + if (verdict !== 'rescan') { const groupMap = new Map() - for (const group of stateData.groups as AggregateGroupState[]) { + for (const group of stateData!.groups as AggregateGroupState[]) { groupMap.set(serializeGroupKey(group.groupKey), group) } this.states.set(def.name, groupMap) this.pendingAdopt.delete(def.name) this.needsBackfill.delete(def.name) prodLog.info( - `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — no rescan` + `[Aggregation] '${def.name}': adopted persisted state (${groupMap.size} groups) — ` + + (verdict === 'catchup' ? 'incremental catch-up pending' : 'no rescan') ) } // No/invalid persisted state: stays in pendingAdopt and resolves @@ -504,22 +536,23 @@ export class AggregationIndex { const currentHash = hashDefinition(def) const stateData = await this.storage.getMetadata(`${STATE_KEY_PREFIX}${def.name}__`) - if ( - stateData && - stateData.groups && - savedHash === currentHash && - this.stateGenerationAdoptable(def.name, stateData) - ) { - // Definition unchanged — load state + const restoreVerdict = + stateData && stateData.groups && savedHash === currentHash + ? this.stateAdoptionVerdict(def.name, stateData) + : 'rescan' + if (restoreVerdict !== 'rescan') { + // Definition unchanged — load state (exact as of its stamp; a + // 'catchup' verdict reconciles the missing window incrementally). const groupMap = new Map() - for (const group of stateData.groups as AggregateGroupState[]) { + for (const group of stateData!.groups as AggregateGroupState[]) { const serialized = serializeGroupKey(group.groupKey) groupMap.set(serialized, group) } this.states.set(def.name, groupMap) this.needsBackfill.delete(def.name) prodLog.info( - `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)` + `[Aggregation] '${def.name}': restored definition + adopted persisted state (${groupMap.size} groups)` + + (restoreVerdict === 'catchup' ? ' — incremental catch-up pending' : '') ) } else { // Definition changed or no saved state — start fresh and backfill from @@ -747,6 +780,119 @@ export class AggregationIndex { this.dirty.add(name) } + // ============= Incremental Catch-Up (behind-stamp adoption) ============= + + /** The aggregates adopted behind the watermark, with their exact missing windows. */ + getPendingCatchUps(): Array<{ name: string; from: number; to: number }> { + return Array.from(this.pendingCatchUp, ([name, w]) => ({ name, ...w })) + } + + /** + * Reconcile ONE entity's contribution across a catch-up window using the + * same exact delta algebra the write-time hooks use: remove the + * contribution the adopted state counted (the entity AS OF the stamp), + * add the contribution it should count (AS OF the window's end). `null` + * on either side means the entity did not exist then. Composes exactly + * with live hooks because every application is a precise old/new pair — + * order between catch-up and post-window writes cannot drift the totals. + */ + reconcileEntity( + name: string, + id: string, + before: Record | null, + after: Record | null + ): void { + const def = this.definitions.get(name) + if (!def) return + if (before && after) { + if (isAggregateEntity(after)) return + const oldMatches = matchesSource(before, def.source) + const newMatches = matchesSource(after, def.source) + if (this.nativeProvider && (oldMatches || newMatches)) { + this.applyNativeResults( + name, + this.nativeProvider.incrementalUpdate(name, def, after, 'update', before) + ) + return + } + if (oldMatches) this.removeContribution(name, def, before) + if (newMatches) this.addContribution(name, def, after) + return + } + if (after) { + if (isAggregateEntity(after) || !matchesSource(after, def.source)) return + if (this.nativeProvider) { + this.applyNativeResults(name, this.nativeProvider.incrementalUpdate(name, def, after, 'add')) + } else { + this.addContribution(name, def, after) + } + return + } + if (before) { + if (isAggregateEntity(before) || !matchesSource(before, def.source)) return + if (this.nativeProvider) { + this.applyNativeResults(name, this.nativeProvider.incrementalUpdate(name, def, before, 'delete')) + } else { + this.removeContribution(name, def, before) + } + } + } + + /** Whether the native provider offers the parallel whole-rebuild path. */ + hasProviderRebuild(): boolean { + return typeof this.nativeProvider?.rebuildAggregate === 'function' + } + + /** The catch-up window for `name` is fully reconciled; state is current. */ + finishCatchUp(name: string): void { + this.pendingCatchUp.delete(name) + this.dirty.add(name) + } + + /** + * A catch-up could not complete (window unreadable, affected set over the + * bound, …): demote to an exact rescan, loudly — never serve un-reconciled. + */ + demoteCatchUpToBackfill(name: string, reason: string): void { + this.pendingCatchUp.delete(name) + this.needsBackfill.add(name) + prodLog.warn(`[Aggregation] '${name}': catch-up demoted to full rescan — ${reason}`) + } + + /** + * Rebuild an aggregate through the native provider's parallel path + * (SELF-ENGINE-LIFECYCLE-SPRINT ask (c) — `rebuildAggregate` existed on + * the provider contract but was never invoked; the JS walk fed + * per-entity FFI calls instead). Returns false when no provider rebuild + * exists — the caller streams the JS walk as before. + */ + rebuildWithProvider(name: string, entities: Array>): boolean { + const def = this.definitions.get(name) + if (!def || !this.nativeProvider?.rebuildAggregate) return false + const rebuilt = this.nativeProvider.rebuildAggregate( + def, + entities.filter(e => !isAggregateEntity(e) && matchesSource(e, def.source)) + ) + this.states.set(name, rebuilt) + this.backfillStaging.delete(name) + this.needsBackfill.delete(name) + this.dirty.add(name) + return true + } + + /** + * A write-path hook could not see the entity it needed (e.g. a delete + * whose before-image was unavailable): flag EVERY defined aggregate for + * an exact rescan, loudly — the counts must never silently drift + * (SELF-ENGINE-LIFECYCLE-SPRINT ask (d): the gated hook used to SKIP). + */ + flagAllForRescan(reason: string): void { + for (const name of this.definitions.keys()) this.needsBackfill.add(name) + prodLog.warn( + `[Aggregation] all ${this.definitions.size} aggregate(s) flagged for rescan — ${reason}` + ) + } + // ============= Write-Time Hooks ============= /** diff --git a/src/brainy.ts b/src/brainy.ts index 600e4474..2e1d2de0 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -683,6 +683,7 @@ export class Brainy implements BrainyInterface { private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets private _aggregationIndex?: AggregationIndex // Incremental aggregation engine private _aggregationBackfillFlight: Promise | null = null // Single-flight backfill walk + private _aggregationCatchUpFlight: Promise | null = null // Single-flight behind-stamp catch-up // A failed walk latches its error: retries within the cooldown rethrow it // instantly instead of re-walking, so a tight caller-side retry loop costs // one loud error per query, never a full store walk per query. @@ -3063,12 +3064,20 @@ export class Brainy implements BrainyInterface { // Aggregation hook (outside transaction — derived data). The view must // carry EVERY reserved field top-level (not a subset): a groupBy on // subtype/visibility/etc. otherwise decrements a nonexistent group and - // the real count never comes down. - if (this._aggregationIndex && metadata) { - this._aggregationIndex.onEntityDeleted( - id, - this.entityForAggFromRawRecord(metadata as Record) - ) + // the real count never comes down. A delete whose before-image is + // unavailable can no longer SKIP the hook silently (the gated skip let + // counts drift upward forever) — it flags an exact rescan, loudly. + if (this._aggregationIndex) { + if (metadata) { + this._aggregationIndex.onEntityDeleted( + id, + this.entityForAggFromRawRecord(metadata as Record) + ) + } else { + this._aggregationIndex.flagAllForRescan( + `delete of ${id} carried no before-image metadata — contribution unknowable` + ) + } } } @@ -9426,6 +9435,14 @@ export class Brainy implements BrainyInterface { this._aggregationIndex.onEntityDeleted(id, entityForAgg) } }) + } else { + // Un-gated (mirror of remove()): a before-image-less delete flags an + // exact rescan instead of silently skipping the decrement. + plan.postCommit.push(() => { + this._aggregationIndex?.flagAllForRescan( + `transact delete of ${id} carried no before-image metadata — contribution unknowable` + ) + }) } state.nouns.delete(id) @@ -10403,7 +10420,22 @@ export class Brainy implements BrainyInterface { // 5. Persist the generation counter (8.0 MVCC — coalesced single-op // bumps become durable on every explicit flush) - this.generationStore.persistCounterNow() + this.generationStore.persistCounterNow(), + + // 6. Persist aggregation state, stamped at the committed generation + // (BRAINY-PROD-LATENCY-TRIAD / SELF-ENGINE-LIFECYCLE-SPRINT ask (a)): + // aggregation used to persist ONLY at close(), so a long-lived + // writer that flushes but never closes — the primary production + // shape — left its stamp behind after every write window, and any + // unclean exit forced a WHOLE-STORE backfill walk on the next + // first stats call (measured >60s and door-starving on a 9k-row + // production brain). Flushing here keeps the stamp current, so a + // reopen adopts (or incrementally catches up) instead of rescanning. + (async () => { + if (this._aggregationIndex) { + await this._aggregationIndex.flush() + } + })() ]) // NOTE (8.9.0): flush() no longer compacts history. Flush is DURABILITY @@ -16105,6 +16137,20 @@ export class Brainy implements BrainyInterface { // persisted state is NOT listed — no walk at all on a clean reopen). await index.ready() + // Behind-stamp catch-up FIRST (SELF-ENGINE-LIFECYCLE-SPRINT ask (b)): + // adopted-but-behind state reconciles its exact missing window + // incrementally — bounded by that window's affected entities — instead + // of the whole-store rescan an unclean exit used to force. Single-flight + // like the walk below; a failed catch-up demotes to a LOUD rescan. + if (index.getPendingCatchUps().length > 0) { + if (!this._aggregationCatchUpFlight) { + this._aggregationCatchUpFlight = this.runAggregationCatchUp().finally(() => { + this._aggregationCatchUpFlight = null + }) + } + await this._aggregationCatchUpFlight + } + // Single-flight: concurrent queries share ONE walk instead of each wiping // the others' partial state and starting their own (the stampede that kept // a busy store from ever converging). The loop covers the rare case where @@ -16133,6 +16179,128 @@ export class Brainy implements BrainyInterface { } } + /** + * @description Build the aggregation view of a LIVE entity — top-level + * engine fields + the user bag, the same shape `entityForIndexing` and + * `entityForAggFromRawRecord` produce, so group keys and source filters + * resolve identically whichever door an entity arrives through. + */ + private aggViewFromEntity(e: Entity): Record { + return { + type: e.type, + ...(e.subtype !== undefined && { subtype: e.subtype }), + ...((e as unknown as Record).visibility !== undefined && { + visibility: (e as unknown as Record).visibility + }), + ...(e.confidence !== undefined && { confidence: e.confidence }), + ...(e.weight !== undefined && { weight: e.weight }), + createdAt: e.createdAt, + updatedAt: e.updatedAt, + ...(e.service !== undefined && { service: e.service }), + ...(e.data !== undefined && { data: e.data }), + ...(e.createdBy !== undefined && { createdBy: e.createdBy }), + metadata: e.metadata ?? {} + } + } + + /** Cap on a catch-up window's affected-entity count before demoting to a rescan. */ + private static readonly AGGREGATION_CATCHUP_MAX_AFFECTED = 5000 + + /** + * Reconcile every behind-stamp aggregate's exact missing window + * `(from, to]` using the fact log for the AFFECTED ID SET and time-travel + * reads for exact before/after states — cost bounded by writes since the + * last flush, never store size. Reconciliation targets the FIXED window + * end (`to` = the committed generation at adoption), so live write hooks + * compose exactly: every application on both paths is a precise old/new + * delta pair, and interleaving cannot drift totals. Any failure or an + * oversized window demotes to the announced full rescan — never a silent + * partial serve. + */ + private async runAggregationCatchUp(): Promise { + const index = this._aggregationIndex! + const catchups = index.getPendingCatchUps() + if (catchups.length === 0) return + + const startedAt = Date.now() + try { + // One fact scan covers every window (they share flush boundaries in + // practice); per-name windows filter per id below. + const from = Math.min(...catchups.map(c => c.from)) + const to = Math.max(...catchups.map(c => c.to)) + const scan = this.scanFacts({ fromGeneration: from + 1, toGeneration: to, kinds: ['noun'] }) + if (!scan) { + for (const c of catchups) { + index.demoteCatchUpToBackfill(c.name, 'no fact log on this store — window unreadable') + } + return + } + + // id → generations it changed at, inside the union window. + const affected = new Map() + for await (const batch of scan.batches()) { + for (const fact of batch.facts) { + for (const op of fact.ops) { + if (op.kind !== 'noun') continue + const gens = affected.get(op.id) + if (gens) gens.push(fact.generation) + else affected.set(op.id, [fact.generation]) + } + } + if (affected.size > Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED) break + } + if (affected.size > Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED) { + for (const c of catchups) { + index.demoteCatchUpToBackfill( + c.name, + `window touches >${Brainy.AGGREGATION_CATCHUP_MAX_AFFECTED} entities — a rescan is cheaper` + ) + } + return + } + + // Exact before/after views per unique generation bound, via time travel. + const dbCache = new Map>() + const dbAt = async (gen: number): Promise> => { + let db = dbCache.get(gen) + if (!db) { + db = await this.asOf(gen) + dbCache.set(gen, db) + } + return db + } + try { + for (const c of catchups) { + const beforeDb = await dbAt(c.from) + const afterDb = await dbAt(c.to) + let reconciled = 0 + for (const [id, gens] of affected) { + if (!gens.some(g => g > c.from && g <= c.to)) continue + const [before, after] = await Promise.all([beforeDb.get(id), afterDb.get(id)]) + index.reconcileEntity( + c.name, + id, + before ? this.aggViewFromEntity(before) : null, + after ? this.aggViewFromEntity(after) : null + ) + reconciled++ + } + index.finishCatchUp(c.name) + prodLog.info( + `[Aggregation] '${c.name}': caught up generations ${c.from}→${c.to} — ` + + `${reconciled} entit${reconciled === 1 ? 'y' : 'ies'} reconciled in ${Date.now() - startedAt}ms (no store rescan)` + ) + } + } finally { + await Promise.all(Array.from(dbCache.values(), db => db.release().catch(() => {}))) + } + } catch (err) { + for (const c of index.getPendingCatchUps()) { + index.demoteCatchUpToBackfill(c.name, `catch-up failed: ${(err as Error).message}`) + } + } + } + /** * One store walk fills EVERY aggregate currently pending backfill — M pending * aggregates cost one enumeration, not M. Only reached when an aggregate @@ -16149,6 +16317,16 @@ export class Brainy implements BrainyInterface { const startedAt = Date.now() for (const n of names) index.beginBackfill(n) + // SELF-ENGINE-LIFECYCLE-SPRINT ask (c): when the native provider offers + // the parallel whole-rebuild (`rebuildAggregate` — on the contract since + // 8.x but never invoked), collect the walk's views and hand them over in + // ONE call per aggregate instead of a per-entity FFI stream. Memory note: + // the collected views are metadata-only records (no vectors); at the + // scales where this walk is even reached the array is the cheap part — + // the per-entity FFI round-trips were the measured cost. + const useProviderRebuild = index.hasProviderRebuild() + const collected: Array> = [] + let scanned = 0 try { const PAGE = 500 @@ -16160,8 +16338,12 @@ export class Brainy implements BrainyInterface { }) for (const noun of page.items) { const record = noun as unknown as Record - for (const n of names) { - index.backfillEntity(n, record) + if (useProviderRebuild) { + collected.push(record) + } else { + for (const n of names) { + index.backfillEntity(n, record) + } } } scanned += page.items.length @@ -16194,10 +16376,21 @@ export class Brainy implements BrainyInterface { throw err } - for (const n of names) index.finishBackfill(n) + if (useProviderRebuild) { + for (const n of names) { + if (!index.rebuildWithProvider(n, collected)) { + // Provider refused/absent for this one — stream it the JS way. + for (const record of collected) index.backfillEntity(n, record) + index.finishBackfill(n) + } + } + } else { + for (const n of names) index.finishBackfill(n) + } this._aggregationBackfillFailure = null prodLog.info( - `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) in ${Date.now() - startedAt}ms` + `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) ` + + `in ${Date.now() - startedAt}ms${useProviderRebuild ? ' (native parallel rebuild)' : ''}` ) } diff --git a/tests/integration/aggregation-lifecycle-catchup.test.ts b/tests/integration/aggregation-lifecycle-catchup.test.ts new file mode 100644 index 00000000..d4f9e6bf --- /dev/null +++ b/tests/integration/aggregation-lifecycle-catchup.test.ts @@ -0,0 +1,143 @@ +/** + * @module tests/integration/aggregation-lifecycle-catchup + * @description THE AGGREGATION LIFECYCLE PINS (SELF-ENGINE-LIFECYCLE-SPRINT / + * BRAINY-PROD-LATENCY-TRIAD asks (a)+(b)). The production disease: the + * aggregation stamp persisted ONLY at close(), so a long-lived writer that + * flushes but never closes left its stamp behind after every write window — + * and the exact-match adoption rule then forced a WHOLE-STORE backfill walk + * (per-entity work, measured >60s and door-starving on a 9k-row production + * brain) on the first stats call after any unclean exit. + * + * The cures pinned here: + * (a) `brain.flush()` persists aggregation state, stamped at the committed + * generation — the stamp tracks every flush, not just close(). + * (b) BEHIND-stamp state is ADOPTED and reconciled INCREMENTALLY over its + * exact missing window (fact-log affected ids + time-travel before/after + * reads) — the full walk never runs for an unclean exit. Pinned by call + * shape (the walk spy), not by latency. + */ +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +const AGG = { + name: 'by_subtype', + source: { type: NounType.Document }, + groupBy: ['system.subtype'] as string[], + metrics: { count: { op: 'count' as const } } +} + +const dirs: string[] = [] +const brains: Brainy[] = [] + +async function open(dir: string): Promise { + const b = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) + await b.init() + brains.push(b) + return b +} + +function countFor(results: Array<{ groupKey: Record; metrics: Record }>, subtype: string): number { + const row = results.find(r => r.groupKey['system.subtype'] === subtype) + return row ? Number(row.metrics.count) : 0 +} + +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +describe('aggregation lifecycle — flush stamps, behind-stamp catches up incrementally', () => { + it('(a) brain.flush() persists aggregation state stamped at the committed generation', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-agg-flush-')) + dirs.push(dir) + const brain = await open(dir) + brain.defineAggregate(AGG) + await brain.add({ data: 'a', type: NounType.Document, subtype: 'invoice', metadata: {} }) + await brain.add({ data: 'b', type: NounType.Document, subtype: 'invoice', metadata: {} }) + await brain.queryAggregate(AGG.name) // settle backfill-on-define + + await brain.flush() + + const internals = brain as unknown as { + storage: { + getMetadata(k: string): Promise<{ sourceGeneration?: number } | null> + committedGeneration?(): number + } + } + const persisted = await internals.storage.getMetadata('__aggregation_state_by_subtype__') + expect(persisted, 'state persisted by flush(), not only close()').toBeTruthy() + expect( + persisted!.sourceGeneration, + 'stamp equals the committed generation at flush time' + ).toBe(internals.storage.committedGeneration?.()) + }) + + it('(b) an unclean exit reconciles incrementally — exact counts, ZERO full-store walks', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-agg-catchup-')) + dirs.push(dir) + + // Session 1: define + write + flush (stamps at G), then MORE writes of + // every kind (add / update-that-moves-groups / delete) and a clean close + // — but we then REWIND the persisted aggregation artifact to its at-G + // bytes, which is byte-for-byte the unclean-exit state: stamp G, store + // committed at G+k. + let brain = await open(dir) + brain.defineAggregate(AGG) + await brain.add({ data: 'a', type: NounType.Document, subtype: 'invoice', metadata: {} }) + await brain.add({ data: 'b', type: NounType.Document, subtype: 'invoice', metadata: {} }) + const moving = await brain.add({ data: 'c', type: NounType.Document, subtype: 'draft', metadata: {} }) + const doomed = await brain.add({ data: 'd', type: NounType.Document, subtype: 'draft', metadata: {} }) + await brain.queryAggregate(AGG.name) + await brain.flush() + + const internals = brain as unknown as { + storage: { + getMetadata(k: string): Promise | null> + saveMetadata(k: string, v: Record): Promise + } + } + const stateAtG = JSON.parse( + JSON.stringify(await internals.storage.getMetadata('__aggregation_state_by_subtype__')) + ) + + // The missing window: one add, one group-moving update, one delete. + await brain.add({ data: 'e', type: NounType.Document, subtype: 'invoice', metadata: {} }) + await brain.update({ id: moving, subtype: 'invoice' }) + await brain.remove(doomed) + await brain.close() + brains.pop() + + // Rewind the aggregation artifact to the at-G bytes (the unclean exit). + { + const reopenForRewind = await open(dir) + const rw = reopenForRewind as unknown as typeof internals + await rw.storage.saveMetadata('__aggregation_state_by_subtype__', stateAtG) + await reopenForRewind.close() + brains.pop() + } + + // Session 2: reopen — adoption must see BEHIND and reconcile, never walk. + brain = await open(dir) + brain.defineAggregate(AGG) + const walkSpy = vi.spyOn( + brain as unknown as { runAggregationBackfillWalk(): Promise }, + 'runAggregationBackfillWalk' + ) + + const results = await brain.queryAggregate(AGG.name) + + // Ground truth after the window: invoice = a,b,e + moved c = 4; draft = 0 + // (c moved out, d deleted). + expect(countFor(results as never, 'invoice'), 'invoice count exact after catch-up').toBe(4) + expect(countFor(results as never, 'draft'), 'draft count exact after catch-up').toBe(0) + + // THE CALL-SHAPE PIN: the whole-store walk never ran. + expect(walkSpy, 'full backfill walk must not run for a behind-stamp reopen').not.toHaveBeenCalled() + + vi.restoreAllMocks() + }, 120000) +}) diff --git a/tests/unit/aggregation/aggregation-provider-rebuild.test.ts b/tests/unit/aggregation/aggregation-provider-rebuild.test.ts new file mode 100644 index 00000000..efd7b4bb --- /dev/null +++ b/tests/unit/aggregation/aggregation-provider-rebuild.test.ts @@ -0,0 +1,134 @@ +/** + * @module tests/unit/aggregation/aggregation-provider-rebuild + * @description Pins for SELF-ENGINE-LIFECYCLE-SPRINT asks (c) + (d): + * (c) the native provider's parallel `rebuildAggregate` — on the provider + * contract since 8.x but NEVER invoked (the JS walk streamed per-entity + * FFI calls instead) — is now the backfill walk's preferred door; + * (d) a write-path hook that cannot see its entity (before-image-less + * delete) flags an exact rescan LOUDLY instead of silently skipping the + * decrement (the skip let counts drift upward forever). + */ +import { describe, it, expect, vi } from 'vitest' +import { AggregationIndex } from '../../../src/aggregation/AggregationIndex.js' +import { NounType } from '../../../src/types/graphTypes.js' +import type { AggregationProvider, AggregateGroupState } from '../../../src/types/brainy.types.js' + +const DEF = { + name: 'by_subtype', + source: { type: NounType.Document }, + groupBy: ['system.subtype'] as string[], + metrics: { count: { op: 'count' as const } } +} + +/** Minimal in-memory storage double for the index's persistence surface. */ +function memStorage() { + const store = new Map() + return { + saveMetadata: async (k: string, v: unknown) => void store.set(k, v), + getMetadata: async (k: string) => store.get(k) ?? null + } as never +} + +function providerDouble(): AggregationProvider & { rebuildAggregate: ReturnType } { + return { + defineAggregate: vi.fn(), + removeAggregate: vi.fn(), + incrementalUpdate: vi.fn(() => []), + computeGroupKey: vi.fn(() => ({})), + rebuildAggregate: vi.fn((): Map => { + return new Map([ + [ + 'system.subtype=invoice', + { + groupKey: { 'system.subtype': 'invoice' }, + metrics: { count: { sum: 0, count: 2, min: Infinity, max: -Infinity, m2: 0 } } + } as AggregateGroupState + ] + ]) + }), + queryAggregate: vi.fn(() => []) + } as never +} + +describe('ask (c) — the native parallel rebuild is invoked, never dead code', () => { + it('rebuildWithProvider hands SOURCE-MATCHED entities to the provider once and swaps state in', () => { + const provider = providerDouble() + const index = new AggregationIndex(memStorage(), provider) + index.defineAggregate(DEF) + + expect(index.hasProviderRebuild()).toBe(true) + + const entities = [ + { type: NounType.Document, subtype: 'invoice', metadata: {} }, + { type: NounType.Document, subtype: 'invoice', metadata: {} }, + // Source-filter mismatch: a different noun type must be filtered OUT + // before the provider sees the batch. + { type: NounType.Person, subtype: 'invoice', metadata: {} } + ] + const handled = index.rebuildWithProvider(DEF.name, entities) + + expect(handled).toBe(true) + expect(provider.rebuildAggregate).toHaveBeenCalledTimes(1) + const [defArg, entArg] = provider.rebuildAggregate.mock.calls[0] + expect(defArg.name).toBe(DEF.name) + expect(entArg).toHaveLength(2) + + // The rebuilt state serves — and the aggregate is no longer pending. + expect(index.getPendingBackfills()).not.toContain(DEF.name) + }) + + it('returns false without a provider rebuild — the caller streams the JS walk', () => { + const index = new AggregationIndex(memStorage()) + index.defineAggregate(DEF) + expect(index.hasProviderRebuild()).toBe(false) + expect(index.rebuildWithProvider(DEF.name, [])).toBe(false) + }) +}) + +describe('ask (d) — the before-image-less delete is LOUD, never a silent skip', () => { + it('flagAllForRescan puts every defined aggregate back on the backfill list', () => { + const index = new AggregationIndex(memStorage()) + index.defineAggregate(DEF) + index.defineAggregate({ ...DEF, name: 'second' }) + // Simulate settled state: nothing pending. + for (const n of index.getPendingBackfills()) { + index.beginBackfill(n) + index.finishBackfill(n) + } + expect(index.getPendingBackfills()).toEqual([]) + + index.flagAllForRescan('delete of X carried no before-image metadata') + + expect(index.getPendingBackfills().sort()).toEqual(['by_subtype', 'second']) + }) +}) + +describe('reconcileEntity — the exact delta algebra at the catch-up boundary', () => { + it('before-only removes, after-only adds, both reconciles a group move', () => { + const index = new AggregationIndex(memStorage()) + index.defineAggregate(DEF) + for (const n of index.getPendingBackfills()) { + index.beginBackfill(n) + index.finishBackfill(n) + } + const doc = (subtype: string) => ({ type: NounType.Document, subtype, metadata: {} }) + + // Pre-window state, applied through the LIVE hooks (as adoption would + // have counted it): c and seed exist as drafts, x1 as an invoice. + index.onEntityAdded('c', doc('draft')) + index.onEntityAdded('seed', doc('draft')) + index.onEntityAdded('x1', doc('invoice')) + + // The window's reconciliation: two adds, one group move, one delete. + index.reconcileEntity(DEF.name, 'a', null, doc('invoice')) + index.reconcileEntity(DEF.name, 'b', null, doc('invoice')) + index.reconcileEntity(DEF.name, 'c', doc('draft'), doc('invoice')) + index.reconcileEntity(DEF.name, 'seed', doc('draft'), null) + + const rows = index.queryAggregate({ name: DEF.name }) + const count = (st: string) => + Number(rows.find(r => r.groupKey['system.subtype'] === st)?.metrics.count ?? 0) + expect(count('invoice')).toBe(4) // x1 + a + b + moved c + expect(count('draft')).toBe(0) // c moved out, seed deleted + }) +}) diff --git a/tests/unit/utils/metadataIndex-nested-orderby.test.ts b/tests/unit/utils/metadataIndex-nested-orderby.test.ts new file mode 100644 index 00000000..55dab59b --- /dev/null +++ b/tests/unit/utils/metadataIndex-nested-orderby.test.ts @@ -0,0 +1,142 @@ +/** + * @module tests/unit/utils/metadataIndex-nested-orderby + * @description THE NESTED-FIELD ADDRESSING PIN for ordered reads (the + * field-addressing law, dotted-path clause). The defect this keeps dead: + * `orderBy` on a nested user metadata field (dotted path, e.g. + * `orderBy: 'profile.score'` over `metadata: { profile: { score: 7 } }`) + * silently returned insertion order — a no-op sort — because the sort + * path's value resolution read flat bag keys only. The law: a dotted user + * address is either SERVED CORRECTLY (the batched resolver walks inside + * the bag) or REFUSED with a typed UnresolvableFieldError — never a silent + * pass-through. Both spellings (`profile.score` / `metadata.profile.score`) + * are the same address; the filter side (`where: { 'profile.score': … }`) + * obeys the same law. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { Brainy, UnresolvableFieldError } from '../../../src/index.js' +import { NounType } from '../../../src/types/graphTypes.js' + +const ROWS = 30 + +describe('nested (dotted-path) user field orderBy — the field-addressing law', () => { + let brain: Brainy + /** id → nested score, for the rows that carry profile.score */ + const scoreById = new Map() + /** ids of the two rows WITHOUT a profile bag */ + let noProfileIds: string[] = [] + + beforeAll(async () => { + brain = new Brainy({ storage: { type: 'memory' }, requireSubtype: false }) + await brain.init() + for (let i = 0; i < ROWS; i++) { + // (i * 11) % 30 is a permutation of 0..29 (gcd(11,30)=1): every score + // distinct, insertion order maximally different from value order — a + // silent insertion-order pass-through cannot accidentally look sorted. + const score = (i * 11) % ROWS + const id = await brain.add({ + data: `row ${i}`, + type: NounType.Document, + metadata: { profile: { score }, plain: i } + }) + scoreById.set(id, score) + } + const a = await brain.add({ + data: 'no-profile a', + type: NounType.Document, + metadata: { plain: 1000 } + }) + const b = await brain.add({ + data: 'no-profile b', + type: NounType.Document, + metadata: { plain: 1001 } + }) + noProfileIds = [a, b].sort() + }, 120000) + + afterAll(async () => { + await brain.close().catch(() => {}) + }) + + /** Assert one complete ordered read against the sealed ordering contract. */ + function assertOrdered( + rows: Array<{ id: string }>, + order: 'asc' | 'desc', + label: string + ): void { + // Rows are NEVER dropped: all 30 scored + 2 profile-less rows come back. + expect(rows.length, `${label}: complete result`).toBe(ROWS + 2) + + // Missing-value rows sort LAST in BOTH directions, ties by id ascending. + const lastTwo = rows.slice(-2).map((r) => r.id) + expect(lastTwo, `${label}: missing-value rows LAST, id asc`).toEqual(noProfileIds) + + // The scored 30 are ordered by the NESTED value — the exact permutation, + // not insertion order. + const observed = rows.slice(0, ROWS).map((r) => scoreById.get(r.id)) + const wanted = [...scoreById.values()].sort((x, y) => + order === 'asc' ? x - y : y - x + ) + expect(observed, `${label}: nested values in ${order} order`).toEqual(wanted) + } + + it('orderBy: "profile.score" desc — served correctly, missing rows LAST (never a silent insertion-order no-op)', async () => { + const rows = await brain.find({ + type: NounType.Document, + orderBy: 'profile.score', + order: 'desc', + limit: 40 + }) + assertOrdered(rows, 'desc', 'bare dotted, desc') + }) + + it('orderBy: "profile.score" asc — same law in the other direction', async () => { + const rows = await brain.find({ + type: NounType.Document, + orderBy: 'profile.score', + order: 'asc', + limit: 40 + }) + assertOrdered(rows, 'asc', 'bare dotted, asc') + }) + + it('explicit spelling "metadata.profile.score" is the SAME address — identical result', async () => { + const bare = await brain.find({ + type: NounType.Document, + orderBy: 'profile.score', + order: 'desc', + limit: 40 + }) + const explicit = await brain.find({ + type: NounType.Document, + orderBy: 'metadata.profile.score', + order: 'desc', + limit: 40 + }) + assertOrdered(explicit, 'desc', 'metadata.-prefixed, desc') + expect( + explicit.map((r) => r.id), + 'both spellings resolve to the identical ordered id sequence' + ).toEqual(bare.map((r) => r.id)) + }) + + it('a dotted path carried by NO entity REFUSES with UnresolvableFieldError — never a silent insertion-order return', async () => { + await expect( + brain.find({ + type: NounType.Document, + orderBy: 'no.such.path', + order: 'desc', + limit: 40 + }) + ).rejects.toThrow(UnresolvableFieldError) + }) + + it('dotted where: { "profile.score": 7 } finds exactly the right row — the filter side of the same law', async () => { + const wantedId = [...scoreById.entries()].find(([, s]) => s === 7)![0] + const rows = await brain.find({ + type: NounType.Document, + where: { 'profile.score': 7 }, + limit: 40 + }) + expect(rows.map((r) => r.id)).toEqual([wantedId]) + }) +})