/** * @module tests/integration/aggregation-state-persistence * @description The boot-order contract for the aggregation engine. Five laws: * (1) STATE ADOPTION — a reopen with an unchanged defineAggregate() adopts the * persisted state and performs NO store walk. (The pre-fix behavior: the * synchronous define always beat the async init, flagged a backfill, and * the first query wiped the just-loaded state and re-walked the whole * store — every restart, forever.) * (2) SINGLE-FLIGHT + BATCH — concurrent cold queries across multiple pending * aggregates share exactly ONE store walk; a query never wipes another's * partial progress and M pending aggregates cost one enumeration, not M. * (3) CHANGED DEFINITION — a real definition change still backfills, exactly. * (4) PERSISTED-ONLY DEFINITIONS — an app that does not re-define at boot can * query a persisted aggregate without racing a spurious "not defined". * (5) QUIET KEYS — the engine's persistence keys (__aggregation_*) are * recognized system keys: no "Unknown key format" warning at boot. */ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import * as path from 'node:path' import { Brainy } from '../../src/index.js' import { NounType } from '../../src/types/graphTypes.js' import type { AggregateDefinition } from '../../src/types/brainy.types.js' import { prodLog } from '../../src/utils/logger.js' const SPENDING: AggregateDefinition = { name: 'spending', source: { type: NounType.Event, where: { domain: 'financial' } }, groupBy: ['category'], metrics: { total: { op: 'sum', field: 'amount' }, count: { op: 'count' } } } /** Same name, different metrics — a REAL definition change (hash differs). */ const SPENDING_CHANGED: AggregateDefinition = { ...SPENDING, metrics: { total: { op: 'sum', field: 'amount' }, count: { op: 'count' }, average: { op: 'avg', field: 'amount' } } } describe('aggregation state persistence — boot-order contract', () => { let dir: string beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-agg-persist-')) }) afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }) }) const open = async (): Promise => { const b: any = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true }) await b.init() return b } /** Count store walks by intercepting the storage adapter's getNouns. */ const countWalks = (brain: any): { count: () => number } => { const storage = brain.storage const orig = storage.getNouns.bind(storage) let calls = 0 storage.getNouns = async (opts: unknown) => { calls++ return orig(opts) } return { count: () => calls } } const seed = async (brain: any): Promise => { for (let i = 0; i < 12; i++) { await brain.add({ data: `tx ${i}`, type: NounType.Event, metadata: { domain: 'financial', category: i % 2 === 0 ? 'food' : 'transport', amount: 10 + i } }) } } it('adopts persisted state on reopen with an unchanged definition — zero walks', async () => { const brain1 = await open() brain1.defineAggregate(SPENDING) await seed(brain1) const before = await brain1.queryAggregate('spending') expect(before.length).toBe(2) await brain1.close() const brain2 = await open() brain2.defineAggregate(SPENDING) // the standard declarative boot pattern await brain2.getNounCount() // settle init paths before counting walks const walks = countWalks(brain2) const after = await brain2.queryAggregate('spending') expect(walks.count()).toBe(0) const key = (r: any) => r.groupKey.category expect( after.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort() ).toEqual( before.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort() ) await brain2.close() }) it('adopted state keeps accumulating: post-reopen writes land on top of it', async () => { const brain1 = await open() brain1.defineAggregate(SPENDING) await seed(brain1) await brain1.queryAggregate('spending') await brain1.close() const brain2 = await open() brain2.defineAggregate(SPENDING) // A write BEFORE the first query: if it lands before state adoption the // engine must choose an exact rescan over adoption — either way the // result must include all 13 entities. await brain2.add({ data: 'late tx', type: NounType.Event, metadata: { domain: 'financial', category: 'food', amount: 100 } }) const rows = await brain2.queryAggregate('spending') const food = rows.find((r: any) => r.groupKey.category === 'food') expect(food.metrics.count).toBe(7) // 6 seeded + 1 late await brain2.close() }) it('a changed definition still backfills — exactly once, with correct results', async () => { const brain1 = await open() brain1.defineAggregate(SPENDING) await seed(brain1) await brain1.queryAggregate('spending') await brain1.close() const brain2 = await open() brain2.defineAggregate(SPENDING_CHANGED) await brain2.getNounCount() const walks = countWalks(brain2) const rows = await brain2.queryAggregate('spending') expect(walks.count()).toBe(1) // 12 entities = one page = one getNouns call const food = rows.find((r: any) => r.groupKey.category === 'food') expect(food.metrics.count).toBe(6) expect(food.metrics.average).toBeCloseTo(food.metrics.total / 6) await brain2.close() }) it('concurrent cold queries across two aggregates share exactly ONE walk', async () => { const brain = await open() brain.defineAggregate(SPENDING) brain.defineAggregate({ ...SPENDING, name: 'by_category_count', metrics: { count: { op: 'count' } } }) await seed(brain) await brain.getNounCount() const walks = countWalks(brain) const results = await Promise.all([ brain.queryAggregate('spending'), brain.queryAggregate('by_category_count'), brain.queryAggregate('spending'), brain.queryAggregate('by_category_count'), brain.queryAggregate('spending'), brain.queryAggregate('by_category_count') ]) expect(walks.count()).toBe(1) // 12 entities = one page; one walk fills both for (const rows of results) { const total = rows.reduce((s: number, r: any) => s + r.metrics.count, 0) expect(total).toBe(12) } // Warm re-query: converged, no further walks. await brain.queryAggregate('spending') expect(walks.count()).toBe(1) await brain.close() }) it('persisted-only definitions are queryable without re-defining at boot', async () => { const brain1 = await open() brain1.defineAggregate(SPENDING) await seed(brain1) await brain1.queryAggregate('spending') await brain1.close() const brain2 = await open() // NO defineAggregate — the app relies on the persisted definition. const walks = countWalks(brain2) const rows = await brain2.queryAggregate('spending') expect(walks.count()).toBe(0) // persisted state adopted here too expect(rows.length).toBe(2) await brain2.close() }) it('generation-mismatched persisted state is rescanned once, loudly — never adopted', async () => { const brain1 = await open() brain1.defineAggregate(SPENDING) await seed(brain1) await brain1.queryAggregate('spending') await brain1.close() // Simulate the copied-store incident class: a fact-log truncation (or an // unclean shutdown) leaves the committed watermark different from the // generation the flushed state was stamped with. const tamper: any = await open() const key = '__aggregation_state_spending__' const stored = await tamper.storage.getMetadata(key) expect(typeof stored.sourceGeneration).toBe('number') // the stamp is really persisted await tamper.storage.saveMetadata(key, { ...stored, sourceGeneration: stored.sourceGeneration + 5 }) await tamper.close() const warnSpy = vi.spyOn(prodLog, 'warn') const brain2 = await open() brain2.defineAggregate(SPENDING) await brain2.getNounCount() const walks = countWalks(brain2) const rows = await brain2.queryAggregate('spending') expect(walks.count()).toBe(1) // exactly ONE rescan — no silent adopt, no spin const food = rows.find((r: any) => r.groupKey.category === 'food') expect(food.metrics.count).toBe(6) // rescan produced exact results expect( warnSpy.mock.calls.some(args => String(args[0]).includes('rescanning instead of adopting')) ).toBe(true) // and it said so out loud warnSpy.mockRestore() await brain2.close() }) it('a failing walk is loud, non-destructive, and latched — never a silent retry loop', async () => { // Fresh define + seeded writes: the write hooks have populated LIVE state, // and the first-query rescan is still pending. The incident shape // (wipe-before-scan + no try/catch + per-query re-walk) would have wiped // that live state and silently re-walked on every query. const brain: any = await open() brain.defineAggregate(SPENDING) await seed(brain) expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2) const storage = brain.storage const origGetNouns = storage.getNouns.bind(storage) let walkAttempts = 0 storage.getNouns = async () => { walkAttempts++ throw new Error('injected storage failure') } // First query: the walk fails LOUDLY with the storage error. await expect(brain.queryAggregate('spending')).rejects.toThrow('injected storage failure') expect(walkAttempts).toBe(1) // Live state was NOT destroyed by the failed walk (staging was dropped). expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2) // Second query inside the cooldown: instant loud failure, NO new walk. await expect(brain.queryAggregate('spending')).rejects.toThrow('failure cooldown') expect(walkAttempts).toBe(1) // Heal the storage + expire the cooldown: one fresh walk succeeds exactly. storage.getNouns = origGetNouns brain._aggregationBackfillFailure.at = Date.now() - 60_000 const rows = await brain.queryAggregate('spending') const food = rows.find((r: any) => r.groupKey.category === 'food') expect(food.metrics.count).toBe(6) await brain.close() }) it('aggregation persistence keys never log "Unknown key format"', async () => { const warnSpy = vi.spyOn(prodLog, 'warn') const brain1 = await open() brain1.defineAggregate(SPENDING) await seed(brain1) await brain1.queryAggregate('spending') await brain1.close() const brain2 = await open() brain2.defineAggregate(SPENDING) await brain2.queryAggregate('spending') await brain2.close() const offenders = warnSpy.mock.calls .map(args => String(args[0])) .filter( msg => msg.includes('Unknown key format') && (msg.includes('__aggregation_') || msg.includes('brainy:entityIdMapper')) ) expect(offenders).toEqual([]) warnSpy.mockRestore() }) })