open-brainy/tests/integration/aggregation-state-persistence.test.ts

/**
 * @module tests/integration/aggregation-state-persistence
 * @description The boot-order contract for the aggregation engine. Five laws:
 * (1) STATE ADOPTION — a reopen with an unchanged defineAggregate() adopts the
 *     persisted state and performs NO store walk. (The pre-fix behavior: the
 *     synchronous define always beat the async init, flagged a backfill, and
 *     the first query wiped the just-loaded state and re-walked the whole
 *     store — every restart, forever.)
 * (2) SINGLE-FLIGHT + BATCH — concurrent cold queries across multiple pending
 *     aggregates share exactly ONE store walk; a query never wipes another's
 *     partial progress and M pending aggregates cost one enumeration, not M.
 * (3) CHANGED DEFINITION — a real definition change still backfills, exactly.
 * (4) PERSISTED-ONLY DEFINITIONS — an app that does not re-define at boot can
 *     query a persisted aggregate without racing a spurious "not defined".
 * (5) QUIET KEYS — the engine's persistence keys (__aggregation_*) are
 *     recognized system keys: no "Unknown key format" warning at boot.
 */
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/index.js'
import { NounType } from '../../src/types/graphTypes.js'
import type { AggregateDefinition } from '../../src/types/brainy.types.js'
import { prodLog } from '../../src/utils/logger.js'

const SPENDING: AggregateDefinition = {
  name: 'spending',
  source: { type: NounType.Event, where: { domain: 'financial' } },
  groupBy: ['category'],
  metrics: {
    total: { op: 'sum', field: 'amount' },
    count: { op: 'count' }
  }
}

/** Same name, different metrics — a REAL definition change (hash differs). */
const SPENDING_CHANGED: AggregateDefinition = {
  ...SPENDING,
  metrics: {
    total: { op: 'sum', field: 'amount' },
    count: { op: 'count' },
    average: { op: 'avg', field: 'amount' }
  }
}

describe('aggregation state persistence — boot-order contract', () => {
  let dir: string

  beforeEach(() => {
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-agg-persist-'))
  })

  afterEach(() => {
    fs.rmSync(dir, { recursive: true, force: true })
  })

  const open = async (): Promise<any> => {
    const b: any = new Brainy({
      requireSubtype: false,
      storage: { type: 'filesystem', path: dir },
      silent: true
    })
    await b.init()
    return b
  }

  /** Count store walks by intercepting the storage adapter's getNouns. */
  const countWalks = (brain: any): { count: () => number } => {
    const storage = brain.storage
    const orig = storage.getNouns.bind(storage)
    let calls = 0
    storage.getNouns = async (opts: unknown) => {
      calls++
      return orig(opts)
    }
    return { count: () => calls }
  }

  const seed = async (brain: any): Promise<void> => {
    for (let i = 0; i < 12; i++) {
      await brain.add({
        data: `tx ${i}`,
        type: NounType.Event,
        metadata: {
          domain: 'financial',
          category: i % 2 === 0 ? 'food' : 'transport',
          amount: 10 + i
        }
      })
    }
  }

  it('adopts persisted state on reopen with an unchanged definition — zero walks', async () => {
    const brain1 = await open()
    brain1.defineAggregate(SPENDING)
    await seed(brain1)
    const before = await brain1.queryAggregate('spending')
    expect(before.length).toBe(2)
    await brain1.close()

    const brain2 = await open()
    brain2.defineAggregate(SPENDING) // the standard declarative boot pattern
    await brain2.getNounCount() // settle init paths before counting walks
    const walks = countWalks(brain2)

    const after = await brain2.queryAggregate('spending')

    expect(walks.count()).toBe(0)
    const key = (r: any) => r.groupKey.category
    expect(
      after.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort()
    ).toEqual(
      before.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort()
    )
    await brain2.close()
  })

  it('adopted state keeps accumulating: post-reopen writes land on top of it', async () => {
    const brain1 = await open()
    brain1.defineAggregate(SPENDING)
    await seed(brain1)
    await brain1.queryAggregate('spending')
    await brain1.close()

    const brain2 = await open()
    brain2.defineAggregate(SPENDING)
    // A write BEFORE the first query: if it lands before state adoption the
    // engine must choose an exact rescan over adoption — either way the
    // result must include all 13 entities.
    await brain2.add({
      data: 'late tx',
      type: NounType.Event,
      metadata: { domain: 'financial', category: 'food', amount: 100 }
    })
    const rows = await brain2.queryAggregate('spending')
    const food = rows.find((r: any) => r.groupKey.category === 'food')
    expect(food.metrics.count).toBe(7) // 6 seeded + 1 late
    await brain2.close()
  })

  it('a changed definition still backfills — exactly once, with correct results', async () => {
    const brain1 = await open()
    brain1.defineAggregate(SPENDING)
    await seed(brain1)
    await brain1.queryAggregate('spending')
    await brain1.close()

    const brain2 = await open()
    brain2.defineAggregate(SPENDING_CHANGED)
    await brain2.getNounCount()
    const walks = countWalks(brain2)

    const rows = await brain2.queryAggregate('spending')

    expect(walks.count()).toBe(1) // 12 entities = one page = one getNouns call
    const food = rows.find((r: any) => r.groupKey.category === 'food')
    expect(food.metrics.count).toBe(6)
    expect(food.metrics.average).toBeCloseTo(food.metrics.total / 6)
    await brain2.close()
  })

  it('concurrent cold queries across two aggregates share exactly ONE walk', async () => {
    const brain = await open()
    brain.defineAggregate(SPENDING)
    brain.defineAggregate({
      ...SPENDING,
      name: 'by_category_count',
      metrics: { count: { op: 'count' } }
    })
    await seed(brain)
    await brain.getNounCount()
    const walks = countWalks(brain)

    const results = await Promise.all([
      brain.queryAggregate('spending'),
      brain.queryAggregate('by_category_count'),
      brain.queryAggregate('spending'),
      brain.queryAggregate('by_category_count'),
      brain.queryAggregate('spending'),
      brain.queryAggregate('by_category_count')
    ])

    expect(walks.count()).toBe(1) // 12 entities = one page; one walk fills both
    for (const rows of results) {
      const total = rows.reduce((s: number, r: any) => s + r.metrics.count, 0)
      expect(total).toBe(12)
    }

    // Warm re-query: converged, no further walks.
    await brain.queryAggregate('spending')
    expect(walks.count()).toBe(1)
    await brain.close()
  })

  it('persisted-only definitions are queryable without re-defining at boot', async () => {
    const brain1 = await open()
    brain1.defineAggregate(SPENDING)
    await seed(brain1)
    await brain1.queryAggregate('spending')
    await brain1.close()

    const brain2 = await open()
    // NO defineAggregate — the app relies on the persisted definition.
    const walks = countWalks(brain2)
    const rows = await brain2.queryAggregate('spending')
    expect(walks.count()).toBe(0) // persisted state adopted here too
    expect(rows.length).toBe(2)
    await brain2.close()
  })

  it('generation-mismatched persisted state is rescanned once, loudly — never adopted', async () => {
    const brain1 = await open()
    brain1.defineAggregate(SPENDING)
    await seed(brain1)
    await brain1.queryAggregate('spending')
    await brain1.close()

    // Simulate the copied-store incident class: a fact-log truncation (or an
    // unclean shutdown) leaves the committed watermark different from the
    // generation the flushed state was stamped with.
    const tamper: any = await open()
    const key = '__aggregation_state_spending__'
    const stored = await tamper.storage.getMetadata(key)
    expect(typeof stored.sourceGeneration).toBe('number') // the stamp is really persisted
    await tamper.storage.saveMetadata(key, {
      ...stored,
      sourceGeneration: stored.sourceGeneration + 5
    })
    await tamper.close()

    const warnSpy = vi.spyOn(prodLog, 'warn')
    const brain2 = await open()
    brain2.defineAggregate(SPENDING)
    await brain2.getNounCount()
    const walks = countWalks(brain2)

    const rows = await brain2.queryAggregate('spending')

    expect(walks.count()).toBe(1) // exactly ONE rescan — no silent adopt, no spin
    const food = rows.find((r: any) => r.groupKey.category === 'food')
    expect(food.metrics.count).toBe(6) // rescan produced exact results
    expect(
      warnSpy.mock.calls.some(args => String(args[0]).includes('rescanning instead of adopting'))
    ).toBe(true) // and it said so out loud
    warnSpy.mockRestore()
    await brain2.close()
  })

  it('a failing walk is loud, non-destructive, and latched — never a silent retry loop', async () => {
    // Fresh define + seeded writes: the write hooks have populated LIVE state,
    // and the first-query rescan is still pending. The incident shape
    // (wipe-before-scan + no try/catch + per-query re-walk) would have wiped
    // that live state and silently re-walked on every query.
    const brain: any = await open()
    brain.defineAggregate(SPENDING)
    await seed(brain)
    expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2)

    const storage = brain.storage
    const origGetNouns = storage.getNouns.bind(storage)
    let walkAttempts = 0
    storage.getNouns = async () => {
      walkAttempts++
      throw new Error('injected storage failure')
    }

    // First query: the walk fails LOUDLY with the storage error.
    await expect(brain.queryAggregate('spending')).rejects.toThrow('injected storage failure')
    expect(walkAttempts).toBe(1)

    // Live state was NOT destroyed by the failed walk (staging was dropped).
    expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2)

    // Second query inside the cooldown: instant loud failure, NO new walk.
    await expect(brain.queryAggregate('spending')).rejects.toThrow('failure cooldown')
    expect(walkAttempts).toBe(1)

    // Heal the storage + expire the cooldown: one fresh walk succeeds exactly.
    storage.getNouns = origGetNouns
    brain._aggregationBackfillFailure.at = Date.now() - 60_000
    const rows = await brain.queryAggregate('spending')
    const food = rows.find((r: any) => r.groupKey.category === 'food')
    expect(food.metrics.count).toBe(6)
    await brain.close()
  })

  it('aggregation persistence keys never log "Unknown key format"', async () => {
    const warnSpy = vi.spyOn(prodLog, 'warn')
    const brain1 = await open()
    brain1.defineAggregate(SPENDING)
    await seed(brain1)
    await brain1.queryAggregate('spending')
    await brain1.close()

    const brain2 = await open()
    brain2.defineAggregate(SPENDING)
    await brain2.queryAggregate('spending')
    await brain2.close()

    const offenders = warnSpy.mock.calls
      .map(args => String(args[0]))
      .filter(
        msg =>
          msg.includes('Unknown key format') &&
          (msg.includes('__aggregation_') || msg.includes('brainy:entityIdMapper'))
      )
    expect(offenders).toEqual([])
    warnSpy.mockRestore()
  })
})