fix: aggregation state adoption on reopen + single-flight backfill + query-cap ratchet removal

- AggregationIndex: defineAggregate before init no longer forces a backfill.
  init reconciles instead of clobbering: the app definition wins, persisted
  state is adopted on hash match, a write landing pre-adoption forces an exact
  rescan, and a successful state load clears the backfill flag. New ready()
  settles every adoption decision before query paths consult backfill state.
- brainy: backfills are single-flight and batched. Concurrent queries share one
  store walk and every pending aggregate fills from that same walk; the old
  behavior let each concurrent query wipe the others' partial state and start
  its own full walk, so a store under steady aggregate traffic never converged.
  queryAggregate also waits for persisted definitions before deciding an
  aggregate does not exist.
- paramValidation: recordQuery is telemetry-only. The duration-based cap
  ratchet (x0.8 per recorded query while lifetime-average exceeded 1s, floored
  at 1000 - below the documented 10000 auto floor, reported under a stale
  basis label) is removed; the cap comes from its construction-time basis or
  explicit overrides alone.
- storage: __aggregation_* and singleton system keys (brainy:entityIdMapper)
  are recognized before the unknown-key warning fires; routing is unchanged.
- docs: find-limits cap-immutability note, aggregation reopen/adoption
  semantics, RELEASES.md 8.5.1 entry.

tests/integration/aggregation-state-persistence.test.ts

@@ -0,0 +1,235 @@
+/**
+ * @module tests/integration/aggregation-state-persistence
+ * @description The boot-order contract for the aggregation engine. Five laws:
+ * (1) STATE ADOPTION — a reopen with an unchanged defineAggregate() adopts the
+ *     persisted state and performs NO store walk. (The pre-fix behavior: the
+ *     synchronous define always beat the async init, flagged a backfill, and
+ *     the first query wiped the just-loaded state and re-walked the whole
+ *     store — every restart, forever.)
+ * (2) SINGLE-FLIGHT + BATCH — concurrent cold queries across multiple pending
+ *     aggregates share exactly ONE store walk; a query never wipes another's
+ *     partial progress and M pending aggregates cost one enumeration, not M.
+ * (3) CHANGED DEFINITION — a real definition change still backfills, exactly.
+ * (4) PERSISTED-ONLY DEFINITIONS — an app that does not re-define at boot can
+ *     query a persisted aggregate without racing a spurious "not defined".
+ * (5) QUIET KEYS — the engine's persistence keys (__aggregation_*) are
+ *     recognized system keys: no "Unknown key format" warning at boot.
+ */
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+import { NounType } from '../../src/types/graphTypes.js'
+import type { AggregateDefinition } from '../../src/types/brainy.types.js'
+import { prodLog } from '../../src/utils/logger.js'
+
+const SPENDING: AggregateDefinition = {
+  name: 'spending',
+  source: { type: NounType.Event, where: { domain: 'financial' } },
+  groupBy: ['category'],
+  metrics: {
+    total: { op: 'sum', field: 'amount' },
+    count: { op: 'count' }
+  }
+}
+
+/** Same name, different metrics — a REAL definition change (hash differs). */
+const SPENDING_CHANGED: AggregateDefinition = {
+  ...SPENDING,
+  metrics: {
+    total: { op: 'sum', field: 'amount' },
+    count: { op: 'count' },
+    average: { op: 'avg', field: 'amount' }
+  }
+}
+
+describe('aggregation state persistence — boot-order contract', () => {
+  let dir: string
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-agg-persist-'))
+  })
+
+  afterEach(() => {
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  const open = async (): Promise<any> => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true
+    })
+    await b.init()
+    return b
+  }
+
+  /** Count store walks by intercepting the storage adapter's getNouns. */
+  const countWalks = (brain: any): { count: () => number } => {
+    const storage = brain.storage
+    const orig = storage.getNouns.bind(storage)
+    let calls = 0
+    storage.getNouns = async (opts: unknown) => {
+      calls++
+      return orig(opts)
+    }
+    return { count: () => calls }
+  }
+
+  const seed = async (brain: any): Promise<void> => {
+    for (let i = 0; i < 12; i++) {
+      await brain.add({
+        data: `tx ${i}`,
+        type: NounType.Event,
+        metadata: {
+          domain: 'financial',
+          category: i % 2 === 0 ? 'food' : 'transport',
+          amount: 10 + i
+        }
+      })
+    }
+  }
+
+  it('adopts persisted state on reopen with an unchanged definition — zero walks', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    const before = await brain1.queryAggregate('spending')
+    expect(before.length).toBe(2)
+    await brain1.close()
+
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING) // the standard declarative boot pattern
+    await brain2.getNounCount() // settle init paths before counting walks
+    const walks = countWalks(brain2)
+
+    const after = await brain2.queryAggregate('spending')
+
+    expect(walks.count()).toBe(0)
+    const key = (r: any) => r.groupKey.category
+    expect(
+      after.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort()
+    ).toEqual(
+      before.map((r: any) => [key(r), r.metrics.total, r.metrics.count]).sort()
+    )
+    await brain2.close()
+  })
+
+  it('adopted state keeps accumulating: post-reopen writes land on top of it', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING)
+    // A write BEFORE the first query: if it lands before state adoption the
+    // engine must choose an exact rescan over adoption — either way the
+    // result must include all 13 entities.
+    await brain2.add({
+      data: 'late tx',
+      type: NounType.Event,
+      metadata: { domain: 'financial', category: 'food', amount: 100 }
+    })
+    const rows = await brain2.queryAggregate('spending')
+    const food = rows.find((r: any) => r.groupKey.category === 'food')
+    expect(food.metrics.count).toBe(7) // 6 seeded + 1 late
+    await brain2.close()
+  })
+
+  it('a changed definition still backfills — exactly once, with correct results', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING_CHANGED)
+    await brain2.getNounCount()
+    const walks = countWalks(brain2)
+
+    const rows = await brain2.queryAggregate('spending')
+
+    expect(walks.count()).toBe(1) // 12 entities = one page = one getNouns call
+    const food = rows.find((r: any) => r.groupKey.category === 'food')
+    expect(food.metrics.count).toBe(6)
+    expect(food.metrics.average).toBeCloseTo(food.metrics.total / 6)
+    await brain2.close()
+  })
+
+  it('concurrent cold queries across two aggregates share exactly ONE walk', async () => {
+    const brain = await open()
+    brain.defineAggregate(SPENDING)
+    brain.defineAggregate({
+      ...SPENDING,
+      name: 'by_category_count',
+      metrics: { count: { op: 'count' } }
+    })
+    await seed(brain)
+    await brain.getNounCount()
+    const walks = countWalks(brain)
+
+    const results = await Promise.all([
+      brain.queryAggregate('spending'),
+      brain.queryAggregate('by_category_count'),
+      brain.queryAggregate('spending'),
+      brain.queryAggregate('by_category_count'),
+      brain.queryAggregate('spending'),
+      brain.queryAggregate('by_category_count')
+    ])
+
+    expect(walks.count()).toBe(1) // 12 entities = one page; one walk fills both
+    for (const rows of results) {
+      const total = rows.reduce((s: number, r: any) => s + r.metrics.count, 0)
+      expect(total).toBe(12)
+    }
+
+    // Warm re-query: converged, no further walks.
+    await brain.queryAggregate('spending')
+    expect(walks.count()).toBe(1)
+    await brain.close()
+  })
+
+  it('persisted-only definitions are queryable without re-defining at boot', async () => {
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    const brain2 = await open()
+    // NO defineAggregate — the app relies on the persisted definition.
+    const walks = countWalks(brain2)
+    const rows = await brain2.queryAggregate('spending')
+    expect(walks.count()).toBe(0) // persisted state adopted here too
+    expect(rows.length).toBe(2)
+    await brain2.close()
+  })
+
+  it('aggregation persistence keys never log "Unknown key format"', async () => {
+    const warnSpy = vi.spyOn(prodLog, 'warn')
+    const brain1 = await open()
+    brain1.defineAggregate(SPENDING)
+    await seed(brain1)
+    await brain1.queryAggregate('spending')
+    await brain1.close()
+
+    const brain2 = await open()
+    brain2.defineAggregate(SPENDING)
+    await brain2.queryAggregate('spending')
+    await brain2.close()
+
+    const offenders = warnSpy.mock.calls
+      .map(args => String(args[0]))
+      .filter(
+        msg =>
+          msg.includes('Unknown key format') &&
+          (msg.includes('__aggregation_') || msg.includes('brainy:entityIdMapper'))
+      )
+    expect(offenders).toEqual([])
+    warnSpy.mockRestore()
+  })
+})

tests/unit/aggregation/AggregationIndex.test.ts

@@ -675,4 +675,104 @@ describe('AggregationIndex', () => {
       await reloaded.close()
     })
   })
+
+  // ============= Boot-order reconciliation =============
+  //
+  // The production boot pattern: defineAggregate() is synchronous and always
+  // beats the async init() that loads persisted state. The old code flagged a
+  // backfill at define time and init never cleared it — so the loaded state
+  // was wiped and the whole store re-walked on EVERY restart.
+
+  describe('boot-order reconciliation (define-before-init)', () => {
+    const DEF: AggregateDefinition = {
+      name: 'boot_agg',
+      source: { type: NounType.Event },
+      groupBy: ['category'],
+      metrics: { count: { op: 'count' } }
+    }
+
+    const entity = (id: string, category: string): Record<string, unknown> => ({
+      id,
+      noun: NounType.Event,
+      metadata: { category },
+      createdAt: Date.now(),
+      updatedAt: Date.now()
+    })
+
+    /** Simulate the previous session: define, contribute, flush. */
+    async function seedAndFlush(store: MemoryStorage): Promise<void> {
+      const first = new AggregationIndex(store)
+      await first.init()
+      first.defineAggregate(DEF)
+      first.onEntityAdded('e1', entity('e1', 'food'))
+      first.onEntityAdded('e2', entity('e2', 'food'))
+      first.onEntityAdded('e3', entity('e3', 'transport'))
+      await first.flush()
+    }
+
+    it('adopts persisted state when define beats init with an unchanged definition', async () => {
+      const store = new MemoryStorage()
+      await store.init()
+      await seedAndFlush(store)
+
+      const second = new AggregationIndex(store)
+      second.defineAggregate(DEF) // synchronous define FIRST — the real boot order
+      await second.init()
+      await second.ready()
+
+      expect(second.getPendingBackfills()).toEqual([])
+      const rows = second.queryAggregate({ name: 'boot_agg' })
+      const food = rows.find(r => r.groupKey.category === 'food')!
+      expect(food.metrics.count).toBe(2)
+    })
+
+    it('a write landing before adoption forces an exact rescan instead', async () => {
+      const store = new MemoryStorage()
+      await store.init()
+      await seedAndFlush(store)
+
+      const second = new AggregationIndex(store)
+      second.defineAggregate(DEF)
+      second.onEntityAdded('e4', entity('e4', 'food')) // lands before init settles
+      await second.init()
+      await second.ready()
+
+      // Adoption would lose e4's contribution — the engine must rescan.
+      expect(second.getPendingBackfills()).toEqual(['boot_agg'])
+    })
+
+    it('init never clobbers a changed app definition registered before it', async () => {
+      const store = new MemoryStorage()
+      await store.init()
+      await seedAndFlush(store)
+
+      const CHANGED: AggregateDefinition = {
+        ...DEF,
+        metrics: { count: { op: 'count' }, total: { op: 'sum', field: 'amount' } }
+      }
+      const second = new AggregationIndex(store)
+      second.defineAggregate(CHANGED)
+      await second.init()
+      await second.ready()
+
+      const def = second.getDefinitions().find(d => d.name === 'boot_agg')!
+      expect(Object.keys(def.metrics).sort()).toEqual(['count', 'total'])
+      expect(second.getPendingBackfills()).toEqual(['boot_agg'])
+    })
+
+    it('init alone restores persisted definitions with adopted state, no backfill', async () => {
+      const store = new MemoryStorage()
+      await store.init()
+      await seedAndFlush(store)
+
+      const second = new AggregationIndex(store)
+      await second.init()
+      await second.ready()
+
+      expect(second.hasAggregate('boot_agg')).toBe(true)
+      expect(second.getPendingBackfills()).toEqual([])
+      const rows = second.queryAggregate({ name: 'boot_agg' })
+      expect(rows.reduce((s, r) => s + (r.metrics.count as number), 0)).toBe(3)
+    })
+  })
 })

tests/unit/utils/paramValidation.test.ts

@@ -270,27 +270,24 @@ describe('Zero-Config Parameter Validation', () => {
       expect(config.availableMemory).toBeGreaterThan(0)
     })
     
-    it('should adapt limits based on query performance', () => {
-      const initialConfig = getValidationConfig()
-      const initialLimit = initialConfig.maxLimit
-      
-      // Simulate fast queries with large results
+    it('never mutates the cap from query timing (telemetry only)', () => {
+      const initialLimit = getValidationConfig().maxLimit
+
+      // Fast queries with large results: no silent growth.
       for (let i = 0; i < 10; i++) {
         recordQueryPerformance(50, initialLimit * 0.9)
       }
-      
-      const updatedConfig = getValidationConfig()
-      // Limit might increase if performance is good
-      expect(updatedConfig.maxLimit).toBeGreaterThanOrEqual(initialLimit)
-      
-      // Simulate slow queries
-      for (let i = 0; i < 10; i++) {
-        recordQueryPerformance(2000, 100)
+      expect(getValidationConfig().maxLimit).toBe(initialLimit)
+
+      // A burst of catastrophically slow queries must not strangle the cap.
+      // The removed "learning" ratchet shrank it 20% per recorded query down
+      // to a floor of 1000 — below the documented MIN_AUTO_QUERY_LIMIT — and
+      // the error message blamed "available free memory" (a production
+      // incident: every find({ limit: 5000 }) failed on an idle 23GB-free box).
+      for (let i = 0; i < 50; i++) {
+        recordQueryPerformance(90_000, 100)
       }
-      
-      const finalConfig = getValidationConfig()
-      // Limit should decrease if performance is poor
-      expect(finalConfig.maxLimit).toBeLessThanOrEqual(updatedConfig.maxLimit)
+      expect(getValidationConfig().maxLimit).toBe(initialLimit)
     })
   })
 })