fix: exception-safe aggregation backfill + generation-verified adoption + loud open-path guards
- Aggregation backfill walks build into a STAGING map and swap in atomically on completion. A mid-walk failure drops the staging map — the previous live state keeps serving, the aggregate stays flagged pending, and the storage error surfaces to the failing query. Previously the walk wiped live state before a scan that could throw, never cleared the pending flag on failure, and re-ran a full walk on every subsequent query: a silent wipe/walk/throw loop at the caller's retry rate. - Failed walks are latched: retries within a 30s cooldown rethrow the recorded error instantly instead of re-walking, so a tight caller-side retry loop costs one loud error per query, never a full store walk per query. - Persisted aggregation state is stamped with the store's committed generation at flush; reopen adoption requires stamp equality. Stale state (unclean shutdown) or over-counting state (a log truncation on a copied store pulled the watermark back) triggers exactly one loud rescan, never a silent adopt. - The backfill/adoption path narrates: adoption decisions, walk start/finish with counts and duration, and failures all log by default. - getNouns/getVerbs refuse a supplied-but-undecodable pagination cursor with a loud error instead of silently restarting the walk at offset 0 (which re-served page 1 forever to any while(hasMore) caller). - The graph cold-load verb walk aborts loudly on a missing or non-advancing cursor with hasMore=true. - A versioned index provider whose generation is AHEAD of the committed watermark (torn copy / crash-recovery truncation) is now named loudly at open, alongside the existing behind-direction message.
This commit is contained in:
parent
01a7f3dd01
commit
a77b064bd7
7 changed files with 335 additions and 36 deletions
|
|
@ -209,6 +209,82 @@ describe('aggregation state persistence — boot-order contract', () => {
|
|||
await brain2.close()
|
||||
})
|
||||
|
||||
it('generation-mismatched persisted state is rescanned once, loudly — never adopted', async () => {
|
||||
const brain1 = await open()
|
||||
brain1.defineAggregate(SPENDING)
|
||||
await seed(brain1)
|
||||
await brain1.queryAggregate('spending')
|
||||
await brain1.close()
|
||||
|
||||
// Simulate the copied-store incident class: a fact-log truncation (or an
|
||||
// unclean shutdown) leaves the committed watermark different from the
|
||||
// generation the flushed state was stamped with.
|
||||
const tamper: any = await open()
|
||||
const key = '__aggregation_state_spending__'
|
||||
const stored = await tamper.storage.getMetadata(key)
|
||||
expect(typeof stored.sourceGeneration).toBe('number') // the stamp is really persisted
|
||||
await tamper.storage.saveMetadata(key, {
|
||||
...stored,
|
||||
sourceGeneration: stored.sourceGeneration + 5
|
||||
})
|
||||
await tamper.close()
|
||||
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
const brain2 = await open()
|
||||
brain2.defineAggregate(SPENDING)
|
||||
await brain2.getNounCount()
|
||||
const walks = countWalks(brain2)
|
||||
|
||||
const rows = await brain2.queryAggregate('spending')
|
||||
|
||||
expect(walks.count()).toBe(1) // exactly ONE rescan — no silent adopt, no spin
|
||||
const food = rows.find((r: any) => r.groupKey.category === 'food')
|
||||
expect(food.metrics.count).toBe(6) // rescan produced exact results
|
||||
expect(
|
||||
warnSpy.mock.calls.some(args => String(args[0]).includes('rescanning instead of adopting'))
|
||||
).toBe(true) // and it said so out loud
|
||||
warnSpy.mockRestore()
|
||||
await brain2.close()
|
||||
})
|
||||
|
||||
it('a failing walk is loud, non-destructive, and latched — never a silent retry loop', async () => {
|
||||
// Fresh define + seeded writes: the write hooks have populated LIVE state,
|
||||
// and the first-query rescan is still pending. The incident shape
|
||||
// (wipe-before-scan + no try/catch + per-query re-walk) would have wiped
|
||||
// that live state and silently re-walked on every query.
|
||||
const brain: any = await open()
|
||||
brain.defineAggregate(SPENDING)
|
||||
await seed(brain)
|
||||
expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2)
|
||||
|
||||
const storage = brain.storage
|
||||
const origGetNouns = storage.getNouns.bind(storage)
|
||||
let walkAttempts = 0
|
||||
storage.getNouns = async () => {
|
||||
walkAttempts++
|
||||
throw new Error('injected storage failure')
|
||||
}
|
||||
|
||||
// First query: the walk fails LOUDLY with the storage error.
|
||||
await expect(brain.queryAggregate('spending')).rejects.toThrow('injected storage failure')
|
||||
expect(walkAttempts).toBe(1)
|
||||
|
||||
// Live state was NOT destroyed by the failed walk (staging was dropped).
|
||||
expect(brain._aggregationIndex.queryAggregate({ name: 'spending' }).length).toBe(2)
|
||||
|
||||
// Second query inside the cooldown: instant loud failure, NO new walk.
|
||||
await expect(brain.queryAggregate('spending')).rejects.toThrow('failure cooldown')
|
||||
expect(walkAttempts).toBe(1)
|
||||
|
||||
// Heal the storage + expire the cooldown: one fresh walk succeeds exactly.
|
||||
storage.getNouns = origGetNouns
|
||||
brain._aggregationBackfillFailure.at = Date.now() - 60_000
|
||||
const rows = await brain.queryAggregate('spending')
|
||||
const food = rows.find((r: any) => r.groupKey.category === 'food')
|
||||
expect(food.metrics.count).toBe(6)
|
||||
await brain.close()
|
||||
})
|
||||
|
||||
it('aggregation persistence keys never log "Unknown key format"', async () => {
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
const brain1 = await open()
|
||||
|
|
|
|||
|
|
@ -95,9 +95,15 @@ describe('verb cursor pagination (graph-perf #2)', () => {
|
|||
expect(new Set(cursorSeen)).toEqual(new Set(offsetSeen))
|
||||
})
|
||||
|
||||
it('a foreign/malformed cursor falls back gracefully (no throw, starts from the beginning)', async () => {
|
||||
const page = await storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
|
||||
expect(page.items.length).toBe(5)
|
||||
expect(page.hasMore).toBe(true)
|
||||
it('a foreign/malformed cursor FAILS LOUDLY — never a silent restart from page 1', async () => {
|
||||
// The old behavior (decode-null → silent offset-0 fallback) re-served page 1
|
||||
// forever to any while(hasMore) walker: an unbounded CPU loop with no log
|
||||
// line. An undecodable resume token now refuses the walk instead.
|
||||
await expect(
|
||||
storage.getVerbs({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
|
||||
).rejects.toThrow('invalid pagination cursor')
|
||||
await expect(
|
||||
storage.getNouns({ pagination: { limit: 5, cursor: 'not-a-cv1-token' } })
|
||||
).rejects.toThrow('invalid pagination cursor')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue