From 6207e48b518bd80bdbb0113099a69a7a619e0957 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Sat, 18 Jul 2026 10:51:37 -0700 Subject: [PATCH] fix: O(1) adaptive retention accounting + historyStats fleet audit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under default adaptive retention, every flush() recomputed total history bytes by walking EVERY committed generation's delta — O(all generations) with disk re-reads past the 4096-entry delta-cache bound. On a production brain with 70,000+ accumulated generations this turned every write into a full-tail scan (60-100s writes, escalating with history growth), even though the free-RAM budget never tripped and nothing was ever reclaimed (SELF-GENERATIONS-GROWTH). historyBytes() now maintains a running total: seeded by one walk on first use, then updated incrementally at both commit paths (+bytes) and the compaction reclaim loop (−bytes), dropped on reopenAfterRestore. The adaptive retention check on every flush is O(1). Invariant regression- pinned: running total ≡ fresh walk through transact commits, single-op group commits, and compaction. New brain.historyStats() (exported HistoryStats): read-only generation count / bytes / generation+timestamp range / horizon / retention mode / effective budget — the one-call per-brain fleet audit for retention exposure. --- RELEASES.md | 26 ++++++++- src/brainy.ts | 28 ++++++++++ src/db/generationStore.ts | 72 +++++++++++++++++++++++-- src/db/types.ts | 30 +++++++++++ src/index.ts | 1 + tests/unit/db/generationStore.test.ts | 76 +++++++++++++++++++++++++++ 6 files changed, 228 insertions(+), 5 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 4a62123a..fd9d64ec 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,7 +10,31 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so --- -## v8.8.1 — 2026-07-18 (the import dedup off-switch is now honest + lifecycle-safe) +## v8.8.1 — 2026-07-18 (flush no longer walks the whole generation history + the import dedup off-switch is now honest) + +### The flush-storm fix (production incident, reported by a long-running deployment) + +Under the default adaptive retention, **every `flush()` re-walked the entire committed +generation history** to compute total history bytes for the budget check — O(all +generations) with disk re-reads past the 4,096-entry delta-cache bound. On a brain with +70,000+ accumulated generations that turned every write into a full-tail scan (60-100s +writes), even though the budget (free-RAM-based) never tripped and nothing was ever +reclaimed. Fixed: + +- `historyBytes()` now maintains a **running total**: seeded by one walk on first use, + then updated incrementally at every commit and reclaim — the adaptive retention check + on every flush is O(1). Invariant regression-pinned (running total ≡ fresh walk through + both commit paths and compaction). +- New **`brain.historyStats()`** (read-only, exported `HistoryStats`): generation count, + total on-disk bytes, generation/timestamp range, compaction horizon, retention mode, + and the effective adaptive budget — the one-call fleet-audit for sizing retention + exposure per brain. +- Interim guidance for keep-everything deployments already affected: `retention: 'all'` + skips the adaptive accounting entirely (and is the correct policy if you never want + history reclaimed). The accumulated files are harmless at rest; this release removes + the per-write cost of their existence. + +### The import dedup off-switch (lifecycle honesty) The post-import background deduplication pass (a merge-DELETE writer that runs ~5 minutes after an import, merging entities judged duplicates by id / name / vector similarity) had diff --git a/src/brainy.ts b/src/brainy.ts index c7b4c89a..008733b9 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -192,6 +192,7 @@ import { MemoryStorage } from './storage/adapters/memoryStorage.js' import type { CompactHistoryOptions, CompactHistoryResult, + HistoryStats, TransactOptions, TransactReceipt, TxLogEntry, @@ -8211,6 +8212,33 @@ export class Brainy implements BrainyInterface { return this.generationStore.compact(options) } + /** + * @description Read-only generational-history footprint for fleet audits: + * generation count, total on-disk bytes, generation/timestamp range, the + * compaction horizon, and the retention policy in force. Touches no data and + * changes nothing. First call pays one walk over committed deltas to seed + * the running byte total (subsequent calls — and every adaptive retention + * check — are then O(1)). + * + * A pool operator's exposure check is one call per brain: + * @example + * const stats = await brain.historyStats() + * console.log(`${stats.generations} generations, ${stats.bytes} bytes, mode=${stats.retentionMode}`) + */ + async historyStats(): Promise { + await this.ensureInitialized() + const stats = await this.generationStore.historyStats() + const policy = this.resolveRetentionPolicy() + return { + ...stats, + retentionMode: policy.mode, + effectiveBudgetBytes: + policy.mode === 'adaptive' + ? this.adaptiveHistoryBudgetBytes(policy.budgetBytes) + : null + } + } + /** * @description Drive the adaptive retention byte budget at runtime — the * settable input a machine-level coordinator (e.g. cor's `ResourceManager`, diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 0d813f4d..dd70f7aa 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -257,6 +257,15 @@ export class GenerationStore { */ private deltaCacheMax = 4096 + /** + * Running total of on-disk history bytes across committed generations — + * `null` until {@link historyBytes} pays its one seeding walk. Maintained + * incrementally at commit/reclaim so the adaptive retention check on every + * flush() is O(1), never a tail re-walk. Never updated by cache re-reads + * ({@link setDelta} inserts are cache population, not new history). + */ + private historyBytesTotal: number | null = null + /** * Model-B per-write group-commit — the in-memory PENDING tier. * @@ -483,6 +492,40 @@ export class GenerationStore { return this.horizonGen } + /** + * @description Read-only history footprint for fleet audits: how much + * generational history this store holds on disk. `bytes` pays (and seeds) + * the one-time {@link historyBytes} walk on first call — subsequent calls + * are O(1). The oldest/newest timestamps come from those generations' + * deltas (cache-bounded reads). + * @returns Counts, bytes, generation range, and the compaction horizon. + */ + async historyStats(): Promise<{ + generations: number + bytes: number + oldestGeneration: number | null + newestGeneration: number | null + oldestTimestamp: number | null + newestTimestamp: number | null + horizon: number + }> { + let oldest: number | null = null + let newest: number | null = null + for (const gen of this.committedGensAsc()) { + if (oldest === null) oldest = gen + newest = gen + } + return { + generations: this.committedCount(), + bytes: await this.historyBytes(), + oldestGeneration: oldest, + newestGeneration: newest, + oldestTimestamp: oldest !== null ? (await this.getDelta(oldest)).timestamp : null, + newestTimestamp: newest !== null ? (await this.getDelta(newest)).timestamp : null, + horizon: this.horizonGen + } + } + /** * @description Read one generation's persisted before-image records — the * compaction fallback for generations written before deltas carried @@ -849,6 +892,9 @@ export class GenerationStore { timestamp, bytes: delta.bytes ?? 0 }) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal += delta.bytes ?? 0 + } this.extendChains(gen, nouns, verbs) const logEntry: TxLogEntry = { generation: gen, timestamp, ...(args.meta && { meta: args.meta }) } await this.storage.appendTxLogLine(JSON.stringify(logEntry)) @@ -1302,6 +1348,9 @@ export class GenerationStore { timestamp: buf.timestamp, bytes: genBytes.get(gen) ?? 0 }) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal += genBytes.get(gen) ?? 0 + } this.pendingBuffer.delete(gen) } this.pendingGens = [] @@ -2122,17 +2171,26 @@ export class GenerationStore { /** * @description Total serialized bytes of the ON-DISK generational history — * the sum of every committed generation's recorded `bytes`. Backs the - * `maxBytes` and adaptive retention caps. Reads each committed generation's - * delta (cached; a re-read only for cache-evicted ones) — O(committed - * generations), bounded by retention itself and invoked only at compaction - * time. Pending (un-flushed) generations are excluded (they are not on disk). + * `maxBytes` and adaptive retention caps. O(1) after the first call: the + * total is computed by ONE walk over committed deltas, then maintained + * incrementally at every commit (+bytes) and reclaim (−bytes) and dropped on + * a wholesale state replacement (restore). Without the running total, the + * adaptive auto-compaction on every flush() re-walked the ENTIRE history — + * O(committed generations) file reads per flush past the delta-cache bound — + * which is how a 70k-generation production brain turned every write into a + * full-tail scan (SELF-GENERATIONS-GROWTH). Pending (un-flushed) generations + * are excluded (they are not on disk). * @returns The total on-disk history byte count. */ async historyBytes(): Promise { + if (this.historyBytesTotal !== null) { + return this.historyBytesTotal + } let total = 0 for (const gen of this.committedGensAsc()) { total += (await this.getDelta(gen)).bytes } + this.historyBytesTotal = total return total } @@ -2206,6 +2264,9 @@ export class GenerationStore { await this.storage.removeRawPrefix(`${GENERATIONS_PREFIX}/${gen}`) this.deltaCache.delete(gen) + if (this.historyBytesTotal !== null) { + this.historyBytesTotal -= delta.bytes + } // AFTER the record-set is gone (over-count-only crash ordering): // release its history references and reclaim any blob left with zero @@ -2266,6 +2327,9 @@ export class GenerationStore { async reopenAfterRestore(floorGeneration: number): Promise { await this.withMutex(async () => { this.deltaCache.clear() + // The running history-byte total describes the REPLACED store — drop it; + // the next historyBytes() re-seeds with one walk over the new state. + this.historyBytesTotal = null // A wholesale state replacement invalidates any buffered single-op // history — discard the pending tier (its live writes are gone with the // replaced store). diff --git a/src/db/types.ts b/src/db/types.ts index d1355dab..521396b8 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -193,6 +193,36 @@ export interface CompactHistoryResult { horizon: number } +/** + * @description Result of `brain.historyStats()` — the read-only generational + * history footprint, for fleet audits and ops doors. A pool operator runs this + * per brain to size retention exposure (how much MVCC history each brain + * carries and under which policy) without touching any data. + */ +export interface HistoryStats { + /** Committed generation record-sets currently on disk. */ + generations: number + /** Total on-disk history bytes across those record-sets. */ + bytes: number + /** Oldest committed generation still on disk (null when history is empty). */ + oldestGeneration: number | null + /** Newest committed generation (null when history is empty). */ + newestGeneration: number | null + /** Commit timestamp (ms) of the oldest on-disk generation. */ + oldestTimestamp: number | null + /** Commit timestamp (ms) of the newest on-disk generation. */ + newestTimestamp: number | null + /** Compaction horizon — generations below it were reclaimed. */ + horizon: number + /** The effective retention mode this brain runs under. */ + retentionMode: 'all' | 'adaptive' | 'explicit' + /** + * The adaptive byte budget in force (coordinator-driven or the local + * free-memory probe); null under 'all' or explicit caps. + */ + effectiveBudgetBytes: number | null +} + // ============================================================================ // Db surfaces // ============================================================================ diff --git a/src/index.ts b/src/index.ts index 5c57fe29..ee01d885 100644 --- a/src/index.ts +++ b/src/index.ts @@ -201,6 +201,7 @@ export type { TxLogEntry, CompactHistoryOptions, CompactHistoryResult, + HistoryStats, ChangedIds, DiffResult, HistoryVersion, diff --git a/tests/unit/db/generationStore.test.ts b/tests/unit/db/generationStore.test.ts index 184d3974..611a97d7 100644 --- a/tests/unit/db/generationStore.test.ts +++ b/tests/unit/db/generationStore.test.ts @@ -489,4 +489,80 @@ describe('db/GenerationStore', () => { store.release(2) }) }) + + // ========================================================================== + describe('history-bytes running total (the O(1) retention check)', () => { + /** A fresh walk with the cache dropped — ground truth for the invariant. */ + async function groundTruthBytes(): Promise { + ;(store as any).historyBytesTotal = null + return store.historyBytes() + } + + it('is seeded once, then maintained through commits WITHOUT re-walks', async () => { + await commitWrite(ID_A, 1) + await commitWrite(ID_A, 2) + const seeded = await store.historyBytes() + expect(seeded).toBe(await groundTruthBytes()) + + // From here every read must come from the running total, not a walk: + // getDelta re-reads are the walk's cost — commits must not trigger any. + const getDeltaSpy = vi.spyOn(store as any, 'getDelta') + await commitWrite(ID_B, 1) + const afterCommit = await store.historyBytes() + expect(getDeltaSpy).not.toHaveBeenCalled() + getDeltaSpy.mockRestore() + expect(afterCommit).toBe(await groundTruthBytes()) + }) + + it('stays exact through single-op group commits and compaction', async () => { + await commitWrite(ID_A, 1) + await store.historyBytes() // seed + // Single-op path: buffered generations flushed as one group commit. + await store.commitSingleOp({ + touched: { nouns: [ID_B] }, + execute: async () => { + await storage.saveNounMetadata(ID_B, metadataFixture(1)) + } + }) + await store.flushPendingSingleOps() + expect(await store.historyBytes()).toBe(await groundTruthBytes()) + + await store.historyBytes() // re-seed after ground-truth reset + await store.compact({ maxGenerations: 1 }) + expect(await store.historyBytes()).toBe(await groundTruthBytes()) + }) + + it('historyStats reports counts, bytes, range, and horizon read-only', async () => { + await commitWrite(ID_A, 1) + await commitWrite(ID_B, 1) + const stats = await store.historyStats() + expect(stats.generations).toBe(2) + expect(stats.bytes).toBe(await store.historyBytes()) + expect(stats.oldestGeneration).toBe(1) + expect(stats.newestGeneration).toBe(2) + expect(stats.oldestTimestamp).toBeLessThanOrEqual(stats.newestTimestamp!) + expect(stats.horizon).toBe(0) + // Read-only: nothing was reclaimed by asking. + expect(store.committedGeneration()).toBe(2) + + await store.compact({ maxGenerations: 1 }) + const after = await store.historyStats() + expect(after.generations).toBe(1) + expect(after.oldestGeneration).toBe(2) + expect(after.horizon).toBe(1) + }) + + it('empty history reports null range and zero bytes', async () => { + const stats = await store.historyStats() + expect(stats).toMatchObject({ + generations: 0, + bytes: 0, + oldestGeneration: null, + newestGeneration: null, + oldestTimestamp: null, + newestTimestamp: null, + horizon: 0 + }) + }) + }) })