diff --git a/src/brainy.ts b/src/brainy.ts index 24407018..013885ff 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -740,6 +740,18 @@ export class Brainy implements BrainyInterface { // Write acks NEVER await it; a failed background flush is LOUD and re-armed. private _persistDirtyWrites = 0 private _persistLastFlushAt = Date.now() + /** + * Whether a write has been committed since the last flush that ran. THE + * ENGINE DOES NO PERIODIC WORK WITHOUT A CAUSE: a brain nobody has written + * to has nothing to make durable, and a flush over it must cost nothing and + * say nothing. Measured on a production process holding 21 brains: with no + * writes for ten minutes it still printed "All indexes flushed to disk in + * 216–601ms" per brain every ~35s and idled at 1.26 cores, because a flush + * called every provider, stamped the watermarks, persisted the generation + * counter and re-stamped the entity tree whether or not anything had + * changed. + */ + private _dirtySinceLastFlush = false private _persistIdleTimer: ReturnType | null = null private _persistBackgroundFlight: Promise | null = null @@ -2668,6 +2680,12 @@ export class Brainy implements BrainyInterface { * engine's own cadence (callers never call flush() in hot paths). */ private noteWriteForPersistence(): void { + // THE DIRTY WITNESS. Set on every committed write — both commit paths + // (single-op and transaction) end here, and the deferred-embed worker + // lands its vectors through the single-op path — BEFORE the policy check, + // so a `'manual'` consumer's explicit flush() is never skipped either. + // Cleared by a flush that actually runs; see flush(). + this._dirtySinceLastFlush = true const cfg = this.config.persistence if (this.isReadOnly || cfg?.policy === 'manual') return this._persistDirtyWrites++ @@ -12246,6 +12264,27 @@ export class Brainy implements BrainyInterface { return } + // A CLEAN BRAIN FLUSHES NOTHING, AND SAYS NOTHING. No write has been + // committed since the last flush, so every step below would re-persist + // state identical to what is already on disk — provider flushes, the + // watermark stamps, the generation counter, the entity-tree stamp — and + // print two lines announcing it. On a process holding 21 brains that + // no-op cost 1.26 cores at idle. The witness is set by every committed + // write (see noteWriteForPersistence) and cleared here; a write landing + // DURING this flush sets it again, so it is never lost — the next flush + // does that write's work. + if (!this._dirtySinceLastFlush) { + return + } + this._dirtySinceLastFlush = false + // An explicit flush IS a flush: tell the cadence so, or the very next + // write sees "30s since the last flush" (the cadence only counted its + // own) and kicks a background flush that has nothing left to do, and the + // idle timer fires two seconds later over writes this flush already + // persisted. + this._persistLastFlushAt = Date.now() + this._persistDirtyWrites = 0 + console.log('Flushing Brainy indexes and caches to disk...') const startTime = Date.now() diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index d002164e..ebd3b90c 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -1052,6 +1052,17 @@ export class GraphAdjacencyIndex implements GraphIndexProvider { */ private startAutoFlush(): void { this.flushTimer = setInterval(async () => { + // NO PERIODIC WORK WITHOUT A CAUSE. Ask first, in two O(1) reads: an + // index nobody has written to since the last flush has nothing to + // write, and calling into the trees (and their logging) on a cadence + // over a quiet store is exactly the idle cost this law exists to + // remove. + if ( + !this.lsmTreeVerbsBySource.hasPendingWrites() && + !this.lsmTreeVerbsByTarget.hasPendingWrites() + ) { + return + } await this.flush() }, this.config.flushInterval) // Background maintenance must never keep the host process alive — diff --git a/src/graph/lsm/LSMTree.ts b/src/graph/lsm/LSMTree.ts index e19ec145..b4f6052f 100644 --- a/src/graph/lsm/LSMTree.ts +++ b/src/graph/lsm/LSMTree.ts @@ -687,6 +687,17 @@ export class LSMTree { } } + /** + * @description Whether this tree holds anything a flush would write — + * the MemTable is non-empty. Synchronous and O(1), so a background cadence + * can ask before it does anything at all: the engine does no periodic work + * without a cause. + * @returns true when a flush would write; false when it would be a no-op. + */ + hasPendingWrites(): boolean { + return !this.memTable.isEmpty() + } + async close(): Promise { this.stopCompactionTimer() diff --git a/tests/integration/idle-costs-nothing.test.ts b/tests/integration/idle-costs-nothing.test.ts new file mode 100644 index 00000000..8f951d46 --- /dev/null +++ b/tests/integration/idle-costs-nothing.test.ts @@ -0,0 +1,147 @@ +/** + * @module tests/integration/idle-costs-nothing + * @description AN IDLE BRAIN DOES NO WORK. + * + * Measured on a production process holding 21 brains: with no writes for ten + * minutes it printed "All indexes flushed to disk in 216–601ms" per brain + * every ~35 seconds and idled at 1.26 cores. Every one of those flushes + * re-persisted state identical to what was already on disk — the provider + * flushes, the watermark stamps, the generation counter, the entity-tree + * stamp — because `flush()` never asked whether anything had changed. + * + * The laws pinned here: + * (a) the persistence cadence arms only on a write — a brain nobody writes + * to flushes zero times, however long it is left open; + * (b) a flush on a clean brain is O(1): no provider is called, nothing is + * written, and nothing is printed; + * (c) one write earns exactly one flush's worth of work, and no more. + */ + +import { describe, it, expect, afterEach, vi } from 'vitest' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/brainy.js' +import { NounType } from '../../src/types/graphTypes.js' + +/** Wait for any in-flight background flush, then let the idle timer settle. */ +async function drainCadence(brain: Brainy): Promise { + const inner = brain as unknown as { _persistBackgroundFlight: Promise | null } + await new Promise((r) => setTimeout(r, 3_000)) + await (inner._persistBackgroundFlight ?? Promise.resolve()) + await new Promise((r) => setTimeout(r, 500)) +} + +/** How long an idle brain is watched. Longer than the 30s flush interval. */ +const IDLE_WATCH_MS = 90_000 + +describe('an idle brain costs nothing', () => { + const dirs: string[] = [] + const brains: Brainy[] = [] + + afterEach(async () => { + for (const b of brains.splice(0)) { + try { await b.close() } catch { /* already closed */ } + } + for (const d of dirs.splice(0)) { + try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ } + } + vi.restoreAllMocks() + }) + + async function openBrain(): Promise { + const dir = mkdtempSync(join(tmpdir(), 'brainy-idle-')) + dirs.push(dir) + const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) + brains.push(brain) + await brain.init() + return brain + } + + it('flushes zero times over 90 idle seconds, and prints nothing', async () => { + const brain = await openBrain() + // One write and one flush to reach a clean, settled state — then nothing. + await brain.add({ data: 'the only write this test performs', type: NounType.Concept }) + await brain.flush() + + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + + // Watch the providers directly: a flush that runs calls all of them. + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex + const graphIndex = (brain as unknown as { graphIndex: { flush: () => Promise } }).graphIndex + const countsSpy = vi.spyOn(storage, 'flushCounts') + const metadataSpy = vi.spyOn(metadataIndex, 'flush') + const graphSpy = vi.spyOn(graphIndex, 'flush') + + try { + await new Promise((r) => setTimeout(r, IDLE_WATCH_MS)) + } finally { + console.log = origLog + } + + // (a) + (b): nothing ran, nothing was said. + expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) + expect(logged.filter((l) => /Flushing Brainy indexes/.test(l))).toEqual([]) + expect(countsSpy).not.toHaveBeenCalled() + expect(metadataSpy).not.toHaveBeenCalled() + expect(graphSpy).not.toHaveBeenCalled() + }, 180_000) + + it('an explicit flush over a clean brain calls no provider and prints nothing', async () => { + const brain = await openBrain() + await brain.add({ data: 'one write', type: NounType.Concept }) + await brain.flush() // this one does the work + + const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage + const metadataIndex = (brain as unknown as { metadataIndex: { flush: () => Promise } }).metadataIndex + const countsSpy = vi.spyOn(storage, 'flushCounts') + const metadataSpy = vi.spyOn(metadataIndex, 'flush') + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + try { + await brain.flush() // ...and this one has nothing to do + await brain.flush() + await brain.flush() + } finally { + console.log = origLog + } + + expect(countsSpy).not.toHaveBeenCalled() + expect(metadataSpy).not.toHaveBeenCalled() + expect(logged.filter((l) => /All indexes flushed to disk/.test(l))).toEqual([]) + }, 120_000) + + it('one write earns exactly one flush', async () => { + const brain = await openBrain() + await brain.add({ data: 'first', type: NounType.Concept }) + await brain.flush() + // Settle: the first write also kicked a BACKGROUND flush, which is not + // awaited by design. Drain it before counting, or its provider calls land + // inside this test's window and are attributed to the write below. + await drainCadence(brain) + + // Count the flushes that actually RAN. (Provider spies cannot answer this: + // the storage adapter's own count ledger is write-through, so a write calls + // flushCounts() on its own account, with no flush involved.) + const logged: string[] = [] + const origLog = console.log + console.log = ((...a: unknown[]) => { logged.push(a.map(String).join(' ')) }) as typeof console.log + const ran = () => logged.filter((l) => /All indexes flushed to disk/.test(l)).length + try { + await brain.add({ data: 'second — this is the cause', type: NounType.Concept }) + await brain.flush() + expect(ran()).toBe(1) + + // No further cause, no further work. + await brain.flush() + await brain.flush() + expect(ran()).toBe(1) + } finally { + console.log = origLog + } + }, 120_000) +})