feat(8.0): Model-B per-write generation-stamping + adaptive retention knob
Every write — transact() AND single-op add/update/remove/relate — is now its
own immutable generation (Model-B), so a now() pin always freezes and
asOf/since/diff/history/transactionLog reflect single-ops exactly like
transacts. Closes the Model-A hole where pins did not freeze against single-op
writes.
Generation-stamping:
- GenerationStore.commitSingleOp: a one-operation commitTransaction with
deferred durability. Wired into add/update/remove/relate/updateRelation/
unrelate + removeMany (the *Many and VFS paths delegate to these).
- Async group-commit (flushPendingSingleOps): the live write is acknowledged
immediately; its before-image is buffered in an in-memory pending tier that
resolveAt/chains/changedBetween/tx-log read like on-disk generations, so the
synchronous now() freezes with no forced flush. One fsync per window
(triggers: size / 50ms timer / flush / close / transact / compactHistory).
- Crash recovery is drop-without-restore for group-commit generations (marked
groupCommit:true): a crash mid-flush discards the partial generation and
never restores its before-images, which would otherwise revert the
already-acknowledged live write.
- Init-time infrastructure (the VFS root) is the un-versioned generation-0
baseline: a fresh brain reports generation()===0 and an empty
transactionLog(); the first user write is generation 1.
- Historical find()/related() overlay bound is the full reserved watermark
(generation()), so un-flushed single-op writes are overlaid too.
Retention knob:
- config `history` -> `retention`: 'all' | 'adaptive' |
{ maxGenerations?, maxAge?, maxBytes?, budgetBytes?, autoCompact? }; unset ->
adaptive (disk/RAM pressure, zero-config). CompactHistoryOptions floors ->
caps (retainGenerations->maxGenerations, retainMs->maxAge, +maxBytes):
reclaim oldest-unpinned while ANY cap is exceeded; pins always exempt.
- brain.setRetentionBudget(bytes) drives the adaptive byte budget at runtime
(a coordinator's fair-share input). Per-generation bytes recorded in each
delta enable historyBytes() introspection without a storage size API.
Tests: per-write generation resolution, pin freeze vs add/update/remove,
drop-without-restore corruption-trap (fault injector), clean-reopen replay,
maxBytes/maxAge/no-cap reclamation, retention-then-reopen. 107 db/generation/
temporal tests green, tsc clean. Docs (ADR-001, consistency-model, snapshots
guide, api reference, RELEASES) updated to per-write granularity + retention.
This commit is contained in:
parent
afac7f9662
commit
5c3bb2c864
15 changed files with 1207 additions and 218 deletions
|
|
@ -58,7 +58,7 @@ async function mkBrain(backend: 'memory' | 'filesystem', dir?: string): Promise<
|
|||
storage,
|
||||
eagerEmbeddings: false, // never load WASM — we pass precomputed vectors
|
||||
// Keep history fully retained for the measurement (no mid-run compaction skewing depth).
|
||||
history: { autoCompact: false } as any,
|
||||
retention: { autoCompact: false } as any,
|
||||
index: { m: 16, efConstruction: 200, efSearch: 50 },
|
||||
cache: { maxSize: 1000, ttl: 3600 }
|
||||
} as BrainyConfig)
|
||||
|
|
@ -176,13 +176,13 @@ async function main() {
|
|||
await b.flush()
|
||||
const beforeBytes = allocBytes(`${dir}/_generations`)
|
||||
const t = process.hrtime.bigint()
|
||||
const res = await b.compactHistory({ retainGenerations: 100 } as any)
|
||||
const res = await b.compactHistory({ maxGenerations: 100 } as any)
|
||||
const compMs = Number(process.hrtime.bigint() - t) / 1e6
|
||||
await b.flush()
|
||||
const afterBytes = allocBytes(`${dir}/_generations`)
|
||||
await b.close()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
say(`N=${N.toLocaleString()} gens → compactHistory({retainGenerations:100}): ${compMs.toFixed(0)} ms, removed ${(res as any)?.removedGenerations ?? '?'} gens`)
|
||||
say(`N=${N.toLocaleString()} gens → compactHistory({maxGenerations:100}): ${compMs.toFixed(0)} ms, removed ${(res as any)?.removedGenerations ?? '?'} gens`)
|
||||
say(` _generations footprint: ${(beforeBytes / 1024 / 1024).toFixed(1)} MiB → ${(afterBytes / 1024 / 1024).toFixed(1)} MiB allocated`)
|
||||
results.compaction.push({ N, compactMs: compMs, removed: (res as any)?.removedGenerations, beforeBytes, afterBytes })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1118,12 +1118,17 @@ describe('8.0 Db API — generational MVCC', () => {
|
|||
await db.release()
|
||||
})
|
||||
|
||||
it('transactionLog() returns committed entries newest first, with meta and limit', async () => {
|
||||
it('transactionLog() includes single-op AND transact generations, newest first, with meta and limit', async () => {
|
||||
const brain = await openMemoryBrain()
|
||||
|
||||
// Nothing committed yet — the log is empty (single-op writes do not log).
|
||||
// Model-B: a single-op write is its OWN generation and IS logged (no meta —
|
||||
// tx metadata is a transact()-only concept). It is generation 1 on a fresh
|
||||
// brain (init-time infrastructure writes are the un-versioned gen-0 baseline).
|
||||
await brain.add({ id: uid('txlog-solo'), type: NounType.Document, data: 'solo', vector: vec(99), subtype: 'note' })
|
||||
expect(await brain.transactionLog()).toEqual([])
|
||||
const soloLog = await brain.transactionLog()
|
||||
expect(soloLog.map((entry) => entry.generation)).toEqual([1])
|
||||
expect(soloLog[0].meta).toBeUndefined()
|
||||
const soloGen = 1
|
||||
|
||||
const first = await brain.transact(
|
||||
[{ op: 'add', id: uid('txlog-a'), type: NounType.Document, data: 'a', vector: vec(100), metadata: {} }],
|
||||
|
|
@ -1136,14 +1141,17 @@ describe('8.0 Db API — generational MVCC', () => {
|
|||
const third = await brain.transact([{ op: 'update', id: uid('txlog-a'), metadata: { v: 3 } }])
|
||||
|
||||
const entries = await brain.transactionLog()
|
||||
// Newest first: the three transacts, then the single-op solo write (gen 1).
|
||||
expect(entries.map((entry) => entry.generation)).toEqual([
|
||||
third.generation,
|
||||
second.generation,
|
||||
first.generation
|
||||
first.generation,
|
||||
soloGen
|
||||
])
|
||||
expect(entries[1].meta).toEqual({ author: 'job-2' })
|
||||
expect(entries[2].meta).toEqual({ author: 'job-1' })
|
||||
expect(entries[0].meta).toBeUndefined()
|
||||
expect(entries[0].meta).toBeUndefined() // third() had no meta
|
||||
expect(entries[3].meta).toBeUndefined() // the single-op solo write carries no meta
|
||||
for (const entry of entries) {
|
||||
expect(entry.timestamp).toBeGreaterThan(0)
|
||||
}
|
||||
|
|
@ -1156,6 +1164,82 @@ describe('8.0 Db API — generational MVCC', () => {
|
|||
await third.release()
|
||||
})
|
||||
|
||||
// ==========================================================================
|
||||
// 8b. Model-B single-op generation-stamping (every write versioned)
|
||||
// ==========================================================================
|
||||
it('Model-B — a now() pin freezes against single-op add/update/remove (the Model-A hole is closed)', async () => {
|
||||
const brain = await openMemoryBrain()
|
||||
const a = uid('mb-a')
|
||||
const b = uid('mb-b')
|
||||
// Seed `a` with a single-op add, then pin the current generation.
|
||||
await brain.add({ id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } })
|
||||
const pin = brain.now()
|
||||
|
||||
// Single-op mutations AFTER the pin: update `a`, add `b`.
|
||||
await brain.update({ id: a, metadata: { v: 2 } })
|
||||
await brain.add({ id: b, type: NounType.Document, data: 'b', vector: vec(2), metadata: { v: 1 } })
|
||||
|
||||
// The pin is frozen at its generation — single-op writes do not leak through
|
||||
// it (pre-Model-B, the pin tracked the live single-op mutation).
|
||||
expect((await pin.get(a))?.metadata?.v).toBe(1)
|
||||
expect(await pin.get(b)).toBeNull()
|
||||
expect((await brain.get(a))?.metadata?.v).toBe(2)
|
||||
expect((await brain.get(b))?.metadata?.v).toBe(1)
|
||||
expect(pin.isHistorical()).toBe(true)
|
||||
|
||||
// A single-op REMOVE also leaves the pin untouched.
|
||||
await brain.remove(a)
|
||||
expect((await pin.get(a))?.metadata?.v).toBe(1)
|
||||
expect(await brain.get(a)).toBeNull()
|
||||
await pin.release()
|
||||
})
|
||||
|
||||
it('Model-B — historical find() overlays an un-flushed single-op write (overlay bound is generation, not committed)', async () => {
|
||||
const brain = await openMemoryBrain()
|
||||
const a = uid('ov-a')
|
||||
const b = uid('ov-b')
|
||||
await (
|
||||
await brain.transact([
|
||||
{ op: 'add', id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } },
|
||||
{ op: 'add', id: b, type: NounType.Document, data: 'b', vector: vec(2), metadata: { v: 1 } }
|
||||
])
|
||||
).release()
|
||||
const at1 = await brain.asOf(1)
|
||||
|
||||
// A single-op REMOVE of `b` lands AFTER the pin and is NOT flushed (pending).
|
||||
await brain.remove(b)
|
||||
|
||||
const liveIds = (await brain.find({})).map((r) => r.id)
|
||||
const pastIds = (await at1.find({})).map((r) => r.id)
|
||||
// Live: `b` is gone. Historical (pinned at gen 1): the un-flushed removal is
|
||||
// overlaid out, so `b` is still present at its pinned state.
|
||||
expect(liveIds).toContain(a)
|
||||
expect(liveIds).not.toContain(b)
|
||||
expect(pastIds).toContain(a)
|
||||
expect(pastIds).toContain(b)
|
||||
await at1.release()
|
||||
})
|
||||
|
||||
it('Model-B retention — explicit caps reclaim single-op history; committed history survives reopen', async () => {
|
||||
const { brain, dir } = await openFsBrain()
|
||||
const a = uid('ret-a')
|
||||
await brain.add({ id: a, type: NounType.Document, data: 'a', vector: vec(1), metadata: { v: 1 } })
|
||||
for (let v = 2; v <= 6; v++) await brain.update({ id: a, metadata: { v } })
|
||||
await brain.flush() // persist the per-write generations to disk
|
||||
expect(brain.generation()).toBe(6)
|
||||
|
||||
// Cap to the 2 most recent generations — older single-op history is reclaimed.
|
||||
const res = await brain.compactHistory({ maxGenerations: 2 })
|
||||
expect(res.removedGenerations).toBeGreaterThan(0)
|
||||
expect(res.horizon).toBeGreaterThan(0)
|
||||
await brain.close()
|
||||
|
||||
// Reopen: the survivors + live value are intact; below-horizon asOf throws.
|
||||
const { brain: reopened } = await openFsBrain(dir)
|
||||
expect((await reopened.get(a))?.metadata?.v).toBe(6)
|
||||
await expect(reopened.asOf(1)).rejects.toBeInstanceOf(GenerationCompactedError)
|
||||
})
|
||||
|
||||
// ==========================================================================
|
||||
// 9. asOf() / released-Db error paths (Y.15 spot-checks)
|
||||
// ==========================================================================
|
||||
|
|
|
|||
|
|
@ -330,8 +330,8 @@ describe('8.0 Db API — temporal range verbs', () => {
|
|||
await dbAtG2.release()
|
||||
})
|
||||
|
||||
// 7. Granularity -------------------------------------------------------------
|
||||
it('granularity: single-operation writes (outside transact) are invisible to the temporal verbs', async () => {
|
||||
// 7. Granularity (Model-B) ---------------------------------------------------
|
||||
it('granularity: single-operation writes ARE versioned and visible to the temporal verbs', async () => {
|
||||
const brain = await openMemoryBrain()
|
||||
const a = uid('gran-a')
|
||||
const r1 = await brain.transact([
|
||||
|
|
@ -340,13 +340,26 @@ describe('8.0 Db API — temporal range verbs', () => {
|
|||
await r1.release()
|
||||
expect((await brain.transactionLog()).length).toBe(1)
|
||||
|
||||
// A single-op write bumps the counter but writes no generation record/log entry.
|
||||
// Model-B: a single-op write is its OWN immutable generation — logged,
|
||||
// diffable, and time-travelable, exactly like a transact() of one op.
|
||||
await brain.update({ id: a, metadata: { v: 2 } })
|
||||
|
||||
expect((await brain.transactionLog()).length).toBe(1) // unchanged — not logged
|
||||
// The single-op update appended a generation/log entry.
|
||||
expect((await brain.transactionLog()).length).toBe(2)
|
||||
expect(brain.generation()).toBe(r1.generation + 1)
|
||||
|
||||
// diff sees the single-op update as a modification of `a`.
|
||||
const d = await brain.diff(r1.generation, brain.generation())
|
||||
expect(d.added.nouns).toEqual([]) // the single-op update is not a recorded change
|
||||
expect(d.modified.nouns).toEqual([])
|
||||
expect(d.added.nouns).toEqual([])
|
||||
expect(d.modified.nouns).toEqual([a])
|
||||
|
||||
// And asOf() resolves both states: v=1 at the transact gen, v=2 at the single-op gen.
|
||||
const atTransact = await brain.asOf(r1.generation)
|
||||
const atSingleOp = await brain.asOf(brain.generation())
|
||||
expect((await atTransact.get(a))?.metadata?.v).toBe(1)
|
||||
expect((await atSingleOp.get(a))?.metadata?.v).toBe(2)
|
||||
await atTransact.release()
|
||||
await atSingleOp.release()
|
||||
})
|
||||
|
||||
// 8. Compaction policy contrast ---------------------------------------------
|
||||
|
|
@ -363,7 +376,7 @@ describe('8.0 Db API — temporal range verbs', () => {
|
|||
).release()
|
||||
} // gens 1..6, all pins released
|
||||
|
||||
await brain.compactHistory({ retainGenerations: 2 }) // raise the horizon
|
||||
await brain.compactHistory({ maxGenerations: 2 }) // raise the horizon
|
||||
const horizon = generationStoreOf(brain).horizon()
|
||||
expect(horizon).toBeGreaterThan(0)
|
||||
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ describe('generation history chains (Model-B scalability)', () => {
|
|||
await bumpX(4)
|
||||
|
||||
// Reclaim everything except the 2 most recent committed generations.
|
||||
const res = await brain.compactHistory({ retainGenerations: 2 })
|
||||
const res = await brain.compactHistory({ maxGenerations: 2 })
|
||||
expect(res.removedGenerations).toBeGreaterThan(0)
|
||||
|
||||
// A pin ABOVE the new horizon still resolves (chains were rebuilt from the survivors)…
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
* resolution, and crash rollback of uncommitted generations.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import {
|
||||
GenerationStore,
|
||||
|
|
@ -253,12 +253,12 @@ describe('db/GenerationStore', () => {
|
|||
expect(remaining).toEqual([])
|
||||
})
|
||||
|
||||
it('retainGenerations keeps the most recent record-sets', async () => {
|
||||
it('maxGenerations cap keeps the most recent record-sets', async () => {
|
||||
await commitWrite(ID_A, 1)
|
||||
await commitWrite(ID_A, 2)
|
||||
await commitWrite(ID_A, 3)
|
||||
|
||||
const result = await store.compact({ retainGenerations: 2 })
|
||||
const result = await store.compact({ maxGenerations: 2 })
|
||||
expect(result.removedGenerations).toBe(1)
|
||||
expect(result.horizon).toBe(1)
|
||||
|
||||
|
|
@ -330,4 +330,141 @@ describe('db/GenerationStore', () => {
|
|||
expect(manifest.generation).toBe(2)
|
||||
})
|
||||
})
|
||||
|
||||
// ==========================================================================
|
||||
// Model-B single-op generation-stamping (per-write history + group-commit)
|
||||
// ==========================================================================
|
||||
describe('Model-B single-op generations', () => {
|
||||
/** Apply one single-op write as its own generation (live write + buffered history). */
|
||||
async function singleOpWrite(id: string, version: number): Promise<number> {
|
||||
const { generation } = await store.commitSingleOp({
|
||||
touched: { nouns: [id] },
|
||||
execute: async () => {
|
||||
await storage.saveNounMetadata(id, metadataFixture(version))
|
||||
}
|
||||
})
|
||||
return generation
|
||||
}
|
||||
|
||||
it('each single-op write is its own generation, resolvable BEFORE any flush', async () => {
|
||||
const g1 = await singleOpWrite(ID_A, 1)
|
||||
const g2 = await singleOpWrite(ID_A, 2)
|
||||
expect(g1).toBe(1)
|
||||
expect(g2).toBe(2)
|
||||
// Counter advanced per write; nothing persisted to disk yet.
|
||||
expect(store.generation()).toBe(2)
|
||||
expect(store.committedGeneration()).toBe(0)
|
||||
// Reads resolve through the in-memory pending tier with NO forced flush:
|
||||
// the state as-of g1 is v1 (the before-image buffered by g2).
|
||||
const atG1 = await store.resolveAt('noun', ID_A, g1)
|
||||
expect(atG1.source).toBe('record')
|
||||
expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1)
|
||||
// hasCommittedAfter counts pending → a now()-style pin at g1 is historical.
|
||||
expect(store.hasCommittedAfter(g1)).toBe(true)
|
||||
})
|
||||
|
||||
it('flush persists pending single-op generations (groupCommit-marked) and advances committed', async () => {
|
||||
await singleOpWrite(ID_A, 1)
|
||||
await singleOpWrite(ID_A, 2)
|
||||
await store.flushPendingSingleOps()
|
||||
expect(store.committedGeneration()).toBe(2)
|
||||
const delta = await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`)
|
||||
expect(delta.groupCommit).toBe(true)
|
||||
expect(typeof delta.bytes).toBe('number')
|
||||
// asOf resolution survives the flush (now reading before-images from disk).
|
||||
const atG1 = await store.resolveAt('noun', ID_A, 1)
|
||||
expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1)
|
||||
})
|
||||
|
||||
it('🛑 a crash mid group-commit flush DROPS the generation WITHOUT reverting the live write', async () => {
|
||||
// gen 1: X = v1, flushed (durable).
|
||||
await singleOpWrite(ID_A, 1)
|
||||
await store.flushPendingSingleOps()
|
||||
expect(store.committedGeneration()).toBe(1)
|
||||
|
||||
// gen 2: X = v2 acknowledged to canonical storage; its history is buffered.
|
||||
await singleOpWrite(ID_A, 2)
|
||||
expect((await storage.readNounRaw(ID_A)).metadata).toMatchObject({ version: 2 })
|
||||
|
||||
// Crash before the manifest rename of the group-commit flush: the gen-2
|
||||
// dir (before-image v1) lands on disk; the manifest stays at gen 1.
|
||||
store.setCommitFaultInjector((phase) => {
|
||||
if (phase === 'before-manifest-rename') throw new Error('simulated crash mid-flush')
|
||||
})
|
||||
await expect(store.flushPendingSingleOps()).rejects.toThrow('simulated crash mid-flush')
|
||||
store.setCommitFaultInjector(undefined)
|
||||
expect(await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`)).not.toBeNull()
|
||||
expect((await storage.readRawObject(MANIFEST_PATH)).generation).toBe(1)
|
||||
|
||||
// Recover on a FRESH store (process restart: in-memory pending is gone).
|
||||
const reopened = new GenerationStore(storage)
|
||||
const result = await reopened.open()
|
||||
expect(result.rolledBackGenerations).toBe(1)
|
||||
expect(reopened.committedGeneration()).toBe(1)
|
||||
|
||||
// THE CORRUPTION TRAP: drop-without-restore — the acknowledged live write
|
||||
// (v2) is NOT reverted to the before-image (v1). Only the crashed flush
|
||||
// window's HISTORY is lost; live data stays correct.
|
||||
expect((await storage.readNounRaw(ID_A)).metadata).toMatchObject({ version: 2 })
|
||||
// The orphan dir is gone and the dropped generation number is never reissued.
|
||||
expect(await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`)).toBeNull()
|
||||
expect(reopened.generation()).toBeGreaterThanOrEqual(2)
|
||||
})
|
||||
|
||||
it('a clean reopen replays committed single-op history (asOf survives restart)', async () => {
|
||||
await singleOpWrite(ID_A, 1)
|
||||
await singleOpWrite(ID_A, 2)
|
||||
await store.flushPendingSingleOps()
|
||||
|
||||
const reopened = new GenerationStore(storage)
|
||||
await reopened.open()
|
||||
expect(reopened.committedGeneration()).toBe(2)
|
||||
const atG1 = await reopened.resolveAt('noun', ID_A, 1)
|
||||
expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// ==========================================================================
|
||||
// Retention CAPS (Model-B `retention` knob)
|
||||
// ==========================================================================
|
||||
describe('retention caps', () => {
|
||||
async function manyGens(n: number): Promise<void> {
|
||||
for (let v = 1; v <= n; v++) await commitWrite(ID_A, v)
|
||||
}
|
||||
|
||||
it('maxBytes reclaims the oldest generations until total history fits the budget', async () => {
|
||||
await manyGens(5)
|
||||
const total = await store.historyBytes()
|
||||
expect(total).toBeGreaterThan(0)
|
||||
const perGen = total / 5
|
||||
const budget = Math.floor(perGen * 2.5)
|
||||
const result = await store.compact({ maxBytes: budget })
|
||||
expect(result.removedGenerations).toBeGreaterThan(0)
|
||||
expect(await store.historyBytes()).toBeLessThanOrEqual(budget)
|
||||
})
|
||||
|
||||
it('maxAge keeps generations within the window and reclaims older ones', async () => {
|
||||
await manyGens(3)
|
||||
// A wide window keeps everything.
|
||||
expect((await store.compact({ maxAge: 60_000 })).removedGenerations).toBe(0)
|
||||
// Advance the clock 10s past the commits, then a 1s window reclaims all
|
||||
// three (fake timers make the age comparison deterministic).
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
vi.setSystemTime(Date.now() + 10_000)
|
||||
expect((await store.compact({ maxAge: 1_000 })).removedGenerations).toBe(3)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('no caps reclaims every unpinned generation; a pin protects newer ones', async () => {
|
||||
await manyGens(3)
|
||||
store.pin(2) // protects generations > 2 (i.e. gen 3) from reclamation
|
||||
const result = await store.compact()
|
||||
// gens 1 and 2 are ≤ the pin → reclaimable; gen 3 is pin-protected.
|
||||
expect(result.removedGenerations).toBe(2)
|
||||
store.release(2)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue