feat(8.0): Model-B per-write generation-stamping + adaptive retention knob
Every write — transact() AND single-op add/update/remove/relate — is now its
own immutable generation (Model-B), so a now() pin always freezes and
asOf/since/diff/history/transactionLog reflect single-ops exactly like
transacts. Closes the Model-A hole where pins did not freeze against single-op
writes.
Generation-stamping:
- GenerationStore.commitSingleOp: a one-operation commitTransaction with
deferred durability. Wired into add/update/remove/relate/updateRelation/
unrelate + removeMany (the *Many and VFS paths delegate to these).
- Async group-commit (flushPendingSingleOps): the live write is acknowledged
immediately; its before-image is buffered in an in-memory pending tier that
resolveAt/chains/changedBetween/tx-log read like on-disk generations, so the
synchronous now() freezes with no forced flush. One fsync per window
(triggers: size / 50ms timer / flush / close / transact / compactHistory).
- Crash recovery is drop-without-restore for group-commit generations (marked
groupCommit:true): a crash mid-flush discards the partial generation and
never restores its before-images, which would otherwise revert the
already-acknowledged live write.
- Init-time infrastructure (the VFS root) is the un-versioned generation-0
baseline: a fresh brain reports generation()===0 and an empty
transactionLog(); the first user write is generation 1.
- Historical find()/related() overlay bound is the full reserved watermark
(generation()), so un-flushed single-op writes are overlaid too.
Retention knob:
- config `history` -> `retention`: 'all' | 'adaptive' |
{ maxGenerations?, maxAge?, maxBytes?, budgetBytes?, autoCompact? }; unset ->
adaptive (disk/RAM pressure, zero-config). CompactHistoryOptions floors ->
caps (retainGenerations->maxGenerations, retainMs->maxAge, +maxBytes):
reclaim oldest-unpinned while ANY cap is exceeded; pins always exempt.
- brain.setRetentionBudget(bytes) drives the adaptive byte budget at runtime
(a coordinator's fair-share input). Per-generation bytes recorded in each
delta enable historyBytes() introspection without a storage size API.
Tests: per-write generation resolution, pin freeze vs add/update/remove,
drop-without-restore corruption-trap (fault injector), clean-reopen replay,
maxBytes/maxAge/no-cap reclamation, retention-then-reopen. 107 db/generation/
temporal tests green, tsc clean. Docs (ADR-001, consistency-model, snapshots
guide, api reference, RELEASES) updated to per-write granularity + retention.
This commit is contained in:
parent
afac7f9662
commit
5c3bb2c864
15 changed files with 1207 additions and 218 deletions
|
|
@ -8,7 +8,7 @@
|
|||
* resolution, and crash rollback of uncommitted generations.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { describe, it, expect, beforeEach, vi } from 'vitest'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import {
|
||||
GenerationStore,
|
||||
|
|
@ -253,12 +253,12 @@ describe('db/GenerationStore', () => {
|
|||
expect(remaining).toEqual([])
|
||||
})
|
||||
|
||||
it('retainGenerations keeps the most recent record-sets', async () => {
|
||||
it('maxGenerations cap keeps the most recent record-sets', async () => {
|
||||
await commitWrite(ID_A, 1)
|
||||
await commitWrite(ID_A, 2)
|
||||
await commitWrite(ID_A, 3)
|
||||
|
||||
const result = await store.compact({ retainGenerations: 2 })
|
||||
const result = await store.compact({ maxGenerations: 2 })
|
||||
expect(result.removedGenerations).toBe(1)
|
||||
expect(result.horizon).toBe(1)
|
||||
|
||||
|
|
@ -330,4 +330,141 @@ describe('db/GenerationStore', () => {
|
|||
expect(manifest.generation).toBe(2)
|
||||
})
|
||||
})
|
||||
|
||||
// ==========================================================================
|
||||
// Model-B single-op generation-stamping (per-write history + group-commit)
|
||||
// ==========================================================================
|
||||
describe('Model-B single-op generations', () => {
|
||||
/** Apply one single-op write as its own generation (live write + buffered history). */
|
||||
async function singleOpWrite(id: string, version: number): Promise<number> {
|
||||
const { generation } = await store.commitSingleOp({
|
||||
touched: { nouns: [id] },
|
||||
execute: async () => {
|
||||
await storage.saveNounMetadata(id, metadataFixture(version))
|
||||
}
|
||||
})
|
||||
return generation
|
||||
}
|
||||
|
||||
it('each single-op write is its own generation, resolvable BEFORE any flush', async () => {
|
||||
const g1 = await singleOpWrite(ID_A, 1)
|
||||
const g2 = await singleOpWrite(ID_A, 2)
|
||||
expect(g1).toBe(1)
|
||||
expect(g2).toBe(2)
|
||||
// Counter advanced per write; nothing persisted to disk yet.
|
||||
expect(store.generation()).toBe(2)
|
||||
expect(store.committedGeneration()).toBe(0)
|
||||
// Reads resolve through the in-memory pending tier with NO forced flush:
|
||||
// the state as-of g1 is v1 (the before-image buffered by g2).
|
||||
const atG1 = await store.resolveAt('noun', ID_A, g1)
|
||||
expect(atG1.source).toBe('record')
|
||||
expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1)
|
||||
// hasCommittedAfter counts pending → a now()-style pin at g1 is historical.
|
||||
expect(store.hasCommittedAfter(g1)).toBe(true)
|
||||
})
|
||||
|
||||
it('flush persists pending single-op generations (groupCommit-marked) and advances committed', async () => {
|
||||
await singleOpWrite(ID_A, 1)
|
||||
await singleOpWrite(ID_A, 2)
|
||||
await store.flushPendingSingleOps()
|
||||
expect(store.committedGeneration()).toBe(2)
|
||||
const delta = await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`)
|
||||
expect(delta.groupCommit).toBe(true)
|
||||
expect(typeof delta.bytes).toBe('number')
|
||||
// asOf resolution survives the flush (now reading before-images from disk).
|
||||
const atG1 = await store.resolveAt('noun', ID_A, 1)
|
||||
expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1)
|
||||
})
|
||||
|
||||
it('🛑 a crash mid group-commit flush DROPS the generation WITHOUT reverting the live write', async () => {
|
||||
// gen 1: X = v1, flushed (durable).
|
||||
await singleOpWrite(ID_A, 1)
|
||||
await store.flushPendingSingleOps()
|
||||
expect(store.committedGeneration()).toBe(1)
|
||||
|
||||
// gen 2: X = v2 acknowledged to canonical storage; its history is buffered.
|
||||
await singleOpWrite(ID_A, 2)
|
||||
expect((await storage.readNounRaw(ID_A)).metadata).toMatchObject({ version: 2 })
|
||||
|
||||
// Crash before the manifest rename of the group-commit flush: the gen-2
|
||||
// dir (before-image v1) lands on disk; the manifest stays at gen 1.
|
||||
store.setCommitFaultInjector((phase) => {
|
||||
if (phase === 'before-manifest-rename') throw new Error('simulated crash mid-flush')
|
||||
})
|
||||
await expect(store.flushPendingSingleOps()).rejects.toThrow('simulated crash mid-flush')
|
||||
store.setCommitFaultInjector(undefined)
|
||||
expect(await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`)).not.toBeNull()
|
||||
expect((await storage.readRawObject(MANIFEST_PATH)).generation).toBe(1)
|
||||
|
||||
// Recover on a FRESH store (process restart: in-memory pending is gone).
|
||||
const reopened = new GenerationStore(storage)
|
||||
const result = await reopened.open()
|
||||
expect(result.rolledBackGenerations).toBe(1)
|
||||
expect(reopened.committedGeneration()).toBe(1)
|
||||
|
||||
// THE CORRUPTION TRAP: drop-without-restore — the acknowledged live write
|
||||
// (v2) is NOT reverted to the before-image (v1). Only the crashed flush
|
||||
// window's HISTORY is lost; live data stays correct.
|
||||
expect((await storage.readNounRaw(ID_A)).metadata).toMatchObject({ version: 2 })
|
||||
// The orphan dir is gone and the dropped generation number is never reissued.
|
||||
expect(await storage.readRawObject(`${GENERATIONS_PREFIX}/2/tx.json`)).toBeNull()
|
||||
expect(reopened.generation()).toBeGreaterThanOrEqual(2)
|
||||
})
|
||||
|
||||
it('a clean reopen replays committed single-op history (asOf survives restart)', async () => {
|
||||
await singleOpWrite(ID_A, 1)
|
||||
await singleOpWrite(ID_A, 2)
|
||||
await store.flushPendingSingleOps()
|
||||
|
||||
const reopened = new GenerationStore(storage)
|
||||
await reopened.open()
|
||||
expect(reopened.committedGeneration()).toBe(2)
|
||||
const atG1 = await reopened.resolveAt('noun', ID_A, 1)
|
||||
expect(((atG1 as { metadata: { version: number } }).metadata).version).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// ==========================================================================
|
||||
// Retention CAPS (Model-B `retention` knob)
|
||||
// ==========================================================================
|
||||
describe('retention caps', () => {
|
||||
async function manyGens(n: number): Promise<void> {
|
||||
for (let v = 1; v <= n; v++) await commitWrite(ID_A, v)
|
||||
}
|
||||
|
||||
it('maxBytes reclaims the oldest generations until total history fits the budget', async () => {
|
||||
await manyGens(5)
|
||||
const total = await store.historyBytes()
|
||||
expect(total).toBeGreaterThan(0)
|
||||
const perGen = total / 5
|
||||
const budget = Math.floor(perGen * 2.5)
|
||||
const result = await store.compact({ maxBytes: budget })
|
||||
expect(result.removedGenerations).toBeGreaterThan(0)
|
||||
expect(await store.historyBytes()).toBeLessThanOrEqual(budget)
|
||||
})
|
||||
|
||||
it('maxAge keeps generations within the window and reclaims older ones', async () => {
|
||||
await manyGens(3)
|
||||
// A wide window keeps everything.
|
||||
expect((await store.compact({ maxAge: 60_000 })).removedGenerations).toBe(0)
|
||||
// Advance the clock 10s past the commits, then a 1s window reclaims all
|
||||
// three (fake timers make the age comparison deterministic).
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
vi.setSystemTime(Date.now() + 10_000)
|
||||
expect((await store.compact({ maxAge: 1_000 })).removedGenerations).toBe(3)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('no caps reclaims every unpinned generation; a pin protects newer ones', async () => {
|
||||
await manyGens(3)
|
||||
store.pin(2) // protects generations > 2 (i.e. gen 3) from reclamation
|
||||
const result = await store.compact()
|
||||
// gens 1 and 2 are ≤ the pin → reclaimable; gen 3 is pin-protected.
|
||||
expect(result.removedGenerations).toBe(2)
|
||||
store.release(2)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue