From d1ecee10f0bca9fd82433ea5c0e1138854313a21 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Wed, 15 Jul 2026 12:44:52 -0700 Subject: [PATCH] feat: committedGeneration capability + pinned durability/stability contracts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - storage.committedGeneration(): the committed watermark exposed on the fact-scan capability, so a provider compares its stamp's sourceGeneration against the store's truth without parsing the private manifest format (the capability source now carries both the live fact log and the committed closure). - Pinned contract tests: fsync-before-ack (transact passes today; the single-op path is pinned via it.fails as the documented target — group commit must become LATENCY batching, ack waiting on the shared fsync, never durability skipping; the pin flips red when that lands so there is no cliff to discover) and scan-stability-under-rotation (a scan snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate beneath it; the reclaim-during-scan variant lands with fact compaction, which does not exist yet). - FactLog accepts a rotateBytes option so tests exercise real rotation. --- RELEASES.md | 8 ++ src/brainy.ts | 9 +- src/coreTypes.ts | 9 ++ src/db/factLog.ts | 7 +- src/storage/baseStorage.ts | 30 ++++- tests/integration/fact-log-contracts.test.ts | 125 +++++++++++++++++++ 6 files changed, 177 insertions(+), 11 deletions(-) create mode 100644 tests/integration/fact-log-contracts.test.ts diff --git a/RELEASES.md b/RELEASES.md index 11f7500e..457b65d0 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -24,6 +24,14 @@ Small additive follow-up to 8.4.0, from the native accelerator's first consumpti providers run literally the same verification function, never a synchronized copy. - **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass. +- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider + compares its stamp's `sourceGeneration` against the store's truth without parsing the private + manifest format. +- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for + `transact()` today; the single-op path is pinned as the documented future target — group commit + becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan + snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate + beneath it). No behavior change for applications; all additions. diff --git a/src/brainy.ts b/src/brainy.ts index a188ad87..d5d4bc33 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -1005,9 +1005,12 @@ export class Brainy implements BrainyInterface { // so a provider's heal can switch from the enumeration walk to one // sequential fact scan whenever the log exists. if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') { - ;(this.storage as BaseStorage).setFactScanSource( - () => this.generationStore?.getFactLog() ?? null - ) + ;(this.storage as BaseStorage).setFactScanSource({ + factLog: () => this.generationStore?.getFactLog() ?? null, + // The committed watermark, exposed as a capability so providers + // never parse the store's private manifest format. + committedGeneration: () => this.generationStore?.committedGeneration() ?? 0 + }) } // Entity-tree stamp coherence: compare the stamped sourceGeneration + diff --git a/src/coreTypes.ts b/src/coreTypes.ts index 26783e0f..e0248d17 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -1182,6 +1182,15 @@ export interface StorageAdapter { */ factLogHeadGeneration?(): number | null + /** + * @description OPTIONAL (rides the fact-scan capability): the COMMITTED + * generation watermark — the manifest truth a projection's + * `sourceGeneration` compares against. Exposed as a capability so a + * provider never parses the store's private manifest format. `null` when + * the capability is unwired. + */ + committedGeneration?(): number | null + /** * @description OPTIONAL (rides the fact-scan capability): the immutable, * sealed fact-segment file paths covering `fromGeneration` — the zero-copy diff --git a/src/db/factLog.ts b/src/db/factLog.ts index b43e52d0..4c5e95fd 100644 --- a/src/db/factLog.ts +++ b/src/db/factLog.ts @@ -294,6 +294,8 @@ function parseSegment( */ export class FactLog { private readonly storage: FactLogStorage + /** Rotation threshold (bytes); tests may lower it to exercise rotation. */ + private readonly rotateBytes: number private manifest: FactsManifest = { formatVersion: FACTS_FORMAT_VERSION, segments: [], @@ -309,8 +311,9 @@ export class FactLog { /** Segment paths appended since the last sync (the fsync batch). */ private readonly dirtySegments = new Set() - constructor(storage: FactLogStorage) { + constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) { this.storage = storage + this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES } /** The highest committed generation the log holds (0 = empty). */ @@ -405,7 +408,7 @@ export class FactLog { } if (this.manifest.tailSegment === null) { await this.startTail(fact.generation) - } else if (this.tailBytes >= SEGMENT_ROTATE_BYTES) { + } else if (this.tailBytes >= this.rotateBytes) { await this.rotate(fact.generation) } const frame = encodeFrame(fact) diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 1983b4bc..73b8e859 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -851,11 +851,19 @@ export abstract class BaseStorage extends BaseStorageAdapter { // fact-log reader themselves — the log's open path is writer-side. // ========================================================================== - /** The host-wired fact-scan source (a closure over the live fact log). */ - private _factScanSource: (() => import('../db/factLog.js').FactLog | null) | null = null + /** The host-wired fact-scan source (closures over the live generation state). */ + private _factScanSource: { + factLog: () => import('../db/factLog.js').FactLog | null + committedGeneration: () => number + } | null = null /** HOST-ONLY: wire (or clear) the fact-scan capability's source. */ - public setFactScanSource(source: (() => import('../db/factLog.js').FactLog | null) | null): void { + public setFactScanSource( + source: { + factLog: () => import('../db/factLog.js').FactLog | null + committedGeneration: () => number + } | null + ): void { this._factScanSource = source } @@ -866,19 +874,29 @@ export abstract class BaseStorage extends BaseStorageAdapter { kinds?: Array<'noun' | 'verb'> batchSize?: number }): import('../db/factLog.js').FactScanHandle | null { - const log = this._factScanSource?.() + const log = this._factScanSource?.factLog() return log ? log.scanFacts(options) : null } /** The fact log's head generation, or `null` when no fact log exists. */ public factLogHeadGeneration(): number | null { - const log = this._factScanSource?.() + const log = this._factScanSource?.factLog() return log ? log.headGeneration() : null } + /** + * The COMMITTED generation watermark (the manifest truth) — the value a + * projection's `sourceGeneration` compares against. Exposed here so a + * provider never parses the store's private manifest format. `null` when + * the capability is unwired (no host, or a bare adapter). + */ + public committedGeneration(): number | null { + return this._factScanSource ? this._factScanSource.committedGeneration() : null + } + /** Sealed fact-segment paths (zero-copy handoff); empty when none. */ public factSegmentPaths(options?: { fromGeneration?: number }): string[] { - const log = this._factScanSource?.() + const log = this._factScanSource?.factLog() return log ? log.segmentPaths(options) : [] } diff --git a/tests/integration/fact-log-contracts.test.ts b/tests/integration/fact-log-contracts.test.ts new file mode 100644 index 00000000..eb579da9 --- /dev/null +++ b/tests/integration/fact-log-contracts.test.ts @@ -0,0 +1,125 @@ +/** + * @module tests/integration/fact-log-contracts + * @description Pinned durability + stability contracts for the fact log. + * + * (1) FSYNC-BEFORE-ACK: an acknowledged write's fact survives an abrupt + * process end (no flush, no close — reopen from disk). + * - transact(): HOLDS TODAY — the fact is fsync'd before transact returns. + * - single-op: PINNED AS `it.fails` — today's group-commit batches + * DURABILITY (ack precedes the group fsync; a hard kill loses the fact + * AND the generation together, coherently — the documented Model-B + * contract, fine while the tree is authoritative). The destination + * (ack-at-log) requires group commit to become LATENCY batching: the + * ack waits for the shared fsync. When that lands, this pin flips red — + * remove `.fails` and the contract is permanent. No cliff to discover. + * + * (2) SCAN STABILITY UNDER ROTATION: a scan handle opened before segment + * rotation yields exactly its snapshot — byte-identical facts, no gaps, + * no duplicates, and no bleed-in of facts appended after the snapshot. + * (The reclaim-during-scan variant lands with fact-log compaction, which + * does not exist yet — segments only rotate today, never reclaim.) + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, type CommitFact } from '../../src/index.js' +import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js' +import { FactLog, type FactLogStorage } from '../../src/db/factLog.js' + +describe('fsync-before-ack contract (fact durability at the ack boundary)', () => { + let dir: string + let brain: any + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(async () => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factack-')) + brain = await open() + }) + afterEach(async () => { + await brain.close?.().catch(() => {}) + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('transact(): the fact is durable the moment the ack returns (kill-after-ack safe)', async () => { + const receipt = await brain.transact([ + { op: 'add', type: 'document', metadata: { durable: 1 }, data: 'ack-at-commit' } + ]) + // Abrupt end: no flush(), no close() — a new instance reads only disk. + brain = await open() + const facts: CommitFact[] = [] + for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts) + expect(facts.some((f) => f.generation === receipt.generation)).toBe(true) + }) + + // PINNED (flips red when group commit becomes latency batching — then + // remove `.fails` and the ack-at-log contract is permanent on every path). + it.fails('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => { + await brain.add({ data: 'acked single-op', type: 'document', metadata: { n: 1 } }) + const ackedHead = brain.scanFacts()!.headGeneration + // Abrupt end immediately after the ack — before any flush window. + brain = await open() + const facts: CommitFact[] = [] + for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts) + expect(facts.some((f) => f.generation === ackedHead)).toBe(true) + }) +}) + +describe('scan stability under rotation (the snapshot contract)', () => { + const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}` + const fact = (generation: number): CommitFact => ({ + generation, + timestamp: 1_700_000_000_000 + generation, + ops: [ + { + kind: 'noun', + id: UUID(generation), + // Padding makes each frame ~1KB so a small rotateBytes forces rotations. + record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation }, vector: null } + } + ] + }) + + it('a scan opened before rotations yields its exact snapshot — no gaps, dups, or bleed-in', async () => { + const mem: any = new MemoryStorage() + await mem.init() + const log = new FactLog(mem as FactLogStorage, { rotateBytes: 4096 }) // ~4 facts per segment + await log.open(0) + for (let g = 1; g <= 10; g++) await log.append(fact(g)) + await log.sync() + + // Open the snapshot, THEN keep appending — forcing further rotations. + const scan = log.scanFacts() + expect(scan.headGeneration).toBe(10) + for (let g = 11; g <= 25; g++) await log.append(fact(g)) + await log.sync() + expect(log.headGeneration()).toBe(25) + + const seen: number[] = [] + for await (const batch of scan.batches()) { + for (const f of batch.facts) seen.push(f.generation) + } + // Exactly the snapshot: 1..10 in order, nothing appended-after bleeds in. + expect(seen).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + expect(scan.summary().factsYielded).toBe(10) + + // And a fresh scan sees everything, across all rotated segments. + const all: number[] = [] + for await (const batch of log.scanFacts().batches()) { + for (const f of batch.facts) all.push(f.generation) + } + expect(all).toEqual(Array.from({ length: 25 }, (_, i) => i + 1)) + expect(log.segmentPaths().length).toBeGreaterThanOrEqual(2) // rotations actually happened + }) +})