feat: committedGeneration capability + pinned durability/stability contracts

- storage.committedGeneration(): the committed watermark exposed on the
  fact-scan capability, so a provider compares its stamp's
  sourceGeneration against the store's truth without parsing the
  private manifest format (the capability source now carries both the
  live fact log and the committed closure).
- Pinned contract tests: fsync-before-ack (transact passes today; the
  single-op path is pinned via it.fails as the documented target —
  group commit must become LATENCY batching, ack waiting on the shared
  fsync, never durability skipping; the pin flips red when that lands
  so there is no cliff to discover) and scan-stability-under-rotation
  (a scan snapshot yields exactly its facts — no gaps, duplicates, or
  bleed-in — while segments rotate beneath it; the reclaim-during-scan
  variant lands with fact compaction, which does not exist yet).
- FactLog accepts a rotateBytes option so tests exercise real rotation.
This commit is contained in:
David Snelling 2026-07-15 12:44:52 -07:00
parent e4f37cd32b
commit d1ecee10f0
6 changed files with 177 additions and 11 deletions

View file

@ -24,6 +24,14 @@ Small additive follow-up to 8.4.0, from the native accelerator's first consumpti
providers run literally the same verification function, never a synchronized copy. providers run literally the same verification function, never a synchronized copy.
- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a - **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a
per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass. per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass.
- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider
compares its stamp's `sourceGeneration` against the store's truth without parsing the private
manifest format.
- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for
`transact()` today; the single-op path is pinned as the documented future target — group commit
becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan
snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate
beneath it).
No behavior change for applications; all additions. No behavior change for applications; all additions.

View file

@ -1005,9 +1005,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
// so a provider's heal can switch from the enumeration walk to one // so a provider's heal can switch from the enumeration walk to one
// sequential fact scan whenever the log exists. // sequential fact scan whenever the log exists.
if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') { if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') {
;(this.storage as BaseStorage).setFactScanSource( ;(this.storage as BaseStorage).setFactScanSource({
() => this.generationStore?.getFactLog() ?? null factLog: () => this.generationStore?.getFactLog() ?? null,
) // The committed watermark, exposed as a capability so providers
// never parse the store's private manifest format.
committedGeneration: () => this.generationStore?.committedGeneration() ?? 0
})
} }
// Entity-tree stamp coherence: compare the stamped sourceGeneration + // Entity-tree stamp coherence: compare the stamped sourceGeneration +

View file

@ -1182,6 +1182,15 @@ export interface StorageAdapter {
*/ */
factLogHeadGeneration?(): number | null factLogHeadGeneration?(): number | null
/**
* @description OPTIONAL (rides the fact-scan capability): the COMMITTED
* generation watermark the manifest truth a projection's
* `sourceGeneration` compares against. Exposed as a capability so a
* provider never parses the store's private manifest format. `null` when
* the capability is unwired.
*/
committedGeneration?(): number | null
/** /**
* @description OPTIONAL (rides the fact-scan capability): the immutable, * @description OPTIONAL (rides the fact-scan capability): the immutable,
* sealed fact-segment file paths covering `fromGeneration` the zero-copy * sealed fact-segment file paths covering `fromGeneration` the zero-copy

View file

@ -294,6 +294,8 @@ function parseSegment(
*/ */
export class FactLog { export class FactLog {
private readonly storage: FactLogStorage private readonly storage: FactLogStorage
/** Rotation threshold (bytes); tests may lower it to exercise rotation. */
private readonly rotateBytes: number
private manifest: FactsManifest = { private manifest: FactsManifest = {
formatVersion: FACTS_FORMAT_VERSION, formatVersion: FACTS_FORMAT_VERSION,
segments: [], segments: [],
@ -309,8 +311,9 @@ export class FactLog {
/** Segment paths appended since the last sync (the fsync batch). */ /** Segment paths appended since the last sync (the fsync batch). */
private readonly dirtySegments = new Set<string>() private readonly dirtySegments = new Set<string>()
constructor(storage: FactLogStorage) { constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) {
this.storage = storage this.storage = storage
this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES
} }
/** The highest committed generation the log holds (0 = empty). */ /** The highest committed generation the log holds (0 = empty). */
@ -405,7 +408,7 @@ export class FactLog {
} }
if (this.manifest.tailSegment === null) { if (this.manifest.tailSegment === null) {
await this.startTail(fact.generation) await this.startTail(fact.generation)
} else if (this.tailBytes >= SEGMENT_ROTATE_BYTES) { } else if (this.tailBytes >= this.rotateBytes) {
await this.rotate(fact.generation) await this.rotate(fact.generation)
} }
const frame = encodeFrame(fact) const frame = encodeFrame(fact)

View file

@ -851,11 +851,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
// fact-log reader themselves — the log's open path is writer-side. // fact-log reader themselves — the log's open path is writer-side.
// ========================================================================== // ==========================================================================
/** The host-wired fact-scan source (a closure over the live fact log). */ /** The host-wired fact-scan source (closures over the live generation state). */
private _factScanSource: (() => import('../db/factLog.js').FactLog | null) | null = null private _factScanSource: {
factLog: () => import('../db/factLog.js').FactLog | null
committedGeneration: () => number
} | null = null
/** HOST-ONLY: wire (or clear) the fact-scan capability's source. */ /** HOST-ONLY: wire (or clear) the fact-scan capability's source. */
public setFactScanSource(source: (() => import('../db/factLog.js').FactLog | null) | null): void { public setFactScanSource(
source: {
factLog: () => import('../db/factLog.js').FactLog | null
committedGeneration: () => number
} | null
): void {
this._factScanSource = source this._factScanSource = source
} }
@ -866,19 +874,29 @@ export abstract class BaseStorage extends BaseStorageAdapter {
kinds?: Array<'noun' | 'verb'> kinds?: Array<'noun' | 'verb'>
batchSize?: number batchSize?: number
}): import('../db/factLog.js').FactScanHandle | null { }): import('../db/factLog.js').FactScanHandle | null {
const log = this._factScanSource?.() const log = this._factScanSource?.factLog()
return log ? log.scanFacts(options) : null return log ? log.scanFacts(options) : null
} }
/** The fact log's head generation, or `null` when no fact log exists. */ /** The fact log's head generation, or `null` when no fact log exists. */
public factLogHeadGeneration(): number | null { public factLogHeadGeneration(): number | null {
const log = this._factScanSource?.() const log = this._factScanSource?.factLog()
return log ? log.headGeneration() : null return log ? log.headGeneration() : null
} }
/**
* The COMMITTED generation watermark (the manifest truth) the value a
* projection's `sourceGeneration` compares against. Exposed here so a
* provider never parses the store's private manifest format. `null` when
* the capability is unwired (no host, or a bare adapter).
*/
public committedGeneration(): number | null {
return this._factScanSource ? this._factScanSource.committedGeneration() : null
}
/** Sealed fact-segment paths (zero-copy handoff); empty when none. */ /** Sealed fact-segment paths (zero-copy handoff); empty when none. */
public factSegmentPaths(options?: { fromGeneration?: number }): string[] { public factSegmentPaths(options?: { fromGeneration?: number }): string[] {
const log = this._factScanSource?.() const log = this._factScanSource?.factLog()
return log ? log.segmentPaths(options) : [] return log ? log.segmentPaths(options) : []
} }

View file

@ -0,0 +1,125 @@
/**
* @module tests/integration/fact-log-contracts
* @description Pinned durability + stability contracts for the fact log.
*
* (1) FSYNC-BEFORE-ACK: an acknowledged write's fact survives an abrupt
* process end (no flush, no close reopen from disk).
* - transact(): HOLDS TODAY the fact is fsync'd before transact returns.
* - single-op: PINNED AS `it.fails` today's group-commit batches
* DURABILITY (ack precedes the group fsync; a hard kill loses the fact
* AND the generation together, coherently the documented Model-B
* contract, fine while the tree is authoritative). The destination
* (ack-at-log) requires group commit to become LATENCY batching: the
* ack waits for the shared fsync. When that lands, this pin flips red
* remove `.fails` and the contract is permanent. No cliff to discover.
*
* (2) SCAN STABILITY UNDER ROTATION: a scan handle opened before segment
* rotation yields exactly its snapshot byte-identical facts, no gaps,
* no duplicates, and no bleed-in of facts appended after the snapshot.
* (The reclaim-during-scan variant lands with fact-log compaction, which
* does not exist yet segments only rotate today, never reclaim.)
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy, type CommitFact } from '../../src/index.js'
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
import { FactLog, type FactLogStorage } from '../../src/db/factLog.js'
describe('fsync-before-ack contract (fact durability at the ack boundary)', () => {
let dir: string
let brain: any
const open = async () => {
const b: any = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', path: dir },
silent: true,
dimensions: 384
})
await b.init()
return b
}
beforeEach(async () => {
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factack-'))
brain = await open()
})
afterEach(async () => {
await brain.close?.().catch(() => {})
fs.rmSync(dir, { recursive: true, force: true })
})
it('transact(): the fact is durable the moment the ack returns (kill-after-ack safe)', async () => {
const receipt = await brain.transact([
{ op: 'add', type: 'document', metadata: { durable: 1 }, data: 'ack-at-commit' }
])
// Abrupt end: no flush(), no close() — a new instance reads only disk.
brain = await open()
const facts: CommitFact[] = []
for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts)
expect(facts.some((f) => f.generation === receipt.generation)).toBe(true)
})
// PINNED (flips red when group commit becomes latency batching — then
// remove `.fails` and the ack-at-log contract is permanent on every path).
it.fails('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => {
await brain.add({ data: 'acked single-op', type: 'document', metadata: { n: 1 } })
const ackedHead = brain.scanFacts()!.headGeneration
// Abrupt end immediately after the ack — before any flush window.
brain = await open()
const facts: CommitFact[] = []
for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts)
expect(facts.some((f) => f.generation === ackedHead)).toBe(true)
})
})
describe('scan stability under rotation (the snapshot contract)', () => {
const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
const fact = (generation: number): CommitFact => ({
generation,
timestamp: 1_700_000_000_000 + generation,
ops: [
{
kind: 'noun',
id: UUID(generation),
// Padding makes each frame ~1KB so a small rotateBytes forces rotations.
record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation }, vector: null }
}
]
})
it('a scan opened before rotations yields its exact snapshot — no gaps, dups, or bleed-in', async () => {
const mem: any = new MemoryStorage()
await mem.init()
const log = new FactLog(mem as FactLogStorage, { rotateBytes: 4096 }) // ~4 facts per segment
await log.open(0)
for (let g = 1; g <= 10; g++) await log.append(fact(g))
await log.sync()
// Open the snapshot, THEN keep appending — forcing further rotations.
const scan = log.scanFacts()
expect(scan.headGeneration).toBe(10)
for (let g = 11; g <= 25; g++) await log.append(fact(g))
await log.sync()
expect(log.headGeneration()).toBe(25)
const seen: number[] = []
for await (const batch of scan.batches()) {
for (const f of batch.facts) seen.push(f.generation)
}
// Exactly the snapshot: 1..10 in order, nothing appended-after bleeds in.
expect(seen).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
expect(scan.summary().factsYielded).toBe(10)
// And a fresh scan sees everything, across all rotated segments.
const all: number[] = []
for await (const batch of log.scanFacts().batches()) {
for (const f of batch.facts) all.push(f.generation)
}
expect(all).toEqual(Array.from({ length: 25 }, (_, i) => i + 1))
expect(log.segmentPaths().length).toBeGreaterThanOrEqual(2) // rotations actually happened
})
})