feat: committedGeneration capability + pinned durability/stability contracts
- storage.committedGeneration(): the committed watermark exposed on the fact-scan capability, so a provider compares its stamp's sourceGeneration against the store's truth without parsing the private manifest format (the capability source now carries both the live fact log and the committed closure). - Pinned contract tests: fsync-before-ack (transact passes today; the single-op path is pinned via it.fails as the documented target — group commit must become LATENCY batching, ack waiting on the shared fsync, never durability skipping; the pin flips red when that lands so there is no cliff to discover) and scan-stability-under-rotation (a scan snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate beneath it; the reclaim-during-scan variant lands with fact compaction, which does not exist yet). - FactLog accepts a rotateBytes option so tests exercise real rotation.
This commit is contained in:
parent
e4f37cd32b
commit
d1ecee10f0
6 changed files with 177 additions and 11 deletions
|
|
@ -24,6 +24,14 @@ Small additive follow-up to 8.4.0, from the native accelerator's first consumpti
|
|||
providers run literally the same verification function, never a synchronized copy.
|
||||
- **Rollup stamp invariants accept strings** (`number | string`) — content fingerprints such as a
|
||||
per-tree SHA-256 are valid invariant values; a type mismatch reads as incoherence, never a pass.
|
||||
- **`storage.committedGeneration()`** — the committed watermark as a capability, so a provider
|
||||
compares its stamp's `sourceGeneration` against the store's truth without parsing the private
|
||||
manifest format.
|
||||
- **Two durability/stability contracts pinned in the suite:** fsync-before-ack (holds for
|
||||
`transact()` today; the single-op path is pinned as the documented future target — group commit
|
||||
becomes latency batching, never durability skipping) and scan-stability-under-rotation (a scan
|
||||
snapshot yields exactly its facts — no gaps, duplicates, or bleed-in — while segments rotate
|
||||
beneath it).
|
||||
|
||||
No behavior change for applications; all additions.
|
||||
|
||||
|
|
|
|||
|
|
@ -1005,9 +1005,12 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
// so a provider's heal can switch from the enumeration walk to one
|
||||
// sequential fact scan whenever the log exists.
|
||||
if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') {
|
||||
;(this.storage as BaseStorage).setFactScanSource(
|
||||
() => this.generationStore?.getFactLog() ?? null
|
||||
)
|
||||
;(this.storage as BaseStorage).setFactScanSource({
|
||||
factLog: () => this.generationStore?.getFactLog() ?? null,
|
||||
// The committed watermark, exposed as a capability so providers
|
||||
// never parse the store's private manifest format.
|
||||
committedGeneration: () => this.generationStore?.committedGeneration() ?? 0
|
||||
})
|
||||
}
|
||||
|
||||
// Entity-tree stamp coherence: compare the stamped sourceGeneration +
|
||||
|
|
|
|||
|
|
@ -1182,6 +1182,15 @@ export interface StorageAdapter {
|
|||
*/
|
||||
factLogHeadGeneration?(): number | null
|
||||
|
||||
/**
|
||||
* @description OPTIONAL (rides the fact-scan capability): the COMMITTED
|
||||
* generation watermark — the manifest truth a projection's
|
||||
* `sourceGeneration` compares against. Exposed as a capability so a
|
||||
* provider never parses the store's private manifest format. `null` when
|
||||
* the capability is unwired.
|
||||
*/
|
||||
committedGeneration?(): number | null
|
||||
|
||||
/**
|
||||
* @description OPTIONAL (rides the fact-scan capability): the immutable,
|
||||
* sealed fact-segment file paths covering `fromGeneration` — the zero-copy
|
||||
|
|
|
|||
|
|
@ -294,6 +294,8 @@ function parseSegment(
|
|||
*/
|
||||
export class FactLog {
|
||||
private readonly storage: FactLogStorage
|
||||
/** Rotation threshold (bytes); tests may lower it to exercise rotation. */
|
||||
private readonly rotateBytes: number
|
||||
private manifest: FactsManifest = {
|
||||
formatVersion: FACTS_FORMAT_VERSION,
|
||||
segments: [],
|
||||
|
|
@ -309,8 +311,9 @@ export class FactLog {
|
|||
/** Segment paths appended since the last sync (the fsync batch). */
|
||||
private readonly dirtySegments = new Set<string>()
|
||||
|
||||
constructor(storage: FactLogStorage) {
|
||||
constructor(storage: FactLogStorage, options?: { rotateBytes?: number }) {
|
||||
this.storage = storage
|
||||
this.rotateBytes = options?.rotateBytes ?? SEGMENT_ROTATE_BYTES
|
||||
}
|
||||
|
||||
/** The highest committed generation the log holds (0 = empty). */
|
||||
|
|
@ -405,7 +408,7 @@ export class FactLog {
|
|||
}
|
||||
if (this.manifest.tailSegment === null) {
|
||||
await this.startTail(fact.generation)
|
||||
} else if (this.tailBytes >= SEGMENT_ROTATE_BYTES) {
|
||||
} else if (this.tailBytes >= this.rotateBytes) {
|
||||
await this.rotate(fact.generation)
|
||||
}
|
||||
const frame = encodeFrame(fact)
|
||||
|
|
|
|||
|
|
@ -851,11 +851,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
// fact-log reader themselves — the log's open path is writer-side.
|
||||
// ==========================================================================
|
||||
|
||||
/** The host-wired fact-scan source (a closure over the live fact log). */
|
||||
private _factScanSource: (() => import('../db/factLog.js').FactLog | null) | null = null
|
||||
/** The host-wired fact-scan source (closures over the live generation state). */
|
||||
private _factScanSource: {
|
||||
factLog: () => import('../db/factLog.js').FactLog | null
|
||||
committedGeneration: () => number
|
||||
} | null = null
|
||||
|
||||
/** HOST-ONLY: wire (or clear) the fact-scan capability's source. */
|
||||
public setFactScanSource(source: (() => import('../db/factLog.js').FactLog | null) | null): void {
|
||||
public setFactScanSource(
|
||||
source: {
|
||||
factLog: () => import('../db/factLog.js').FactLog | null
|
||||
committedGeneration: () => number
|
||||
} | null
|
||||
): void {
|
||||
this._factScanSource = source
|
||||
}
|
||||
|
||||
|
|
@ -866,19 +874,29 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
|||
kinds?: Array<'noun' | 'verb'>
|
||||
batchSize?: number
|
||||
}): import('../db/factLog.js').FactScanHandle | null {
|
||||
const log = this._factScanSource?.()
|
||||
const log = this._factScanSource?.factLog()
|
||||
return log ? log.scanFacts(options) : null
|
||||
}
|
||||
|
||||
/** The fact log's head generation, or `null` when no fact log exists. */
|
||||
public factLogHeadGeneration(): number | null {
|
||||
const log = this._factScanSource?.()
|
||||
const log = this._factScanSource?.factLog()
|
||||
return log ? log.headGeneration() : null
|
||||
}
|
||||
|
||||
/**
|
||||
* The COMMITTED generation watermark (the manifest truth) — the value a
|
||||
* projection's `sourceGeneration` compares against. Exposed here so a
|
||||
* provider never parses the store's private manifest format. `null` when
|
||||
* the capability is unwired (no host, or a bare adapter).
|
||||
*/
|
||||
public committedGeneration(): number | null {
|
||||
return this._factScanSource ? this._factScanSource.committedGeneration() : null
|
||||
}
|
||||
|
||||
/** Sealed fact-segment paths (zero-copy handoff); empty when none. */
|
||||
public factSegmentPaths(options?: { fromGeneration?: number }): string[] {
|
||||
const log = this._factScanSource?.()
|
||||
const log = this._factScanSource?.factLog()
|
||||
return log ? log.segmentPaths(options) : []
|
||||
}
|
||||
|
||||
|
|
|
|||
125
tests/integration/fact-log-contracts.test.ts
Normal file
125
tests/integration/fact-log-contracts.test.ts
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
/**
|
||||
* @module tests/integration/fact-log-contracts
|
||||
* @description Pinned durability + stability contracts for the fact log.
|
||||
*
|
||||
* (1) FSYNC-BEFORE-ACK: an acknowledged write's fact survives an abrupt
|
||||
* process end (no flush, no close — reopen from disk).
|
||||
* - transact(): HOLDS TODAY — the fact is fsync'd before transact returns.
|
||||
* - single-op: PINNED AS `it.fails` — today's group-commit batches
|
||||
* DURABILITY (ack precedes the group fsync; a hard kill loses the fact
|
||||
* AND the generation together, coherently — the documented Model-B
|
||||
* contract, fine while the tree is authoritative). The destination
|
||||
* (ack-at-log) requires group commit to become LATENCY batching: the
|
||||
* ack waits for the shared fsync. When that lands, this pin flips red —
|
||||
* remove `.fails` and the contract is permanent. No cliff to discover.
|
||||
*
|
||||
* (2) SCAN STABILITY UNDER ROTATION: a scan handle opened before segment
|
||||
* rotation yields exactly its snapshot — byte-identical facts, no gaps,
|
||||
* no duplicates, and no bleed-in of facts appended after the snapshot.
|
||||
* (The reclaim-during-scan variant lands with fact-log compaction, which
|
||||
* does not exist yet — segments only rotate today, never reclaim.)
|
||||
*/
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import * as fs from 'node:fs'
|
||||
import * as os from 'node:os'
|
||||
import * as path from 'node:path'
|
||||
import { Brainy, type CommitFact } from '../../src/index.js'
|
||||
import { MemoryStorage } from '../../src/storage/adapters/memoryStorage.js'
|
||||
import { FactLog, type FactLogStorage } from '../../src/db/factLog.js'
|
||||
|
||||
describe('fsync-before-ack contract (fact durability at the ack boundary)', () => {
|
||||
let dir: string
|
||||
let brain: any
|
||||
|
||||
const open = async () => {
|
||||
const b: any = new Brainy({
|
||||
requireSubtype: false,
|
||||
storage: { type: 'filesystem', path: dir },
|
||||
silent: true,
|
||||
dimensions: 384
|
||||
})
|
||||
await b.init()
|
||||
return b
|
||||
}
|
||||
|
||||
beforeEach(async () => {
|
||||
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-factack-'))
|
||||
brain = await open()
|
||||
})
|
||||
afterEach(async () => {
|
||||
await brain.close?.().catch(() => {})
|
||||
fs.rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('transact(): the fact is durable the moment the ack returns (kill-after-ack safe)', async () => {
|
||||
const receipt = await brain.transact([
|
||||
{ op: 'add', type: 'document', metadata: { durable: 1 }, data: 'ack-at-commit' }
|
||||
])
|
||||
// Abrupt end: no flush(), no close() — a new instance reads only disk.
|
||||
brain = await open()
|
||||
const facts: CommitFact[] = []
|
||||
for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts)
|
||||
expect(facts.some((f) => f.generation === receipt.generation)).toBe(true)
|
||||
})
|
||||
|
||||
// PINNED (flips red when group commit becomes latency batching — then
|
||||
// remove `.fails` and the ack-at-log contract is permanent on every path).
|
||||
it.fails('single-op: the fact is durable the moment the ack returns (the ack-at-log target)', async () => {
|
||||
await brain.add({ data: 'acked single-op', type: 'document', metadata: { n: 1 } })
|
||||
const ackedHead = brain.scanFacts()!.headGeneration
|
||||
// Abrupt end immediately after the ack — before any flush window.
|
||||
brain = await open()
|
||||
const facts: CommitFact[] = []
|
||||
for await (const b of brain.scanFacts()!.batches()) facts.push(...b.facts)
|
||||
expect(facts.some((f) => f.generation === ackedHead)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('scan stability under rotation (the snapshot contract)', () => {
|
||||
const UUID = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, '0')}`
|
||||
const fact = (generation: number): CommitFact => ({
|
||||
generation,
|
||||
timestamp: 1_700_000_000_000 + generation,
|
||||
ops: [
|
||||
{
|
||||
kind: 'noun',
|
||||
id: UUID(generation),
|
||||
// Padding makes each frame ~1KB so a small rotateBytes forces rotations.
|
||||
record: { metadata: { noun: 'document', pad: 'x'.repeat(900), g: generation }, vector: null }
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
it('a scan opened before rotations yields its exact snapshot — no gaps, dups, or bleed-in', async () => {
|
||||
const mem: any = new MemoryStorage()
|
||||
await mem.init()
|
||||
const log = new FactLog(mem as FactLogStorage, { rotateBytes: 4096 }) // ~4 facts per segment
|
||||
await log.open(0)
|
||||
for (let g = 1; g <= 10; g++) await log.append(fact(g))
|
||||
await log.sync()
|
||||
|
||||
// Open the snapshot, THEN keep appending — forcing further rotations.
|
||||
const scan = log.scanFacts()
|
||||
expect(scan.headGeneration).toBe(10)
|
||||
for (let g = 11; g <= 25; g++) await log.append(fact(g))
|
||||
await log.sync()
|
||||
expect(log.headGeneration()).toBe(25)
|
||||
|
||||
const seen: number[] = []
|
||||
for await (const batch of scan.batches()) {
|
||||
for (const f of batch.facts) seen.push(f.generation)
|
||||
}
|
||||
// Exactly the snapshot: 1..10 in order, nothing appended-after bleeds in.
|
||||
expect(seen).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
|
||||
expect(scan.summary().factsYielded).toBe(10)
|
||||
|
||||
// And a fresh scan sees everything, across all rotated segments.
|
||||
const all: number[] = []
|
||||
for await (const batch of log.scanFacts().batches()) {
|
||||
for (const f of batch.facts) all.push(f.generation)
|
||||
}
|
||||
expect(all).toEqual(Array.from({ length: 25 }, (_, i) => i + 1))
|
||||
expect(log.segmentPaths().length).toBeGreaterThanOrEqual(2) // rotations actually happened
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue