feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data
Every persisted projection artifact (metadata field indexes + column segments, HNSW node records, graph adjacency LSM trees) now carries a stamp asserting 'this state reflects every committed generation ≤ W, atomically' — written LAST in each owner's flush (stamp-after-data: a crash between data and stamp = unstamped = rescan, never trust). At load, each owner computes the three-way verdict: stamped==committed → adopt (zero work) · behind → catchup (gap reported) · above/unstamped → RESCAN, loudly. Legacy artifacts re-derive once, then are stamped forever. Shared law in projectionWatermark.ts (the aggregation verdict machinery, generalized); vector artifacts carry model dimensions. Verdicts are computed and exposed (watermark()/watermarkVerdict()/watermarkGap()); rebuild triggers unchanged — acting on 'catchup' is the fold train. Pins: 22 unit (7 metadata · 8 hnsw · 7 graph, incl. spy-order stamp-after-data) + the end-to-end reopen-adopts pin.
This commit is contained in:
parent
26c6025158
commit
b35d87a7ab
8 changed files with 1259 additions and 1 deletions
50
tests/integration/watermark-adopt-reopen.test.ts
Normal file
50
tests/integration/watermark-adopt-reopen.test.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* @module tests/integration/watermark-adopt-reopen
|
||||
* @description End-to-end LC1 watermark adoption: a clean flush+close stamps
|
||||
* every projection at the committed generation; the reopen verdicts all read
|
||||
* 'adopt' — a same-version reopen owes ZERO rebuild work, provably, via the
|
||||
* stamps rather than via absence of complaint.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
|
||||
const dirs: string[] = []
|
||||
const brains: Brainy[] = []
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('watermark stamps ride the flush fan-out', () => {
|
||||
it('flush stamps all three projections at the committed generation; reopen adopts', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-wm-'))
|
||||
dirs.push(dir)
|
||||
let brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
await brain.add({ data: 'stamped row', type: NounType.Document, metadata: { k: 1 } })
|
||||
await brain.flush()
|
||||
|
||||
const committed = (brain as unknown as {
|
||||
storage: { committedGeneration(): number }
|
||||
}).storage.committedGeneration()
|
||||
const mi = (brain as unknown as { metadataIndex: { watermark(): number | null } }).metadataIndex
|
||||
expect(mi.watermark(), 'metadata stamp = committed').toBe(committed)
|
||||
await brain.close()
|
||||
brains.pop()
|
||||
|
||||
brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
const mi2 = (brain as unknown as {
|
||||
metadataIndex: { watermarkVerdict(): string | null }
|
||||
}).metadataIndex
|
||||
expect(mi2.watermarkVerdict(), 'clean reopen adopts').toBe('adopt')
|
||||
// And the brain serves.
|
||||
expect((await brain.find({ where: { k: 1 }, limit: 5 })).length).toBe(1)
|
||||
}, 60000)
|
||||
})
|
||||
Reference in a new issue