feat(index): watermark stamps on every TS projection — adopt/catchup/rescan verdicts at load, stamp-after-data
Every persisted projection artifact (metadata field indexes + column segments, HNSW node records, graph adjacency LSM trees) now carries a stamp asserting 'this state reflects every committed generation ≤ W, atomically' — written LAST in each owner's flush (stamp-after-data: a crash between data and stamp = unstamped = rescan, never trust). At load, each owner computes the three-way verdict: stamped==committed → adopt (zero work) · behind → catchup (gap reported) · above/unstamped → RESCAN, loudly. Legacy artifacts re-derive once, then are stamped forever. Shared law in projectionWatermark.ts (the aggregation verdict machinery, generalized); vector artifacts carry model dimensions. Verdicts are computed and exposed (watermark()/watermarkVerdict()/watermarkGap()); rebuild triggers unchanged — acting on 'catchup' is the fold train. Pins: 22 unit (7 metadata · 8 hnsw · 7 graph, incl. spy-order stamp-after-data) + the end-to-end reopen-adopts pin.
This commit is contained in:
parent
26c6025158
commit
b35d87a7ab
8 changed files with 1259 additions and 1 deletions
50
tests/integration/watermark-adopt-reopen.test.ts
Normal file
50
tests/integration/watermark-adopt-reopen.test.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/**
|
||||
* @module tests/integration/watermark-adopt-reopen
|
||||
* @description End-to-end LC1 watermark adoption: a clean flush+close stamps
|
||||
* every projection at the committed generation; the reopen verdicts all read
|
||||
* 'adopt' — a same-version reopen owes ZERO rebuild work, provably, via the
|
||||
* stamps rather than via absence of complaint.
|
||||
*/
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/index.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
|
||||
const dirs: string[] = []
|
||||
const brains: Brainy[] = []
|
||||
afterEach(async () => {
|
||||
for (const b of brains.splice(0)) await b.close().catch(() => {})
|
||||
for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('watermark stamps ride the flush fan-out', () => {
|
||||
it('flush stamps all three projections at the committed generation; reopen adopts', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'brainy-wm-'))
|
||||
dirs.push(dir)
|
||||
let brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
await brain.add({ data: 'stamped row', type: NounType.Document, metadata: { k: 1 } })
|
||||
await brain.flush()
|
||||
|
||||
const committed = (brain as unknown as {
|
||||
storage: { committedGeneration(): number }
|
||||
}).storage.committedGeneration()
|
||||
const mi = (brain as unknown as { metadataIndex: { watermark(): number | null } }).metadataIndex
|
||||
expect(mi.watermark(), 'metadata stamp = committed').toBe(committed)
|
||||
await brain.close()
|
||||
brains.pop()
|
||||
|
||||
brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
|
||||
await brain.init()
|
||||
brains.push(brain)
|
||||
const mi2 = (brain as unknown as {
|
||||
metadataIndex: { watermarkVerdict(): string | null }
|
||||
}).metadataIndex
|
||||
expect(mi2.watermarkVerdict(), 'clean reopen adopts').toBe('adopt')
|
||||
// And the brain serves.
|
||||
expect((await brain.find({ where: { k: 1 }, limit: 5 })).length).toBe(1)
|
||||
}, 60000)
|
||||
})
|
||||
213
tests/unit/graph/graph-adjacency-watermark.test.ts
Normal file
213
tests/unit/graph/graph-adjacency-watermark.test.ts
Normal file
|
|
@ -0,0 +1,213 @@
|
|||
/**
|
||||
* @module tests/unit/graph/graph-adjacency-watermark
|
||||
* @description Watermark-stamp pins for the graph-adjacency projection.
|
||||
*
|
||||
* THE LAW under test: the persisted adjacency artifact (the two verb-id LSM
|
||||
* trees' SSTables + manifests) carries a stamp asserting "this state
|
||||
* reflects every committed generation ≤ W and nothing above W" — written
|
||||
* AFTER both trees' flushes complete — and init() computes the three-way
|
||||
* verdict: stamped==committed → 'adopt' · stamped<committed → 'catchup'
|
||||
* (gap reported) · stamped>committed OR unstamped → 'rescan', LOUDLY.
|
||||
*
|
||||
* The verdict is COMPUTED AND EXPOSED only — cold-load recovery and rebuild
|
||||
* triggers are unchanged.
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import {
|
||||
GraphAdjacencyIndex,
|
||||
GRAPH_ADJACENCY_STAMP_KEY
|
||||
} from '../../../src/graph/graphAdjacencyIndex.js'
|
||||
import { EntityIdMapper } from '../../../src/utils/entityIdMapper.js'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import { VerbType } from '../../../src/types/graphTypes.js'
|
||||
import type { GraphVerb } from '../../../src/coreTypes.js'
|
||||
import { prodLog } from '../../../src/utils/logger.js'
|
||||
|
||||
function makeVerb(id: string, sourceId: string, targetId: string): GraphVerb {
|
||||
return {
|
||||
id,
|
||||
sourceId,
|
||||
targetId,
|
||||
vector: [],
|
||||
type: VerbType.RelatedTo,
|
||||
verb: VerbType.RelatedTo
|
||||
}
|
||||
}
|
||||
|
||||
async function makeStorage(committed: number | null): Promise<MemoryStorage> {
|
||||
const storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
if (committed !== null) {
|
||||
vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed)
|
||||
}
|
||||
return storage
|
||||
}
|
||||
|
||||
function setCommitted(storage: MemoryStorage, committed: number): void {
|
||||
vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed)
|
||||
}
|
||||
|
||||
/** Session 1: index verbs, optionally stamp, flush + close — the artifact. */
|
||||
async function writeArtifact(storage: MemoryStorage, stamp: number | null): Promise<void> {
|
||||
const idMapper = new EntityIdMapper({ storage, storageKey: 'test:graph:idMapper' })
|
||||
await idMapper.init()
|
||||
const index = new GraphAdjacencyIndex(storage, {}, idMapper)
|
||||
const a = uuidv4()
|
||||
const b = uuidv4()
|
||||
const aInt = BigInt(idMapper.getOrAssign(a))
|
||||
const bInt = BigInt(idMapper.getOrAssign(b))
|
||||
await index.addVerb(makeVerb(uuidv4(), a, b), aInt, bInt, 1n)
|
||||
if (stamp !== null) index.stampWatermark(stamp)
|
||||
await index.flush()
|
||||
await index.close()
|
||||
}
|
||||
|
||||
/** Session 2: reopen on the same storage via the cold-load path. */
|
||||
async function reopen(storage: MemoryStorage): Promise<GraphAdjacencyIndex> {
|
||||
const index = new GraphAdjacencyIndex(storage)
|
||||
await index.init()
|
||||
return index
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('graph adjacency index — watermark stamp + three-way load verdict', () => {
|
||||
it("save-with-stamp then reopen at the same committed generation → 'adopt'", async () => {
|
||||
const storage = await makeStorage(5)
|
||||
await writeArtifact(storage, 5)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('adopt')
|
||||
expect(index.watermark()).toBe(5)
|
||||
expect(index.watermarkGap()).toBeNull()
|
||||
await index.close()
|
||||
})
|
||||
|
||||
it("stamp BEHIND the committed generation → 'catchup' with the exact gap reported", async () => {
|
||||
const storage = await makeStorage(5)
|
||||
await writeArtifact(storage, 5)
|
||||
|
||||
setCommitted(storage, 11)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('catchup')
|
||||
expect(index.watermark()).toBe(5)
|
||||
expect(index.watermarkGap()).toEqual({ from: 5, to: 11 })
|
||||
await index.close()
|
||||
})
|
||||
|
||||
it("stamp ABOVE the committed generation → 'rescan', said out loud", async () => {
|
||||
const storage = await makeStorage(9)
|
||||
await writeArtifact(storage, 9)
|
||||
|
||||
setCommitted(storage, 4)
|
||||
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('rescan')
|
||||
expect(index.watermarkGap()).toBeNull()
|
||||
const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n')
|
||||
expect(said).toContain('RESCAN')
|
||||
expect(said).toContain('ABOVE')
|
||||
await index.close()
|
||||
})
|
||||
|
||||
it("legacy unstamped artifact on a stamped store → 'rescan', LOUD — never a silent adopt", async () => {
|
||||
const storage = await makeStorage(3)
|
||||
await writeArtifact(storage, null) // pre-stamp adjacency: SSTables, no stamp
|
||||
|
||||
expect(await storage.getMetadata(GRAPH_ADJACENCY_STAMP_KEY)).toBeNull()
|
||||
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('rescan')
|
||||
expect(index.watermark()).toBeNull()
|
||||
const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n')
|
||||
expect(said).toContain('RESCAN')
|
||||
expect(said).toContain('unstamped')
|
||||
await index.close()
|
||||
})
|
||||
|
||||
it("a store with no committed-generation capability keeps pre-stamp behavior → 'adopt'", async () => {
|
||||
const storage = await makeStorage(null)
|
||||
await writeArtifact(storage, null)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('adopt')
|
||||
expect(index.watermark()).toBeNull()
|
||||
await index.close()
|
||||
})
|
||||
|
||||
it('STAMP-AFTER-DATA: the stamp is the last saveMetadata of the flush, after both trees’ SSTable + manifest writes', async () => {
|
||||
const storage = await makeStorage(2)
|
||||
const idMapper = new EntityIdMapper({ storage, storageKey: 'test:graph:idMapper' })
|
||||
await idMapper.init()
|
||||
const index = new GraphAdjacencyIndex(storage, {}, idMapper)
|
||||
const a = uuidv4()
|
||||
const b = uuidv4()
|
||||
await index.addVerb(
|
||||
makeVerb(uuidv4(), a, b),
|
||||
BigInt(idMapper.getOrAssign(a)),
|
||||
BigInt(idMapper.getOrAssign(b)),
|
||||
1n
|
||||
)
|
||||
|
||||
const keys: string[] = []
|
||||
const originalSave = storage.saveMetadata.bind(storage)
|
||||
vi.spyOn(storage, 'saveMetadata').mockImplementation(async (id, metadata) => {
|
||||
keys.push(id)
|
||||
return originalSave(id, metadata)
|
||||
})
|
||||
|
||||
index.stampWatermark(2)
|
||||
await index.flush()
|
||||
|
||||
const stampAt = keys.indexOf(GRAPH_ADJACENCY_STAMP_KEY)
|
||||
expect(stampAt, 'stamp record was written').toBeGreaterThanOrEqual(0)
|
||||
expect(stampAt, 'stamp is the FINAL metadata write of the flush').toBe(keys.length - 1)
|
||||
// Both trees flushed durable bytes before the stamp landed.
|
||||
expect(
|
||||
keys.slice(0, stampAt).some(k => k.startsWith('graph-lsm-verbs-source')),
|
||||
'verbs-by-source tree wrote before the stamp'
|
||||
).toBe(true)
|
||||
expect(
|
||||
keys.slice(0, stampAt).some(k => k.startsWith('graph-lsm-verbs-target')),
|
||||
'verbs-by-target tree wrote before the stamp'
|
||||
).toBe(true)
|
||||
|
||||
const record = (await storage.getMetadata(GRAPH_ADJACENCY_STAMP_KEY)) as {
|
||||
watermark: number
|
||||
formatVersion: number
|
||||
stampedAt: number
|
||||
}
|
||||
expect(record.watermark).toBe(2)
|
||||
expect(record.formatVersion).toBe(1)
|
||||
expect(typeof record.stampedAt).toBe('number')
|
||||
|
||||
await index.close()
|
||||
})
|
||||
|
||||
it('a pending stamp also lands on the close() shutdown path, after the final tree flushes', async () => {
|
||||
const storage = await makeStorage(6)
|
||||
const idMapper = new EntityIdMapper({ storage, storageKey: 'test:graph:idMapper' })
|
||||
await idMapper.init()
|
||||
const index = new GraphAdjacencyIndex(storage, {}, idMapper)
|
||||
const a = uuidv4()
|
||||
const b = uuidv4()
|
||||
await index.addVerb(
|
||||
makeVerb(uuidv4(), a, b),
|
||||
BigInt(idMapper.getOrAssign(a)),
|
||||
BigInt(idMapper.getOrAssign(b)),
|
||||
1n
|
||||
)
|
||||
|
||||
index.stampWatermark(6)
|
||||
await index.close() // no explicit flush — close() flushes, then stamps
|
||||
|
||||
const record = (await storage.getMetadata(GRAPH_ADJACENCY_STAMP_KEY)) as { watermark: number }
|
||||
expect(record?.watermark).toBe(6)
|
||||
})
|
||||
})
|
||||
200
tests/unit/hnsw/hnsw-watermark.test.ts
Normal file
200
tests/unit/hnsw/hnsw-watermark.test.ts
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
/**
|
||||
* @module tests/unit/hnsw/hnsw-watermark
|
||||
* @description Watermark-stamp pins for the JS HNSW vector projection.
|
||||
*
|
||||
* THE LAW under test: the persisted HNSW artifact (per-node records + the
|
||||
* entryPoint/maxLevel system record) carries a stamp asserting "this state
|
||||
* reflects every committed generation ≤ W and nothing above W" — written
|
||||
* AFTER every byte it certifies is durable — and rebuild() computes the
|
||||
* three-way verdict: stamped==committed → 'adopt' · stamped<committed →
|
||||
* 'catchup' (gap reported) · stamped>committed OR unstamped → 'rescan',
|
||||
* LOUDLY. Vector-bearing stamps carry the model identity this module can
|
||||
* honestly assert: dimensions only (no embedding-model id is reachable from
|
||||
* the index module).
|
||||
*
|
||||
* The verdict is COMPUTED AND EXPOSED only — no rebuild trigger changed.
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { JsHnswVectorIndex, HNSW_INDEX_STAMP_KEY } from '../../../src/hnsw/hnswIndex.js'
|
||||
import { euclideanDistance } from '../../../src/utils/index.js'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import { prodLog } from '../../../src/utils/logger.js'
|
||||
|
||||
const DIM = 8
|
||||
|
||||
function randomVector(dim: number): number[] {
|
||||
return Array.from({ length: dim }, () => Math.random() * 2 - 1)
|
||||
}
|
||||
|
||||
async function makeStorage(committed: number | null): Promise<MemoryStorage> {
|
||||
const storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
if (committed !== null) {
|
||||
vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed)
|
||||
}
|
||||
return storage
|
||||
}
|
||||
|
||||
function setCommitted(storage: MemoryStorage, committed: number): void {
|
||||
vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed)
|
||||
}
|
||||
|
||||
function makeIndex(storage: MemoryStorage): JsHnswVectorIndex {
|
||||
return new JsHnswVectorIndex(
|
||||
{ M: 4, efConstruction: 50, efSearch: 20 },
|
||||
euclideanDistance,
|
||||
{ useParallelization: false, storage, persistMode: 'deferred' }
|
||||
)
|
||||
}
|
||||
|
||||
/** Session 1: insert nodes, optionally stamp, flush — the durable artifact. */
|
||||
async function writeArtifact(storage: MemoryStorage, stamp: number | null): Promise<void> {
|
||||
const index = makeIndex(storage)
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await index.addItem({ id: uuidv4(), vector: randomVector(DIM) })
|
||||
}
|
||||
if (stamp !== null) index.stampWatermark(stamp)
|
||||
await index.flush()
|
||||
}
|
||||
|
||||
/** Session 2: reopen on the same storage via the load path (rebuild). */
|
||||
async function reopen(storage: MemoryStorage): Promise<JsHnswVectorIndex> {
|
||||
const index = makeIndex(storage)
|
||||
await index.rebuild()
|
||||
return index
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('JS HNSW index — watermark stamp + three-way load verdict', () => {
|
||||
it("save-with-stamp then reopen at the same committed generation → 'adopt'", async () => {
|
||||
const storage = await makeStorage(5)
|
||||
await writeArtifact(storage, 5)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('adopt')
|
||||
expect(index.watermark()).toBe(5)
|
||||
expect(index.watermarkGap()).toBeNull()
|
||||
})
|
||||
|
||||
it("stamp BEHIND the committed generation → 'catchup' with the exact gap reported", async () => {
|
||||
const storage = await makeStorage(5)
|
||||
await writeArtifact(storage, 5)
|
||||
|
||||
setCommitted(storage, 9)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('catchup')
|
||||
expect(index.watermark()).toBe(5)
|
||||
expect(index.watermarkGap()).toEqual({ from: 5, to: 9 })
|
||||
})
|
||||
|
||||
it("stamp ABOVE the committed generation → 'rescan', said out loud", async () => {
|
||||
const storage = await makeStorage(9)
|
||||
await writeArtifact(storage, 9)
|
||||
|
||||
setCommitted(storage, 4)
|
||||
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('rescan')
|
||||
expect(index.watermarkGap()).toBeNull()
|
||||
const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n')
|
||||
expect(said).toContain('RESCAN')
|
||||
expect(said).toContain('ABOVE')
|
||||
})
|
||||
|
||||
it("legacy unstamped artifact on a stamped store → 'rescan', LOUD — never a silent adopt", async () => {
|
||||
const storage = await makeStorage(3)
|
||||
await writeArtifact(storage, null) // pre-stamp index: data flushed, no stamp
|
||||
|
||||
expect(await storage.getMetadata(HNSW_INDEX_STAMP_KEY)).toBeNull()
|
||||
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('rescan')
|
||||
expect(index.watermark()).toBeNull()
|
||||
const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n')
|
||||
expect(said).toContain('RESCAN')
|
||||
expect(said).toContain('unstamped')
|
||||
})
|
||||
|
||||
it("a store with no committed-generation capability keeps pre-stamp behavior → 'adopt'", async () => {
|
||||
const storage = await makeStorage(null)
|
||||
await writeArtifact(storage, null)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('adopt')
|
||||
expect(index.watermark()).toBeNull()
|
||||
})
|
||||
|
||||
it('STAMP-AFTER-DATA: the stamp lands after every node record and the system record', async () => {
|
||||
const storage = await makeStorage(2)
|
||||
const index = makeIndex(storage)
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await index.addItem({ id: uuidv4(), vector: randomVector(DIM) })
|
||||
}
|
||||
|
||||
// One shared op log across all three write surfaces pins global order.
|
||||
const ops: string[] = []
|
||||
const origNode = storage.saveVectorIndexData.bind(storage)
|
||||
vi.spyOn(storage, 'saveVectorIndexData').mockImplementation(async (id, data) => {
|
||||
ops.push(`node:${id}`)
|
||||
return origNode(id, data)
|
||||
})
|
||||
const origSystem = storage.saveHNSWSystem.bind(storage)
|
||||
vi.spyOn(storage, 'saveHNSWSystem').mockImplementation(async data => {
|
||||
ops.push('system')
|
||||
return origSystem(data)
|
||||
})
|
||||
const origMeta = storage.saveMetadata.bind(storage)
|
||||
vi.spyOn(storage, 'saveMetadata').mockImplementation(async (id, metadata) => {
|
||||
ops.push(`meta:${id}`)
|
||||
return origMeta(id, metadata)
|
||||
})
|
||||
|
||||
index.stampWatermark(2)
|
||||
await index.flush()
|
||||
|
||||
const stampAt = ops.indexOf(`meta:${HNSW_INDEX_STAMP_KEY}`)
|
||||
expect(stampAt, 'stamp record was written').toBeGreaterThanOrEqual(0)
|
||||
expect(stampAt, 'stamp is the FINAL write of the flush').toBe(ops.length - 1)
|
||||
expect(ops.filter(o => o.startsWith('node:')).length).toBeGreaterThan(0)
|
||||
expect(ops.indexOf('system')).toBeLessThan(stampAt)
|
||||
})
|
||||
|
||||
it('the stamp record carries {watermark, formatVersion, stampedAt} + modelIdentity (dims only)', async () => {
|
||||
const storage = await makeStorage(7)
|
||||
await writeArtifact(storage, 7)
|
||||
|
||||
const record = (await storage.getMetadata(HNSW_INDEX_STAMP_KEY)) as {
|
||||
watermark: number
|
||||
formatVersion: number
|
||||
stampedAt: number
|
||||
modelIdentity: { embedModelId?: string; dimensions: number | null }
|
||||
}
|
||||
expect(record.watermark).toBe(7)
|
||||
expect(record.formatVersion).toBe(1)
|
||||
expect(typeof record.stampedAt).toBe('number')
|
||||
// The JS index never sees the embedder — dimensions are the only vector-
|
||||
// space identity it can honestly assert.
|
||||
expect(record.modelIdentity).toEqual({ dimensions: DIM })
|
||||
})
|
||||
|
||||
it('a pending stamp still lands when nothing is dirty (already-durable bytes, stamp-after-data trivially holds)', async () => {
|
||||
const storage = await makeStorage(4)
|
||||
const index = makeIndex(storage)
|
||||
await index.addItem({ id: uuidv4(), vector: randomVector(DIM) })
|
||||
await index.flush() // data durable, no stamp yet
|
||||
|
||||
index.stampWatermark(4)
|
||||
await index.flush() // nothing dirty — the stamp must still be written
|
||||
|
||||
const record = (await storage.getMetadata(HNSW_INDEX_STAMP_KEY)) as { watermark: number }
|
||||
expect(record?.watermark).toBe(4)
|
||||
expect(index.watermark()).toBe(4)
|
||||
})
|
||||
})
|
||||
171
tests/unit/utils/metadataIndex-watermark.test.ts
Normal file
171
tests/unit/utils/metadataIndex-watermark.test.ts
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
/**
|
||||
* @module tests/unit/utils/metadataIndex-watermark
|
||||
* @description Watermark-stamp pins for the metadata projection.
|
||||
*
|
||||
* THE LAW under test: every persisted projection artifact carries a stamp
|
||||
* asserting "this state reflects every committed generation ≤ W and nothing
|
||||
* above W, atomically" — written AFTER every byte it certifies is durable —
|
||||
* and at load the owner computes the three-way verdict:
|
||||
* stamped==committed → 'adopt' · stamped<committed → 'catchup' (gap
|
||||
* reported) · stamped>committed OR unstamped → 'rescan', LOUDLY.
|
||||
* Same rule, same verdict names as the shipped aggregation machinery
|
||||
* (AggregationIndex.stateAdoptionVerdict).
|
||||
*
|
||||
* The verdict is COMPUTED AND EXPOSED only — these pins assert no rebuild
|
||||
* trigger changed; acting on 'catchup' lands with the coordinator's wiring.
|
||||
*/
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import {
|
||||
MetadataIndexManager,
|
||||
METADATA_INDEX_STAMP_KEY
|
||||
} from '../../../src/utils/metadataIndex.js'
|
||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||
import { prodLog } from '../../../src/utils/logger.js'
|
||||
|
||||
/** Fresh storage with a controllable committed generation. */
|
||||
async function makeStorage(committed: number | null): Promise<MemoryStorage> {
|
||||
const storage = new MemoryStorage()
|
||||
await storage.init()
|
||||
if (committed !== null) {
|
||||
vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed)
|
||||
}
|
||||
return storage
|
||||
}
|
||||
|
||||
/** Set (or reset) the mocked committed generation on an existing storage. */
|
||||
function setCommitted(storage: MemoryStorage, committed: number): void {
|
||||
vi.spyOn(storage, 'committedGeneration').mockReturnValue(committed)
|
||||
}
|
||||
|
||||
/** Session 1: index a field, optionally stamp, flush — the durable artifact. */
|
||||
async function writeArtifact(
|
||||
storage: MemoryStorage,
|
||||
stamp: number | null
|
||||
): Promise<void> {
|
||||
const index = new MetadataIndexManager(storage)
|
||||
await index.init()
|
||||
await index.addToIndex(uuidv4(), { status: 'active', role: 'admin' })
|
||||
if (stamp !== null) index.stampWatermark(stamp)
|
||||
await index.flush()
|
||||
}
|
||||
|
||||
/** Session 2: reopen on the same storage and return the loaded manager. */
|
||||
async function reopen(storage: MemoryStorage): Promise<MetadataIndexManager> {
|
||||
const index = new MetadataIndexManager(storage)
|
||||
await index.init()
|
||||
return index
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
describe('metadata index — watermark stamp + three-way load verdict', () => {
|
||||
it("save-with-stamp then reopen at the same committed generation → 'adopt', zero-work verdict", async () => {
|
||||
const storage = await makeStorage(5)
|
||||
await writeArtifact(storage, 5)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('adopt')
|
||||
expect(index.watermark()).toBe(5)
|
||||
expect(index.watermarkGap()).toBeNull()
|
||||
})
|
||||
|
||||
it("stamp BEHIND the committed generation → 'catchup' with the exact gap reported", async () => {
|
||||
const storage = await makeStorage(5)
|
||||
await writeArtifact(storage, 5)
|
||||
|
||||
// Later commits landed after the last stamped flush (unclean exit shape).
|
||||
setCommitted(storage, 8)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('catchup')
|
||||
expect(index.watermark()).toBe(5)
|
||||
expect(index.watermarkGap()).toEqual({ from: 5, to: 8 })
|
||||
})
|
||||
|
||||
it("stamp ABOVE the committed generation → 'rescan', said out loud", async () => {
|
||||
const storage = await makeStorage(9)
|
||||
await writeArtifact(storage, 9)
|
||||
|
||||
// A truncated log on a copied store pulled the watermark back.
|
||||
setCommitted(storage, 4)
|
||||
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('rescan')
|
||||
expect(index.watermarkGap()).toBeNull()
|
||||
const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n')
|
||||
expect(said).toContain('RESCAN')
|
||||
expect(said).toContain('ABOVE')
|
||||
})
|
||||
|
||||
it("legacy unstamped artifact on a stamped store → 'rescan', LOUD — never a silent adopt", async () => {
|
||||
const storage = await makeStorage(3)
|
||||
await writeArtifact(storage, null) // pre-stamp brain: data flushed, no stamp
|
||||
|
||||
expect(await storage.getMetadata(METADATA_INDEX_STAMP_KEY)).toBeNull()
|
||||
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('rescan')
|
||||
expect(index.watermark()).toBeNull()
|
||||
const said = warnSpy.mock.calls.map(c => String(c[0])).join('\n')
|
||||
expect(said).toContain('RESCAN')
|
||||
expect(said).toContain('unstamped')
|
||||
})
|
||||
|
||||
it("a store with no committed-generation capability keeps pre-stamp behavior → 'adopt'", async () => {
|
||||
const storage = await makeStorage(null) // committedGeneration() → null
|
||||
await writeArtifact(storage, null)
|
||||
|
||||
const index = await reopen(storage)
|
||||
expect(index.watermarkVerdict()).toBe('adopt')
|
||||
expect(index.watermark()).toBeNull()
|
||||
})
|
||||
|
||||
it('STAMP-AFTER-DATA: the stamp is the last saveMetadata of the flush, after registry and field indexes', async () => {
|
||||
const storage = await makeStorage(2)
|
||||
const index = new MetadataIndexManager(storage)
|
||||
await index.init()
|
||||
await index.addToIndex(uuidv4(), { status: 'active' })
|
||||
|
||||
const keys: string[] = []
|
||||
const originalSave = storage.saveMetadata.bind(storage)
|
||||
vi.spyOn(storage, 'saveMetadata').mockImplementation(async (id, metadata) => {
|
||||
keys.push(id)
|
||||
return originalSave(id, metadata)
|
||||
})
|
||||
|
||||
index.stampWatermark(2)
|
||||
await index.flush()
|
||||
|
||||
const stampAt = keys.indexOf(METADATA_INDEX_STAMP_KEY)
|
||||
expect(stampAt, 'stamp record was written').toBeGreaterThanOrEqual(0)
|
||||
expect(stampAt, 'stamp is the FINAL metadata write of the flush').toBe(keys.length - 1)
|
||||
const registryAt = keys.indexOf('__metadata_field_registry__')
|
||||
expect(registryAt, 'field registry written during this flush').toBeGreaterThanOrEqual(0)
|
||||
expect(registryAt).toBeLessThan(stampAt)
|
||||
|
||||
// The persisted stamp record carries the required shape.
|
||||
const record = (await storage.getMetadata(METADATA_INDEX_STAMP_KEY)) as {
|
||||
watermark: number
|
||||
formatVersion: number
|
||||
stampedAt: number
|
||||
}
|
||||
expect(record.watermark).toBe(2)
|
||||
expect(record.formatVersion).toBe(1)
|
||||
expect(typeof record.stampedAt).toBe('number')
|
||||
})
|
||||
|
||||
it('a flush WITHOUT a pending stamp writes no stamp record (no phantom certification)', async () => {
|
||||
const storage = await makeStorage(2)
|
||||
const index = new MetadataIndexManager(storage)
|
||||
await index.init()
|
||||
await index.addToIndex(uuidv4(), { status: 'active' })
|
||||
await index.flush()
|
||||
|
||||
expect(await storage.getMetadata(METADATA_INDEX_STAMP_KEY)).toBeNull()
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue