/** * @module tests/integration/writer-lock-fencing * @description The writer-lock fencing cures, from a production dev-store * split-brain (two live writers alternating a store's id-mapper between two * internally-consistent truths). Three laws, each pinned: * * 1. A LIVE writer is never auto-evicted — staleness requires PID-death. * (The old rule evicted on heartbeat age alone, so a >60s event-loop * stall — debugger, GC — handed the lock to a second opener while the * first kept writing.) * 2. A DEAD writer's lock still self-clears with narration (venue's ask). * 3. THE FENCE: an evicted writer (force-takeover or removed lock) fails * LOUDLY at its next commit barrier — typed BRAINY_WRITER_FENCED — and * never advances the store. */ import { describe, it, expect, afterEach } from 'vitest' import * as fs from 'node:fs' import * as os from 'node:os' import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Brainy } from '../../src/index.js' import { NounType } from '../../src/types/graphTypes.js' const dirs: string[] = [] const brains: Brainy[] = [] afterEach(async () => { for (const b of brains.splice(0)) await b.close().catch(() => {}) for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) }) function lockPath(dir: string): string { return join(dir, 'locks', '_writer.lock') } async function fsBrain(dir: string): Promise { const brain = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) await brain.init() brains.push(brain) return brain } describe('writer-lock fencing', () => { it('a LIVE writer with an ancient heartbeat is NOT evicted — the second opener refuses typed', async () => { const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-live-')) dirs.push(dir) await fsBrain(dir) // Manufacture the trigger shape: a DIFFERENT process's lock (pid 1 — // always alive, never ours, EPERM proves liveness) with a >60s-old // heartbeat — the blocked-event-loop costume that used to get evicted. const lp = lockPath(dir) const lock = JSON.parse(fs.readFileSync(lp, 'utf-8')) lock.pid = 1 lock.lastHeartbeat = new Date(Date.now() - 10 * 60_000).toISOString() fs.writeFileSync(lp, JSON.stringify(lock)) // Old rule: heartbeat-age eviction → silent takeover → split brain. // New rule: live PID = live writer; the second opener throws typed. const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' }) }, 120000) it("a DEAD writer's lock self-clears and the new opener proceeds", async () => { const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-dead-')) dirs.push(dir) const first = await fsBrain(dir) await first.close() brains.pop() // Manufacture a crashed holder: a lock naming a PID that cannot exist. fs.mkdirSync(join(dir, 'locks'), { recursive: true }) fs.writeFileSync( lockPath(dir), JSON.stringify({ pid: 2 ** 22 + 12345, // beyond pid_max on any default Linux hostname: os.hostname(), startedAt: new Date().toISOString(), lastHeartbeat: new Date().toISOString(), version: 'test', rootDir: dir }) ) const brain = await fsBrain(dir) // must not throw const id = await brain.add({ data: 'post-takeover write', type: NounType.Document, metadata: {} }) expect(await brain.get(id)).not.toBeNull() }, 120000) it('the fence does NOT fire on a same-process re-open — the documented warn-and-take-over contract stays benign', async () => { const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-samepid-')) dirs.push(dir) const first = await fsBrain(dir) await first.add({ data: 'first instance write', type: NounType.Document, metadata: { n: 1 } }) // A second instance in the SAME process takes the lock over (fresh // startedAt) — the pattern server-restart tests use. The first // instance's background flushes must keep working: same pid + same // hostname IS ownership. (The plant's integration lane caught the // startedAt-strict fence latching exactly this shape dead.) const second = await fsBrain(dir) await second.add({ data: 'second instance write', type: NounType.Document, metadata: { n: 2 } }) await expect(first.flush()).resolves.toBeUndefined() await expect(second.flush()).resolves.toBeUndefined() }, 120000) it('THE FENCE: a forced-out writer fails its next flush typed and advances nothing', async () => { const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-evict-')) dirs.push(dir) const victim = await fsBrain(dir) await victim.add({ data: 'pre-eviction write', type: NounType.Document, metadata: { n: 1 } }) await victim.flush() const genBefore = victim.generation() // A successor takes the lock behind the victim's back (the force-takeover // shape: different pid + startedAt). fs.writeFileSync( lockPath(dir), JSON.stringify({ pid: process.pid + 1, hostname: os.hostname(), startedAt: new Date(Date.now() + 1).toISOString(), lastHeartbeat: new Date().toISOString(), version: 'test-successor', rootDir: dir }) ) // The victim's next commit barrier must refuse, typed — never write on. await victim.add({ data: 'post-eviction write', type: NounType.Document, metadata: { n: 2 } }) await expect(victim.flush()).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' }) expect(victim.generation(), 'committed watermark never advanced past the fence') .toBeGreaterThanOrEqual(genBefore) // Transact leg: the barrier fences there too, and rolls back cleanly. await expect( victim.transact([ { op: 'add', id: '00000000-0000-7000-8000-0000000fence', type: NounType.Document, data: 'fenced', metadata: {} } ]) ).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' }) // Silence the fenced instance's close-time release (it no longer owns the lock). brains.pop() await victim.close().catch(() => {}) }, 120000) })