diff --git a/src/db/generationStore.ts b/src/db/generationStore.ts index 7439a025..065b3659 100644 --- a/src/db/generationStore.ts +++ b/src/db/generationStore.ts @@ -1394,6 +1394,10 @@ export class GenerationStore { // The transaction's entire canonical footprint is now durable, so the // counter/manifest advance below can never outrun the entity bytes. await this.storage.flushWriteBarrier?.() + // THE FENCE (transact leg): verify lock ownership before the commit + // point — an aborted-by-fence transact rolls back cleanly through the + // catch below; a fenced writer must never advance counter or manifest. + await this.storage.assertWriterFenceHeld?.() faultPoint('after-execute') // Fact log (dual-write): append + fsync this generation's AFTER-IMAGE @@ -1915,6 +1919,11 @@ export class GenerationStore { private async flushPendingSingleOpsUnlocked(): Promise { return this.withMutex(async () => { if (this.pendingGens.length === 0) return + // THE FENCE: an evicted writer (force-takeover, removed lock) must fail + // HERE, before a single staged byte or manifest advance — writing on + // after eviction is how split-brain stores are made. One small read + // per flush window. + await this.storage.assertWriterFenceHeld?.() this.clearPendingFlushTimer() const gens = [...this.pendingGens].sort((a, b) => a - b) diff --git a/src/db/types.ts b/src/db/types.ts index 866ca47f..363de086 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -485,6 +485,16 @@ export interface GenerationStorage { */ syncEntityCanonical?(nouns: string[], verbs: string[]): Promise + /** + * OPTIONAL writer fence: throw `BRAINY_WRITER_FENCED` when this instance + * no longer owns the store's writer lock (an operator force-takeover or a + * removed lock file). Called at every flush commit and transact barrier — + * one small read per commit window — so an evicted writer fails loudly on + * its next commit instead of split-braining the store. Adapters without a + * cross-process lock model omit it. + */ + assertWriterFenceHeld?(): Promise + /** Read an entity's raw stored metadata+vector objects. */ readNounRaw(id: string): Promise<{ metadata: any | null; vector: any | null }> /** Restore an entity's raw stored objects (`null` part ⇒ delete that file). */ diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 81c6e545..8fb2d2f1 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -1920,14 +1920,24 @@ export class FileSystemStorage extends BaseStorage { rootDir: this.rootDir } - // The atomic claim: create-exclusive, so exactly ONE racer wins. + // The atomic claim: write the FULL contents to a temp file, then + // hard-link it into place — link(2) fails EEXIST if the target exists, + // and the lock file appears with its complete JSON in one atomic step. + // (The previous claim was writeFile with O_EXCL, whose open→write→close + // is NOT atomic: a concurrent opener could read the file in its empty + // window, judge it torn, unlink a LIVE claim, and take the lock — two + // live writers. The link claim leaves no empty window to misread.) + const claimTmp = `${lockFile}.claim-${myPid}-${Date.now()}` try { - await fs.promises.writeFile(lockFile, JSON.stringify(info, null, 2), { flag: 'wx' }) + await fs.promises.writeFile(claimTmp, JSON.stringify(info, null, 2)) + await fs.promises.link(claimTmp, lockFile) } catch (err: any) { if (err.code === 'EEXIST') { continue // someone else claimed between our read and create — re-evaluate } throw err + } finally { + await fs.promises.unlink(claimTmp).catch(() => {}) } this.installWriterLock(info) @@ -1972,6 +1982,44 @@ export class FileSystemStorage extends BaseStorage { } } + /** + * THE FENCE: verify this instance still owns the writer lock before a + * commit barrier proceeds. An evicted writer (an operator's + * `{ force: true }` takeover, or an operator deleting the lock file) must + * fail LOUDLY on its next flush instead of writing on unaware — the + * unfenced evicted writer was half of a production split-brain (each + * writer flushing its own internally-consistent id-mapper snapshot, + * alternating the store between two truths). One small file read per + * flush window, never per record. No-op when this instance holds no + * writer lock (read-only opens, in-memory stores). + * + * @throws `BRAINY_WRITER_FENCED` when the lock is gone or held by another. + */ + public override async assertWriterFenceHeld(): Promise { + if (!this.writerLockInfo) return + const current = await this.readWriterLock() + if ( + current && + current.pid === this.writerLockInfo.pid && + current.hostname === this.writerLockInfo.hostname && + current.startedAt === this.writerLockInfo.startedAt + ) { + return + } + const err = new Error( + `Writer fence lost for ${this.rootDir}: this process (PID ${this.writerLockInfo.pid}) ` + + `no longer holds the writer lock — ` + + (current + ? `it is now held by PID ${current.pid} on ${current.hostname} (since ${current.startedAt}).` + : `the lock file is gone (released or removed by an operator).`) + + `\nThis instance refuses to commit further writes: a fenced-out writer continuing to ` + + `flush is how split-brain stores are made. Close this instance; if the takeover was a ` + + `mistake, close the successor and re-open.` + ) as Error & { code: string } + err.code = 'BRAINY_WRITER_FENCED' + throw err + } + /** The consumer-facing BRAINY_WRITER_LOCKED error, holder details attached. */ private writerLockedError(existing: WriterLockInfo): Error { const err = new Error( @@ -2060,18 +2108,25 @@ export class FileSystemStorage extends BaseStorage { /** * Determine whether an existing writer lock is stale (safe to overwrite). - * Same hostname and (dead PID OR heartbeat older than threshold) → stale. - * Different hostname → cannot prove stale, treat as live. + * Same hostname and DEAD PID → stale. That is the whole rule: a LIVE + * process is never auto-evicted, however old its heartbeat — a >60s + * event-loop stall (debugger pause, GC, heavy sync work) is a slow writer, + * not a dead one, and heartbeat-age eviction of live writers was the + * dominant mechanism behind a production split-brain (two live unaware + * writers alternating a store's id-mapper between two truths). A holder + * that LOOKS alive but is truly wedged is the operator's call via + * `{ force: true }` — and the fence check on every flush + * ({@link assertWriterFenceHeld}) guarantees a forced-out holder fails + * loudly instead of writing on. Different hostname → cannot prove + * anything, treat as live. The heartbeat remains for OBSERVABILITY (the + * lock error names it so an operator can judge staleness themselves). */ private async isWriterLockStale(lock: WriterLockInfo): Promise { const os = await import('node:os') if (lock.hostname !== os.hostname()) { return false } - const heartbeatAge = Date.now() - new Date(lock.lastHeartbeat).getTime() - const pidAlive = this.isPidAlive(lock.pid) - if (!pidAlive) return true - return heartbeatAge > FileSystemStorage.WRITER_STALE_THRESHOLD_MS + return !this.isPidAlive(lock.pid) } /** diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index c3b3b3bf..b65e938e 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -612,6 +612,18 @@ export abstract class BaseStorage extends BaseStorageAdapter { return null } + /** + * THE FENCE: verify this instance still owns its writer lock before a + * commit barrier proceeds; throw `BRAINY_WRITER_FENCED` if evicted. The + * default is a no-op — adapters without a cross-process lock model (memory, + * per-request cloud stores) have no eviction to fence against. The + * filesystem adapter overrides this; the generation store calls it at + * every flush commit and transact barrier. + */ + public async assertWriterFenceHeld(): Promise { + // No-op by default — no lock model, nothing to be evicted from. + } + /** * Start watching for cross-process flush requests. The writer Brainy * instance calls this so that out-of-process inspectors can ask for a diff --git a/tests/integration/writer-lock-fencing.test.ts b/tests/integration/writer-lock-fencing.test.ts new file mode 100644 index 00000000..86e79b99 --- /dev/null +++ b/tests/integration/writer-lock-fencing.test.ts @@ -0,0 +1,131 @@ +/** + * @module tests/integration/writer-lock-fencing + * @description The writer-lock fencing cures, from a production dev-store + * split-brain (two live writers alternating a store's id-mapper between two + * internally-consistent truths). Three laws, each pinned: + * + * 1. A LIVE writer is never auto-evicted — staleness requires PID-death. + * (The old rule evicted on heartbeat age alone, so a >60s event-loop + * stall — debugger, GC — handed the lock to a second opener while the + * first kept writing.) + * 2. A DEAD writer's lock still self-clears with narration (venue's ask). + * 3. THE FENCE: an evicted writer (force-takeover or removed lock) fails + * LOUDLY at its next commit barrier — typed BRAINY_WRITER_FENCED — and + * never advances the store. + */ +import { describe, it, expect, afterEach } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Brainy } from '../../src/index.js' +import { NounType } from '../../src/types/graphTypes.js' + +const dirs: string[] = [] +const brains: Brainy[] = [] +afterEach(async () => { + for (const b of brains.splice(0)) await b.close().catch(() => {}) + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) +}) + +function lockPath(dir: string): string { + return join(dir, 'locks', '_writer.lock') +} + +async function fsBrain(dir: string): Promise { + const brain = new Brainy({ + storage: { type: 'filesystem', path: dir }, + requireSubtype: false + }) + await brain.init() + brains.push(brain) + return brain +} + +describe('writer-lock fencing', () => { + it('a LIVE writer with an ancient heartbeat is NOT evicted — the second opener refuses typed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-live-')) + dirs.push(dir) + await fsBrain(dir) + + // Manufacture the trigger shape: a DIFFERENT process's lock (pid 1 — + // always alive, never ours, EPERM proves liveness) with a >60s-old + // heartbeat — the blocked-event-loop costume that used to get evicted. + const lp = lockPath(dir) + const lock = JSON.parse(fs.readFileSync(lp, 'utf-8')) + lock.pid = 1 + lock.lastHeartbeat = new Date(Date.now() - 10 * 60_000).toISOString() + fs.writeFileSync(lp, JSON.stringify(lock)) + + // Old rule: heartbeat-age eviction → silent takeover → split brain. + // New rule: live PID = live writer; the second opener throws typed. + const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false }) + await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' }) + }, 120000) + + it("a DEAD writer's lock self-clears and the new opener proceeds", async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-dead-')) + dirs.push(dir) + const first = await fsBrain(dir) + await first.close() + brains.pop() + + // Manufacture a crashed holder: a lock naming a PID that cannot exist. + fs.mkdirSync(join(dir, 'locks'), { recursive: true }) + fs.writeFileSync( + lockPath(dir), + JSON.stringify({ + pid: 2 ** 22 + 12345, // beyond pid_max on any default Linux + hostname: os.hostname(), + startedAt: new Date().toISOString(), + lastHeartbeat: new Date().toISOString(), + version: 'test', + rootDir: dir + }) + ) + const brain = await fsBrain(dir) // must not throw + const id = await brain.add({ data: 'post-takeover write', type: NounType.Document, metadata: {} }) + expect(await brain.get(id)).not.toBeNull() + }, 120000) + + it('THE FENCE: a forced-out writer fails its next flush typed and advances nothing', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-evict-')) + dirs.push(dir) + const victim = await fsBrain(dir) + await victim.add({ data: 'pre-eviction write', type: NounType.Document, metadata: { n: 1 } }) + await victim.flush() + const genBefore = victim.generation() + + // A successor takes the lock behind the victim's back (the force-takeover + // shape: different pid + startedAt). + fs.writeFileSync( + lockPath(dir), + JSON.stringify({ + pid: process.pid + 1, + hostname: os.hostname(), + startedAt: new Date(Date.now() + 1).toISOString(), + lastHeartbeat: new Date().toISOString(), + version: 'test-successor', + rootDir: dir + }) + ) + + // The victim's next commit barrier must refuse, typed — never write on. + await victim.add({ data: 'post-eviction write', type: NounType.Document, metadata: { n: 2 } }) + await expect(victim.flush()).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' }) + expect(victim.generation(), 'committed watermark never advanced past the fence') + .toBeGreaterThanOrEqual(genBefore) + + // Transact leg: the barrier fences there too, and rolls back cleanly. + await expect( + victim.transact([ + { op: 'add', id: '00000000-0000-7000-8000-0000000fence', type: NounType.Document, data: 'fenced', metadata: {} } + ]) + ).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' }) + + // Silence the fenced instance's close-time release (it no longer owns the lock). + brains.pop() + await victim.close().catch(() => {}) + }, 120000) +})