/** * @module tests/integration/writer-lock-clean-close * @description THE CLEAN-CLOSE CONTRACT for the writer lock. * * A production restart made this lane necessary: a service stopped with exit * code 0, having awaited `close()` on every pooled brain, and its next boot * announced `[brainy] Overwriting stale writer lock … appears dead` for every * store it owned. "The pid is gone" is equally true of an orderly restart and * of a crash, so the message could not tell an operator which one they had. * * The contract pinned here: * 1. A completed close leaves NO lock file and DOES leave a clean-close * record; the next open says nothing about staleness. * 2. The next lock claim CONSUMES that record — it may never outlive the * lock generation it describes, or a later crash would read as clean. * 3. A close whose durable steps FAIL still releases the lock (and still * rethrows the failure). * 4. A killed process (SIGKILL, no close at all) leaves the lock behind with * NO record, and the next open says exactly that — crash, recovery ahead. * 5. A host application with its own SIGTERM handler is never force-exited * out from under its own shutdown by Brainy's handler. */ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs' import { spawn } from 'node:child_process' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Brainy } from '../../src/brainy.js' import { NounType } from '../../src/types/graphTypes.js' const REPO_ROOT = process.cwd() const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx') function makeTempDir(): string { return mkdtempSync(join(tmpdir(), 'brainy-clean-close-')) } /** * Write a child script to disk and start it under tsx. A file (not `tsx -e`) * because the eval form compiles to CommonJS, which has no top-level await. * The script imports Brainy by ABSOLUTE path, so its own dependency * resolution still happens from inside the repository. */ function startChild(dir: string, body: string): ReturnType { const scriptPath = join(dir, 'child-process.mts') writeFileSync(scriptPath, body) // `detached` puts the child in its own process GROUP: tsx runs the script in // a grandchild process, and only a group-wide signal reaches the process // that actually holds the writer lock. return spawn(TSX, [scriptPath], { cwd: REPO_ROOT, stdio: ['ignore', 'pipe', 'pipe'], detached: true }) } /** Capture every console.warn/error line emitted while `fn` runs. */ async function captureConsole(fn: () => Promise): Promise<{ result: T; lines: string[] }> { const lines: string[] = [] const origWarn = console.warn const origError = console.error const sink = (...args: unknown[]) => { lines.push(args.map((a) => String(a)).join(' ')) } console.warn = sink as typeof console.warn console.error = sink as typeof console.error try { const result = await fn() return { result, lines } } finally { console.warn = origWarn console.error = origError } } /** * Run a child process that opens `dir`, writes one row, prints `READY`, and * then waits forever. Resolves with the child once READY is seen. */ function spawnHoldingChild(dir: string): Promise<{ child: ReturnType output: () => string }> { const script = ` import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) await brain.init() await brain.add({ data: 'row from the child', type: 'concept' }) await brain.flush() console.log('READY') setInterval(() => {}, 1000) ` const child = startChild(dir, script) let out = '' child.stdout.on('data', (d) => { out += String(d) }) child.stderr.on('data', (d) => { out += String(d) }) return new Promise((resolvePromise, rejectPromise) => { const timer = setTimeout(() => rejectPromise(new Error(`child never became READY:\n${out}`)), 120_000) child.stdout.on('data', () => { if (out.includes('READY')) { clearTimeout(timer) resolvePromise({ child, output: () => out }) } }) child.on('exit', (code) => { clearTimeout(timer) if (!out.includes('READY')) rejectPromise(new Error(`child exited ${code} before READY:\n${out}`)) }) }) } describe('writer lock — the clean-close contract', () => { let dir: string let brain: Brainy | null = null beforeEach(() => { dir = makeTempDir() }) afterEach(async () => { if (brain) { try { await brain.close() } catch { /* may already be closed */ } brain = null } try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ } }) const lockPath = () => join(dir, 'locks', '_writer.lock') const recordPath = () => join(dir, 'locks', '_writer.close') it('a completed close leaves no lock, leaves a record, and the reopen is silent about staleness', async () => { brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) await brain.init() expect(existsSync(lockPath())).toBe(true) await brain.add({ data: 'seed entity', type: NounType.Concept }) await brain.flush() await brain.close() brain = null // 1. The lock is gone and the release is RECORDED. expect(existsSync(lockPath())).toBe(false) expect(existsSync(recordPath())).toBe(true) const record = JSON.parse(readFileSync(recordPath(), 'utf-8')) expect(record.pid).toBe(process.pid) expect(typeof record.closedAt).toBe('string') expect(typeof record.startedAt).toBe('string') // 2. The reopen says nothing about a stale lock. const { result: reopened, lines } = await captureConsole(async () => { const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) await next.init() return next }) brain = reopened expect(lines.filter((l) => /stale writer lock|appears dead/i.test(l))).toEqual([]) // 3. The claim CONSUMED the record — it must not outlive its lock generation. expect(existsSync(recordPath())).toBe(false) expect(existsSync(lockPath())).toBe(true) }, 120_000) it('releases the writer lock even when a durable close step fails — and still rethrows', async () => { brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) await brain.init() await brain.add({ data: 'seed entity', type: NounType.Concept }) await brain.flush() expect(existsSync(lockPath())).toBe(true) // Inject a failure into a durable close step (the counts flush). const storage = (brain as unknown as { storage: { flushCounts: () => Promise } }).storage const boom = new Error('injected: counts flush failed during close') storage.flushCounts = async () => { throw boom } await expect(brain.close()).rejects.toThrow(/injected: counts flush failed/) brain = null // The lock is released regardless: a process on its way out holds nothing. expect(existsSync(lockPath())).toBe(false) // And the next writer opens without a stale-lock verdict. const { lines } = await captureConsole(async () => { const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) await next.init() await next.close() }) expect(lines.filter((l) => /appears dead/i.test(l))).toEqual([]) }, 120_000) it('a SIGKILLed writer leaves the lock with no record, and the next open names the crash', async () => { const { child } = await spawnHoldingChild(dir) expect(existsSync(lockPath())).toBe(true) expect(existsSync(recordPath())).toBe(false) // Group-wide: the lock holder is tsx's grandchild, not the spawned pid. process.kill(-(child.pid as number), 'SIGKILL') await new Promise((r) => child.on('exit', () => r())) // The grandchild's death is asynchronous with the wrapper's exit event. await new Promise((r) => setTimeout(r, 500)) // The lock survives the kill — a dead process releases nothing. expect(existsSync(lockPath())).toBe(true) expect(existsSync(recordPath())).toBe(false) const { lines } = await captureConsole(async () => { const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } }) await next.init() await next.close() }) const verdict = lines.filter((l) => /Overwriting stale writer lock/i.test(l)) expect(verdict.length).toBe(1) // The verdict must name the ABSENT record and the recovery it implies — // not merely that a pid is gone. expect(verdict[0]).toMatch(/NO\s+clean-close record/i) expect(verdict[0]).toMatch(/crash recovery/i) }, 180_000) it("does not force-exit a host application that owns its own SIGTERM handler", async () => { const script = ` import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))} const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } }) await brain.init() await brain.add({ data: 'row from the host app', type: 'concept' }) await brain.flush() // The host application's OWN graceful shutdown, registered after Brainy's. process.on('SIGTERM', async () => { await new Promise((r) => setTimeout(r, 1500)) console.log('APP-CLOSE-DONE') process.exit(0) }) console.log('READY') setInterval(() => {}, 1000) ` const child = startChild(dir, script) let out = '' child.stdout.on('data', (d) => { out += String(d) }) child.stderr.on('data', (d) => { out += String(d) }) await new Promise((r, reject) => { const timer = setTimeout(() => reject(new Error(`child never became READY:\n${out}`)), 120_000) child.stdout.on('data', () => { if (out.includes('READY')) { clearTimeout(timer); r() } }) child.on('exit', () => { clearTimeout(timer); if (!out.includes('READY')) reject(new Error(`child died:\n${out}`)) }) }) process.kill(-(child.pid as number), 'SIGTERM') const code = await new Promise((r) => child.on('exit', (c) => r(c))) expect(code).toBe(0) // The host's own shutdown ran to completion — Brainy's handler did not // exit the process out from under it. expect(out).toContain('APP-CLOSE-DONE') }, 180_000) })