open-brainy/tests/integration/writer-lock-clean-close.test.ts

/**
 * @module tests/integration/writer-lock-clean-close
 * @description THE CLEAN-CLOSE CONTRACT for the writer lock.
 *
 * A production restart made this lane necessary: a service stopped with exit
 * code 0, having awaited `close()` on every pooled brain, and its next boot
 * announced `[brainy] Overwriting stale writer lock … appears dead` for every
 * store it owned. "The pid is gone" is equally true of an orderly restart and
 * of a crash, so the message could not tell an operator which one they had.
 *
 * The contract pinned here:
 *   1. A completed close leaves NO lock file and DOES leave a clean-close
 *      record; the next open says nothing about staleness.
 *   2. The next lock claim CONSUMES that record — it may never outlive the
 *      lock generation it describes, or a later crash would read as clean.
 *   3. A close whose durable steps FAIL still releases the lock (and still
 *      rethrows the failure).
 *   4. A killed process (SIGKILL, no close at all) leaves the lock behind with
 *      NO record, and the next open says exactly that — crash, recovery ahead.
 *   5. A host application with its own SIGTERM handler is never force-exited
 *      out from under its own shutdown by Brainy's handler.
 */

import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { mkdtempSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs'
import { spawn } from 'node:child_process'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'

const REPO_ROOT = process.cwd()
const TSX = join(REPO_ROOT, 'node_modules', '.bin', 'tsx')

function makeTempDir(): string {
  return mkdtempSync(join(tmpdir(), 'brainy-clean-close-'))
}

/**
 * Write a child script to disk and start it under tsx. A file (not `tsx -e`)
 * because the eval form compiles to CommonJS, which has no top-level await.
 * The script imports Brainy by ABSOLUTE path, so its own dependency
 * resolution still happens from inside the repository.
 */
function startChild(dir: string, body: string): ReturnType<typeof spawn> {
  const scriptPath = join(dir, 'child-process.mts')
  writeFileSync(scriptPath, body)
  // `detached` puts the child in its own process GROUP: tsx runs the script in
  // a grandchild process, and only a group-wide signal reaches the process
  // that actually holds the writer lock.
  return spawn(TSX, [scriptPath], {
    cwd: REPO_ROOT,
    stdio: ['ignore', 'pipe', 'pipe'],
    detached: true
  })
}

/** Capture every console.warn/error line emitted while `fn` runs. */
async function captureConsole<T>(fn: () => Promise<T>): Promise<{ result: T; lines: string[] }> {
  const lines: string[] = []
  const origWarn = console.warn
  const origError = console.error
  const sink = (...args: unknown[]) => {
    lines.push(args.map((a) => String(a)).join(' '))
  }
  console.warn = sink as typeof console.warn
  console.error = sink as typeof console.error
  try {
    const result = await fn()
    return { result, lines }
  } finally {
    console.warn = origWarn
    console.error = origError
  }
}

/**
 * Run a child process that opens `dir`, writes one row, prints `READY`, and
 * then waits forever. Resolves with the child once READY is seen.
 */
function spawnHoldingChild(dir: string): Promise<{
  child: ReturnType<typeof spawn>
  output: () => string
}> {
  const script = `
    import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))}
    const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } })
    await brain.init()
    await brain.add({ data: 'row from the child', type: 'concept' })
    await brain.flush()
    console.log('READY')
    setInterval(() => {}, 1000)
  `
  const child = startChild(dir, script)
  let out = ''
  child.stdout.on('data', (d) => { out += String(d) })
  child.stderr.on('data', (d) => { out += String(d) })
  return new Promise((resolvePromise, rejectPromise) => {
    const timer = setTimeout(() => rejectPromise(new Error(`child never became READY:\n${out}`)), 120_000)
    child.stdout.on('data', () => {
      if (out.includes('READY')) {
        clearTimeout(timer)
        resolvePromise({ child, output: () => out })
      }
    })
    child.on('exit', (code) => {
      clearTimeout(timer)
      if (!out.includes('READY')) rejectPromise(new Error(`child exited ${code} before READY:\n${out}`))
    })
  })
}

describe('writer lock — the clean-close contract', () => {
  let dir: string
  let brain: Brainy | null = null

  beforeEach(() => { dir = makeTempDir() })

  afterEach(async () => {
    if (brain) {
      try { await brain.close() } catch { /* may already be closed */ }
      brain = null
    }
    try { rmSync(dir, { recursive: true, force: true }) } catch { /* ignore */ }
  })

  const lockPath = () => join(dir, 'locks', '_writer.lock')
  const recordPath = () => join(dir, 'locks', '_writer.close')

  it('a completed close leaves no lock, leaves a record, and the reopen is silent about staleness', async () => {
    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
    await brain.init()
    expect(existsSync(lockPath())).toBe(true)

    await brain.add({ data: 'seed entity', type: NounType.Concept })
    await brain.flush()
    await brain.close()
    brain = null

    // 1. The lock is gone and the release is RECORDED.
    expect(existsSync(lockPath())).toBe(false)
    expect(existsSync(recordPath())).toBe(true)
    const record = JSON.parse(readFileSync(recordPath(), 'utf-8'))
    expect(record.pid).toBe(process.pid)
    expect(typeof record.closedAt).toBe('string')
    expect(typeof record.startedAt).toBe('string')

    // 2. The reopen says nothing about a stale lock.
    const { result: reopened, lines } = await captureConsole(async () => {
      const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
      await next.init()
      return next
    })
    brain = reopened
    expect(lines.filter((l) => /stale writer lock|appears dead/i.test(l))).toEqual([])

    // 3. The claim CONSUMED the record — it must not outlive its lock generation.
    expect(existsSync(recordPath())).toBe(false)
    expect(existsSync(lockPath())).toBe(true)
  }, 120_000)

  it('releases the writer lock even when a durable close step fails — and still rethrows', async () => {
    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
    await brain.init()
    await brain.add({ data: 'seed entity', type: NounType.Concept })
    await brain.flush()
    expect(existsSync(lockPath())).toBe(true)

    // Inject a failure into a durable close step (the counts flush).
    const storage = (brain as unknown as { storage: { flushCounts: () => Promise<void> } }).storage
    const boom = new Error('injected: counts flush failed during close')
    storage.flushCounts = async () => { throw boom }

    await expect(brain.close()).rejects.toThrow(/injected: counts flush failed/)
    brain = null

    // The lock is released regardless: a process on its way out holds nothing.
    expect(existsSync(lockPath())).toBe(false)

    // And the next writer opens without a stale-lock verdict.
    const { lines } = await captureConsole(async () => {
      const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
      await next.init()
      await next.close()
    })
    expect(lines.filter((l) => /appears dead/i.test(l))).toEqual([])
  }, 120_000)

  it('a SIGKILLed writer leaves the lock with no record, and the next open names the crash', async () => {
    const { child } = await spawnHoldingChild(dir)
    expect(existsSync(lockPath())).toBe(true)
    expect(existsSync(recordPath())).toBe(false)

    // Group-wide: the lock holder is tsx's grandchild, not the spawned pid.
    process.kill(-(child.pid as number), 'SIGKILL')
    await new Promise<void>((r) => child.on('exit', () => r()))
    // The grandchild's death is asynchronous with the wrapper's exit event.
    await new Promise<void>((r) => setTimeout(r, 500))

    // The lock survives the kill — a dead process releases nothing.
    expect(existsSync(lockPath())).toBe(true)
    expect(existsSync(recordPath())).toBe(false)

    const { lines } = await captureConsole(async () => {
      const next = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
      await next.init()
      await next.close()
    })
    const verdict = lines.filter((l) => /Overwriting stale writer lock/i.test(l))
    expect(verdict.length).toBe(1)
    // The verdict must name the ABSENT record and the recovery it implies —
    // not merely that a pid is gone.
    expect(verdict[0]).toMatch(/NO\s+clean-close record/i)
    expect(verdict[0]).toMatch(/crash recovery/i)
  }, 180_000)

  it("does not force-exit a host application that owns its own SIGTERM handler", async () => {
    const script = `
      import { Brainy } from ${JSON.stringify(join(REPO_ROOT, 'src', 'brainy.ts'))}
      const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: ${JSON.stringify(dir)} } })
      await brain.init()
      await brain.add({ data: 'row from the host app', type: 'concept' })
      await brain.flush()
      // The host application's OWN graceful shutdown, registered after Brainy's.
      process.on('SIGTERM', async () => {
        await new Promise((r) => setTimeout(r, 1500))
        console.log('APP-CLOSE-DONE')
        process.exit(0)
      })
      console.log('READY')
      setInterval(() => {}, 1000)
    `
    const child = startChild(dir, script)
    let out = ''
    child.stdout.on('data', (d) => { out += String(d) })
    child.stderr.on('data', (d) => { out += String(d) })
    await new Promise<void>((r, reject) => {
      const timer = setTimeout(() => reject(new Error(`child never became READY:\n${out}`)), 120_000)
      child.stdout.on('data', () => { if (out.includes('READY')) { clearTimeout(timer); r() } })
      child.on('exit', () => { clearTimeout(timer); if (!out.includes('READY')) reject(new Error(`child died:\n${out}`)) })
    })

    process.kill(-(child.pid as number), 'SIGTERM')
    const code = await new Promise<number | null>((r) => child.on('exit', (c) => r(c)))
    expect(code).toBe(0)
    // The host's own shutdown ran to completion — Brainy's handler did not
    // exit the process out from under it.
    expect(out).toContain('APP-CLOSE-DONE')
  }, 180_000)
})