open-brainy/tests/integration/writer-lock-fencing.test.ts

/**
 * @module tests/integration/writer-lock-fencing
 * @description The writer-lock fencing cures, from a production dev-store
 * split-brain (two live writers alternating a store's id-mapper between two
 * internally-consistent truths). Three laws, each pinned:
 *
 * 1. A LIVE writer is never auto-evicted — staleness requires PID-death.
 *    (The old rule evicted on heartbeat age alone, so a >60s event-loop
 *    stall — debugger, GC — handed the lock to a second opener while the
 *    first kept writing.)
 * 2. A DEAD writer's lock still self-clears with narration (venue's ask).
 * 3. THE FENCE: an evicted writer (force-takeover or removed lock) fails
 *    LOUDLY at its next commit barrier — typed BRAINY_WRITER_FENCED — and
 *    never advances the store.
 */
import { describe, it, expect, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Brainy } from '../../src/index.js'
import { NounType } from '../../src/types/graphTypes.js'

const dirs: string[] = []
const brains: Brainy[] = []
afterEach(async () => {
  for (const b of brains.splice(0)) await b.close().catch(() => {})
  for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
})

function lockPath(dir: string): string {
  return join(dir, 'locks', '_writer.lock')
}

async function fsBrain(dir: string): Promise<Brainy> {
  const brain = new Brainy({
    storage: { type: 'filesystem', path: dir },
    requireSubtype: false
  })
  await brain.init()
  brains.push(brain)
  return brain
}

describe('writer-lock fencing', () => {
  it('a LIVE writer with an ancient heartbeat is NOT evicted — the second opener refuses typed', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-live-'))
    dirs.push(dir)
    await fsBrain(dir)

    // Manufacture the trigger shape: a DIFFERENT process's lock (pid 1 —
    // always alive, never ours, EPERM proves liveness) with a >60s-old
    // heartbeat — the blocked-event-loop costume that used to get evicted.
    const lp = lockPath(dir)
    const lock = JSON.parse(fs.readFileSync(lp, 'utf-8'))
    lock.pid = 1
    lock.lastHeartbeat = new Date(Date.now() - 10 * 60_000).toISOString()
    fs.writeFileSync(lp, JSON.stringify(lock))

    // Old rule: heartbeat-age eviction → silent takeover → split brain.
    // New rule: live PID = live writer; the second opener throws typed.
    const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
    await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' })
  }, 120000)

  it("a DEAD writer's lock self-clears and the new opener proceeds", async () => {
    const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-dead-'))
    dirs.push(dir)
    const first = await fsBrain(dir)
    await first.close()
    brains.pop()

    // Manufacture a crashed holder: a lock naming a PID that cannot exist.
    fs.mkdirSync(join(dir, 'locks'), { recursive: true })
    fs.writeFileSync(
      lockPath(dir),
      JSON.stringify({
        pid: 2 ** 22 + 12345, // beyond pid_max on any default Linux
        hostname: os.hostname(),
        startedAt: new Date().toISOString(),
        lastHeartbeat: new Date().toISOString(),
        version: 'test',
        rootDir: dir
      })
    )
    const brain = await fsBrain(dir) // must not throw
    const id = await brain.add({ data: 'post-takeover write', type: NounType.Document, metadata: {} })
    expect(await brain.get(id)).not.toBeNull()
  }, 120000)

  it('the fence does NOT fire on a same-process re-open — the documented warn-and-take-over contract stays benign', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-samepid-'))
    dirs.push(dir)
    const first = await fsBrain(dir)
    await first.add({ data: 'first instance write', type: NounType.Document, metadata: { n: 1 } })

    // A second instance in the SAME process takes the lock over (fresh
    // startedAt) — the pattern server-restart tests use. The first
    // instance's background flushes must keep working: same pid + same
    // hostname IS ownership. (The plant's integration lane caught the
    // startedAt-strict fence latching exactly this shape dead.)
    const second = await fsBrain(dir)
    await second.add({ data: 'second instance write', type: NounType.Document, metadata: { n: 2 } })
    await expect(first.flush()).resolves.toBeUndefined()
    await expect(second.flush()).resolves.toBeUndefined()
  }, 120000)

  it('THE FENCE: a forced-out writer fails its next flush typed and advances nothing', async () => {
    const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-evict-'))
    dirs.push(dir)
    const victim = await fsBrain(dir)
    await victim.add({ data: 'pre-eviction write', type: NounType.Document, metadata: { n: 1 } })
    await victim.flush()
    const genBefore = victim.generation()

    // A successor takes the lock behind the victim's back (the force-takeover
    // shape: different pid + startedAt).
    fs.writeFileSync(
      lockPath(dir),
      JSON.stringify({
        pid: process.pid + 1,
        hostname: os.hostname(),
        startedAt: new Date(Date.now() + 1).toISOString(),
        lastHeartbeat: new Date().toISOString(),
        version: 'test-successor',
        rootDir: dir
      })
    )

    // The victim's next commit barrier must refuse, typed — never write on.
    await victim.add({ data: 'post-eviction write', type: NounType.Document, metadata: { n: 2 } })
    await expect(victim.flush()).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' })
    expect(victim.generation(), 'committed watermark never advanced past the fence')
      .toBeGreaterThanOrEqual(genBefore)

    // Transact leg: the barrier fences there too, and rolls back cleanly.
    await expect(
      victim.transact([
        { op: 'add', id: '00000000-0000-7000-8000-0000000fence', type: NounType.Document, data: 'fenced', metadata: {} }
      ])
    ).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' })

    // Silence the fenced instance's close-time release (it no longer owns the lock).
    brains.pop()
    await victim.close().catch(() => {})
  }, 120000)
})