fix(locks): live writers are never auto-evicted; evicted writers are fenced at every commit barrier

The production dev-store split-brain (two live writers alternating a store's
id-mapper between two internally-consistent truths), cured at all three of
its roots. (1) STALENESS REQUIRES PID-DEATH: the old rule evicted on
heartbeat age alone, so a >60s event-loop stall (debugger pause, GC, heavy
sync work) handed the lock to a second opener while the first kept writing;
a live process is now never auto-evicted — a wedged-but-alive holder is the
operator's call via {force:true}, and the heartbeat stays for observability.
(2) THE CLAIM IS ATOMIC: writeFile(wx)'s open→write→close left an empty-file
window a concurrent opener could read as torn, unlink a LIVE claim, and take
the lock; the claim is now tmp-write + hard-link — the lock appears with its
full contents in one step. (3) THE FENCE: every flush commit and transact
barrier verifies lock ownership first (one small read per window) — a
forced-out or lock-deleted writer fails typed (BRAINY_WRITER_FENCED) before
a single staged byte or manifest advance, instead of writing on unaware.

Pinned: live-with-ancient-heartbeat refuses typed; dead-PID self-clears
narrated; a forced-out writer's flush and transact both fence, advancing
nothing. Requested by a downstream team as single-writer guard or loud
lockout — this is both.

src/db/generationStore.ts

@@ -1394,6 +1394,10 @@ export class GenerationStore {
         // The transaction's entire canonical footprint is now durable, so the
         // counter/manifest advance below can never outrun the entity bytes.
         await this.storage.flushWriteBarrier?.()
+        // THE FENCE (transact leg): verify lock ownership before the commit
+        // point — an aborted-by-fence transact rolls back cleanly through the
+        // catch below; a fenced writer must never advance counter or manifest.
+        await this.storage.assertWriterFenceHeld?.()
         faultPoint('after-execute')
 
         // Fact log (dual-write): append + fsync this generation's AFTER-IMAGE
@@ -1915,6 +1919,11 @@ export class GenerationStore {
   private async flushPendingSingleOpsUnlocked(): Promise<void> {
     return this.withMutex(async () => {
       if (this.pendingGens.length === 0) return
+      // THE FENCE: an evicted writer (force-takeover, removed lock) must fail
+      // HERE, before a single staged byte or manifest advance — writing on
+      // after eviction is how split-brain stores are made. One small read
+      // per flush window.
+      await this.storage.assertWriterFenceHeld?.()
       this.clearPendingFlushTimer()
 
       const gens = [...this.pendingGens].sort((a, b) => a - b)

src/db/types.ts

@@ -485,6 +485,16 @@ export interface GenerationStorage {
    */
   syncEntityCanonical?(nouns: string[], verbs: string[]): Promise<void>
 
+  /**
+   * OPTIONAL writer fence: throw `BRAINY_WRITER_FENCED` when this instance
+   * no longer owns the store's writer lock (an operator force-takeover or a
+   * removed lock file). Called at every flush commit and transact barrier —
+   * one small read per commit window — so an evicted writer fails loudly on
+   * its next commit instead of split-braining the store. Adapters without a
+   * cross-process lock model omit it.
+   */
+  assertWriterFenceHeld?(): Promise<void>
+
   /** Read an entity's raw stored metadata+vector objects. */
   readNounRaw(id: string): Promise<{ metadata: any | null; vector: any | null }>
   /** Restore an entity's raw stored objects (`null` part ⇒ delete that file). */

src/storage/adapters/fileSystemStorage.ts

@@ -1920,14 +1920,24 @@ export class FileSystemStorage extends BaseStorage {
         rootDir: this.rootDir
       }
 
-      // The atomic claim: create-exclusive, so exactly ONE racer wins.
+      // The atomic claim: write the FULL contents to a temp file, then
+      // hard-link it into place — link(2) fails EEXIST if the target exists,
+      // and the lock file appears with its complete JSON in one atomic step.
+      // (The previous claim was writeFile with O_EXCL, whose open→write→close
+      // is NOT atomic: a concurrent opener could read the file in its empty
+      // window, judge it torn, unlink a LIVE claim, and take the lock — two
+      // live writers. The link claim leaves no empty window to misread.)
+      const claimTmp = `${lockFile}.claim-${myPid}-${Date.now()}`
       try {
-        await fs.promises.writeFile(lockFile, JSON.stringify(info, null, 2), { flag: 'wx' })
+        await fs.promises.writeFile(claimTmp, JSON.stringify(info, null, 2))
+        await fs.promises.link(claimTmp, lockFile)
       } catch (err: any) {
         if (err.code === 'EEXIST') {
           continue // someone else claimed between our read and create — re-evaluate
         }
         throw err
+      } finally {
+        await fs.promises.unlink(claimTmp).catch(() => {})
       }
 
       this.installWriterLock(info)
@@ -1972,6 +1982,44 @@ export class FileSystemStorage extends BaseStorage {
     }
   }
 
+  /**
+   * THE FENCE: verify this instance still owns the writer lock before a
+   * commit barrier proceeds. An evicted writer (an operator's
+   * `{ force: true }` takeover, or an operator deleting the lock file) must
+   * fail LOUDLY on its next flush instead of writing on unaware — the
+   * unfenced evicted writer was half of a production split-brain (each
+   * writer flushing its own internally-consistent id-mapper snapshot,
+   * alternating the store between two truths). One small file read per
+   * flush window, never per record. No-op when this instance holds no
+   * writer lock (read-only opens, in-memory stores).
+   *
+   * @throws `BRAINY_WRITER_FENCED` when the lock is gone or held by another.
+   */
+  public override async assertWriterFenceHeld(): Promise<void> {
+    if (!this.writerLockInfo) return
+    const current = await this.readWriterLock()
+    if (
+      current &&
+      current.pid === this.writerLockInfo.pid &&
+      current.hostname === this.writerLockInfo.hostname &&
+      current.startedAt === this.writerLockInfo.startedAt
+    ) {
+      return
+    }
+    const err = new Error(
+      `Writer fence lost for ${this.rootDir}: this process (PID ${this.writerLockInfo.pid}) ` +
+        `no longer holds the writer lock — ` +
+        (current
+          ? `it is now held by PID ${current.pid} on ${current.hostname} (since ${current.startedAt}).`
+          : `the lock file is gone (released or removed by an operator).`) +
+        `\nThis instance refuses to commit further writes: a fenced-out writer continuing to ` +
+        `flush is how split-brain stores are made. Close this instance; if the takeover was a ` +
+        `mistake, close the successor and re-open.`
+    ) as Error & { code: string }
+    err.code = 'BRAINY_WRITER_FENCED'
+    throw err
+  }
+
   /** The consumer-facing BRAINY_WRITER_LOCKED error, holder details attached. */
   private writerLockedError(existing: WriterLockInfo): Error {
     const err = new Error(
@@ -2060,18 +2108,25 @@ export class FileSystemStorage extends BaseStorage {
 
   /**
    * Determine whether an existing writer lock is stale (safe to overwrite).
-   * Same hostname and (dead PID OR heartbeat older than threshold) → stale.
+   * Same hostname and DEAD PID → stale. That is the whole rule: a LIVE
-   * Different hostname → cannot prove stale, treat as live.
+   * process is never auto-evicted, however old its heartbeat — a >60s
+   * event-loop stall (debugger pause, GC, heavy sync work) is a slow writer,
+   * not a dead one, and heartbeat-age eviction of live writers was the
+   * dominant mechanism behind a production split-brain (two live unaware
+   * writers alternating a store's id-mapper between two truths). A holder
+   * that LOOKS alive but is truly wedged is the operator's call via
+   * `{ force: true }` — and the fence check on every flush
+   * ({@link assertWriterFenceHeld}) guarantees a forced-out holder fails
+   * loudly instead of writing on. Different hostname → cannot prove
+   * anything, treat as live. The heartbeat remains for OBSERVABILITY (the
+   * lock error names it so an operator can judge staleness themselves).
    */
   private async isWriterLockStale(lock: WriterLockInfo): Promise<boolean> {
     const os = await import('node:os')
     if (lock.hostname !== os.hostname()) {
       return false
     }
-    const heartbeatAge = Date.now() - new Date(lock.lastHeartbeat).getTime()
+    return !this.isPidAlive(lock.pid)
-    const pidAlive = this.isPidAlive(lock.pid)
-    if (!pidAlive) return true
-    return heartbeatAge > FileSystemStorage.WRITER_STALE_THRESHOLD_MS
   }
 
   /**

src/storage/baseStorage.ts

@@ -612,6 +612,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     return null
   }
 
+  /**
+   * THE FENCE: verify this instance still owns its writer lock before a
+   * commit barrier proceeds; throw `BRAINY_WRITER_FENCED` if evicted. The
+   * default is a no-op — adapters without a cross-process lock model (memory,
+   * per-request cloud stores) have no eviction to fence against. The
+   * filesystem adapter overrides this; the generation store calls it at
+   * every flush commit and transact barrier.
+   */
+  public async assertWriterFenceHeld(): Promise<void> {
+    // No-op by default — no lock model, nothing to be evicted from.
+  }
+
   /**
    * Start watching for cross-process flush requests. The writer Brainy
    * instance calls this so that out-of-process inspectors can ask for a

tests/integration/writer-lock-fencing.test.ts

@@ -0,0 +1,131 @@
+/**
+ * @module tests/integration/writer-lock-fencing
+ * @description The writer-lock fencing cures, from a production dev-store
+ * split-brain (two live writers alternating a store's id-mapper between two
+ * internally-consistent truths). Three laws, each pinned:
+ *
+ * 1. A LIVE writer is never auto-evicted — staleness requires PID-death.
+ *    (The old rule evicted on heartbeat age alone, so a >60s event-loop
+ *    stall — debugger, GC — handed the lock to a second opener while the
+ *    first kept writing.)
+ * 2. A DEAD writer's lock still self-clears with narration (venue's ask).
+ * 3. THE FENCE: an evicted writer (force-takeover or removed lock) fails
+ *    LOUDLY at its next commit barrier — typed BRAINY_WRITER_FENCED — and
+ *    never advances the store.
+ */
+import { describe, it, expect, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import { mkdtempSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { Brainy } from '../../src/index.js'
+import { NounType } from '../../src/types/graphTypes.js'
+
+const dirs: string[] = []
+const brains: Brainy[] = []
+afterEach(async () => {
+  for (const b of brains.splice(0)) await b.close().catch(() => {})
+  for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true })
+})
+
+function lockPath(dir: string): string {
+  return join(dir, 'locks', '_writer.lock')
+}
+
+async function fsBrain(dir: string): Promise<Brainy> {
+  const brain = new Brainy({
+    storage: { type: 'filesystem', path: dir },
+    requireSubtype: false
+  })
+  await brain.init()
+  brains.push(brain)
+  return brain
+}
+
+describe('writer-lock fencing', () => {
+  it('a LIVE writer with an ancient heartbeat is NOT evicted — the second opener refuses typed', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-live-'))
+    dirs.push(dir)
+    await fsBrain(dir)
+
+    // Manufacture the trigger shape: a DIFFERENT process's lock (pid 1 —
+    // always alive, never ours, EPERM proves liveness) with a >60s-old
+    // heartbeat — the blocked-event-loop costume that used to get evicted.
+    const lp = lockPath(dir)
+    const lock = JSON.parse(fs.readFileSync(lp, 'utf-8'))
+    lock.pid = 1
+    lock.lastHeartbeat = new Date(Date.now() - 10 * 60_000).toISOString()
+    fs.writeFileSync(lp, JSON.stringify(lock))
+
+    // Old rule: heartbeat-age eviction → silent takeover → split brain.
+    // New rule: live PID = live writer; the second opener throws typed.
+    const second = new Brainy({ storage: { type: 'filesystem', path: dir }, requireSubtype: false })
+    await expect(second.init()).rejects.toMatchObject({ code: 'BRAINY_WRITER_LOCKED' })
+  }, 120000)
+
+  it("a DEAD writer's lock self-clears and the new opener proceeds", async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-dead-'))
+    dirs.push(dir)
+    const first = await fsBrain(dir)
+    await first.close()
+    brains.pop()
+
+    // Manufacture a crashed holder: a lock naming a PID that cannot exist.
+    fs.mkdirSync(join(dir, 'locks'), { recursive: true })
+    fs.writeFileSync(
+      lockPath(dir),
+      JSON.stringify({
+        pid: 2 ** 22 + 12345, // beyond pid_max on any default Linux
+        hostname: os.hostname(),
+        startedAt: new Date().toISOString(),
+        lastHeartbeat: new Date().toISOString(),
+        version: 'test',
+        rootDir: dir
+      })
+    )
+    const brain = await fsBrain(dir) // must not throw
+    const id = await brain.add({ data: 'post-takeover write', type: NounType.Document, metadata: {} })
+    expect(await brain.get(id)).not.toBeNull()
+  }, 120000)
+
+  it('THE FENCE: a forced-out writer fails its next flush typed and advances nothing', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'brainy-fence-evict-'))
+    dirs.push(dir)
+    const victim = await fsBrain(dir)
+    await victim.add({ data: 'pre-eviction write', type: NounType.Document, metadata: { n: 1 } })
+    await victim.flush()
+    const genBefore = victim.generation()
+
+    // A successor takes the lock behind the victim's back (the force-takeover
+    // shape: different pid + startedAt).
+    fs.writeFileSync(
+      lockPath(dir),
+      JSON.stringify({
+        pid: process.pid + 1,
+        hostname: os.hostname(),
+        startedAt: new Date(Date.now() + 1).toISOString(),
+        lastHeartbeat: new Date().toISOString(),
+        version: 'test-successor',
+        rootDir: dir
+      })
+    )
+
+    // The victim's next commit barrier must refuse, typed — never write on.
+    await victim.add({ data: 'post-eviction write', type: NounType.Document, metadata: { n: 2 } })
+    await expect(victim.flush()).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' })
+    expect(victim.generation(), 'committed watermark never advanced past the fence')
+      .toBeGreaterThanOrEqual(genBefore)
+
+    // Transact leg: the barrier fences there too, and rolls back cleanly.
+    await expect(
+      victim.transact([
+        { op: 'add', id: '00000000-0000-7000-8000-0000000fence', type: NounType.Document, data: 'fenced', metadata: {} }
+      ])
+    ).rejects.toMatchObject({ code: 'BRAINY_WRITER_FENCED' })
+
+    // Silence the fenced instance's close-time release (it no longer owns the lock).
+    brains.pop()
+    await victim.close().catch(() => {})
+  }, 120000)
+})