fix: full-removal canonical deletes + family-scoped migration gate

Two production-reported spine fixes:

- Canonical delete is a FULL removal: remove()/removeMany() deleted the
  metadata leg but never the canonical vectors.json leg or the <id>/
  directory, leaving ghost rows that inflated enumerated counts forever,
  read as damage scars, and caused duplicate readdir entries for
  re-created VFS paths (the stale-unpost path). The delete operation now
  routes through storage.deleteNoun — both legs + the entity container —
  with a full two-leg before-image rollback; deleteVerb symmetric; the
  blind 'no metadata file' catch that masked real faults is gone. The
  generation log still holds the delete's before-image, so asOf() history
  is unchanged. repairIndex() gains a conservative, loud orphan-prune
  sweep (containers with no metadata content leg) + count recompute for
  stores damaged by earlier versions.

- Family-scoped migration gate: every read blocked on the whole-brain
  migration lock even when the migrating index was irrelevant. The gate
  now scopes to the index families a read actually consults — canonical
  reads never wait; find() waits only on its query shape's families;
  graph traversals wait only on the graph family. Writes keep the
  conservative whole-brain wait. A read that needs the migrating family
  still blocks (bounded) with the retryable MigrationInProgressError.

tests/integration/delete-full-removal.test.ts

@@ -0,0 +1,161 @@
+/**
+ * @module tests/integration/delete-full-removal
+ * @description Canonical noun/verb deletes are FULL removals: both legs
+ * (metadata + vectors) AND the entity's `<id>/` container are removed, so a
+ * delete leaves NOTHING behind. Regression for the ghost/scar defect where
+ * remove() deleted the vector INDEX entry + the canonical metadata leg but never
+ * the canonical vectors.json leg or the directory — leaving an orphan that reads
+ * as absent (getNoun needs both legs) yet inflated the enumerated count and
+ * confused locator resolution. Also covers the operator repair sweep
+ * (brain.repairIndex()) that prunes orphans left by the pre-fix behavior.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+
+/** All entity-directory names (the `<id>` dirs) under entities/<kind>. */
+function entityDirs(root: string, kind: 'nouns' | 'verbs'): string[] {
+  const base = path.join(root, 'entities', kind)
+  if (!fs.existsSync(base)) return []
+  const ids: string[] = []
+  for (const shard of fs.readdirSync(base)) {
+    const shardDir = path.join(base, shard)
+    if (!fs.statSync(shardDir).isDirectory()) continue
+    for (const id of fs.readdirSync(shardDir)) {
+      if (fs.statSync(path.join(shardDir, id)).isDirectory()) ids.push(id)
+    }
+  }
+  return ids
+}
+
+/** Absolute path of one entity's `<id>` directory (or null if not present). */
+function entityDir(root: string, kind: 'nouns' | 'verbs', id: string): string | null {
+  const base = path.join(root, 'entities', kind)
+  if (!fs.existsSync(base)) return null
+  for (const shard of fs.readdirSync(base)) {
+    const candidate = path.join(base, shard, id)
+    if (fs.existsSync(candidate)) return candidate
+  }
+  return null
+}
+
+describe('canonical delete is a full removal (no ghost/scar directory)', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-del-'))
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
+    await brain.init()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('remove() deletes BOTH legs and the container — nothing left on disk', async () => {
+    const ids: string[] = []
+    for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } }))
+    await brain.flush()
+
+    // (A filesystem brain also has its VFS-root noun, so don't assume an exact set.)
+    const before = entityDirs(dir, 'nouns')
+    expect(before).toEqual(expect.arrayContaining(ids))
+
+    await brain.remove(ids[1])
+    await brain.flush()
+
+    // getNoun is null AND the on-disk container is fully gone (no orphan), and
+    // EXACTLY one directory disappeared (the removed entity's).
+    expect(await brain.get(ids[1])).toBeNull()
+    expect(entityDir(dir, 'nouns', ids[1])).toBeNull()
+    const after = entityDirs(dir, 'nouns')
+    expect(after).toHaveLength(before.length - 1)
+    expect(after).toEqual(expect.arrayContaining([ids[0], ids[2]]))
+    expect(after).not.toContain(ids[1])
+  })
+
+  it('the enumerated count is honest after a delete (no monotonic inflation)', async () => {
+    const ids: string[] = []
+    for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `n${i}`, type: 'document', metadata: { i } }))
+    await brain.flush()
+
+    const before = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 })
+
+    await brain.remove(ids[0])
+    await brain.remove(ids[1])
+    await brain.flush()
+
+    // The count drops by EXACTLY the two removed entities — no ghost lingers to
+    // hold the total up (the pre-fix defect left it monotonic).
+    const after = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 })
+    expect(after.totalCount).toBe(before.totalCount - 2)
+    const remaining = after.items.map((n: any) => n.id)
+    expect(remaining).toEqual(expect.arrayContaining([ids[2], ids[3]]))
+    expect(remaining).not.toContain(ids[0])
+    expect(remaining).not.toContain(ids[1])
+  })
+})
+
+describe('repairIndex() prunes orphan containers left by the pre-fix partial delete', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-orphan-'))
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
+    await brain.init()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('a vector-only "ghost" (metadata leg deleted out-of-band) is pruned', async () => {
+    const ids: string[] = []
+    for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `g${i}`, type: 'document', metadata: { i } }))
+    await brain.flush()
+
+    // Simulate the pre-fix defect on ids[0]: remove ONLY its metadata leg,
+    // leaving vectors.json + the directory (the exact ghost shape).
+    const ghostDir = entityDir(dir, 'nouns', ids[0])!
+    for (const f of fs.readdirSync(ghostDir)) {
+      if (f.startsWith('metadata.json')) fs.rmSync(path.join(ghostDir, f))
+    }
+    // The ghost dir (vectors.json, no metadata content leg) still sits on disk.
+    expect(entityDir(dir, 'nouns', ids[0])).not.toBeNull()
+
+    await brain.repairIndex()
+
+    // The sweep removes the ghost container; the two healthy entities survive.
+    expect(entityDir(dir, 'nouns', ids[0])).toBeNull()
+    expect(entityDir(dir, 'nouns', ids[1])).not.toBeNull()
+    expect(entityDir(dir, 'nouns', ids[2])).not.toBeNull()
+  })
+
+  it('an empty "scar" directory is pruned', async () => {
+    const id = await brain.add({ data: 'lonely', type: 'document', metadata: {} })
+    await brain.flush()
+    const scarDir = entityDir(dir, 'nouns', id)!
+    for (const f of fs.readdirSync(scarDir)) fs.rmSync(path.join(scarDir, f)) // empty the dir, keep it
+    expect(fs.existsSync(scarDir)).toBe(true)
+
+    await brain.repairIndex()
+
+    expect(fs.existsSync(scarDir)).toBe(false)
+  })
+
+  it('a healthy entity (both legs present) is NEVER pruned', async () => {
+    const id = await brain.add({ data: 'keep me', type: 'document', metadata: { keep: true } })
+    await brain.flush()
+
+    await brain.repairIndex()
+
+    expect(entityDir(dir, 'nouns', id)).not.toBeNull()
+    expect(await brain.get(id)).not.toBeNull()
+  })
+})

tests/integration/readdir-no-duplicate-replace.test.ts

@@ -0,0 +1,56 @@
+/**
+ * @module tests/integration/readdir-no-duplicate-replace
+ * @description A re-created path must NOT show up twice in readdir. The reported
+ * defect (memory-vm) was readdir serving DUPLICATE entries for re-created paths,
+ * rooted in a delete that left a ghost (partial removal) whose next delete read
+ * null metadata and skipped unposting the stale Contains edge. With full-removal
+ * deletes there is no ghost, so the edge is always unposted and the path appears
+ * exactly once after any number of delete→recreate cycles.
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy } from '../../src/index.js'
+
+describe('readdir shows a re-created path exactly once (no duplicate on replace)', () => {
+  let dir: string
+  let brain: any
+
+  beforeEach(async () => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-readdir-'))
+    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
+    await brain.init()
+  })
+  afterEach(async () => {
+    await brain.close?.().catch(() => {})
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('delete then re-create the same file path — readdir lists it once', async () => {
+    await brain.vfs.mkdir('/d', { recursive: true })
+    await brain.vfs.writeFile('/d/x.txt', 'v1')
+    expect(await brain.vfs.readdir('/d')).toEqual(['x.txt'])
+
+    await brain.vfs.unlink('/d/x.txt')
+    expect(await brain.vfs.readdir('/d')).toEqual([])
+
+    await brain.vfs.writeFile('/d/x.txt', 'v2')
+    const listing = (await brain.vfs.readdir('/d')) as string[]
+    expect(listing).toEqual(['x.txt']) // exactly once — no ghost duplicate
+    expect(listing.filter((n) => n === 'x.txt')).toHaveLength(1)
+  })
+
+  it('survives several delete→recreate cycles without accumulating duplicates', async () => {
+    await brain.vfs.mkdir('/c', { recursive: true })
+    for (let i = 0; i < 4; i++) {
+      await brain.vfs.writeFile('/c/f.txt', `gen ${i}`)
+      await brain.vfs.unlink('/c/f.txt')
+    }
+    await brain.vfs.writeFile('/c/f.txt', 'final')
+    const listing = (await brain.vfs.readdir('/c')) as string[]
+    expect(listing).toEqual(['f.txt'])
+    expect(await brain.vfs.readFile('/c/f.txt')).toBeDefined()
+  })
+})