brainy/tests/integration/delete-full-removal.test.ts

/**
 * @module tests/integration/delete-full-removal
 * @description Canonical noun/verb deletes are FULL removals: both legs
 * (metadata + vectors) AND the entity's `<id>/` container are removed, so a
 * delete leaves NOTHING behind. Regression for the ghost/scar defect where
 * remove() deleted the vector INDEX entry + the canonical metadata leg but never
 * the canonical vectors.json leg or the directory — leaving an orphan that reads
 * as absent (getNoun needs both legs) yet inflated the enumerated count and
 * confused locator resolution. Also covers the operator repair sweep
 * (brain.repairIndex()) that prunes orphans left by the pre-fix behavior.
 */
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/index.js'

/** All entity-directory names (the `<id>` dirs) under entities/<kind>. */
function entityDirs(root: string, kind: 'nouns' | 'verbs'): string[] {
  const base = path.join(root, 'entities', kind)
  if (!fs.existsSync(base)) return []
  const ids: string[] = []
  for (const shard of fs.readdirSync(base)) {
    const shardDir = path.join(base, shard)
    if (!fs.statSync(shardDir).isDirectory()) continue
    for (const id of fs.readdirSync(shardDir)) {
      if (fs.statSync(path.join(shardDir, id)).isDirectory()) ids.push(id)
    }
  }
  return ids
}

/** Absolute path of one entity's `<id>` directory (or null if not present). */
function entityDir(root: string, kind: 'nouns' | 'verbs', id: string): string | null {
  const base = path.join(root, 'entities', kind)
  if (!fs.existsSync(base)) return null
  for (const shard of fs.readdirSync(base)) {
    const candidate = path.join(base, shard, id)
    if (fs.existsSync(candidate)) return candidate
  }
  return null
}

describe('canonical delete is a full removal (no ghost/scar directory)', () => {
  let dir: string
  let brain: any

  beforeEach(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-del-'))
    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
    await brain.init()
  })
  afterEach(async () => {
    await brain.close?.().catch(() => {})
    fs.rmSync(dir, { recursive: true, force: true })
  })

  it('remove() deletes BOTH legs and the container — nothing left on disk', async () => {
    const ids: string[] = []
    for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `doc ${i}`, type: 'document', metadata: { i } }))
    await brain.flush()

    // (A filesystem brain also has its VFS-root noun, so don't assume an exact set.)
    const before = entityDirs(dir, 'nouns')
    expect(before).toEqual(expect.arrayContaining(ids))

    await brain.remove(ids[1])
    await brain.flush()

    // getNoun is null AND the on-disk container is fully gone (no orphan), and
    // EXACTLY one directory disappeared (the removed entity's).
    expect(await brain.get(ids[1])).toBeNull()
    expect(entityDir(dir, 'nouns', ids[1])).toBeNull()
    const after = entityDirs(dir, 'nouns')
    expect(after).toHaveLength(before.length - 1)
    expect(after).toEqual(expect.arrayContaining([ids[0], ids[2]]))
    expect(after).not.toContain(ids[1])
  })

  it('the enumerated count is honest after a delete (no monotonic inflation)', async () => {
    const ids: string[] = []
    for (let i = 0; i < 4; i++) ids.push(await brain.add({ data: `n${i}`, type: 'document', metadata: { i } }))
    await brain.flush()

    const before = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 })

    await brain.remove(ids[0])
    await brain.remove(ids[1])
    await brain.flush()

    // The count drops by EXACTLY the two removed entities — no ghost lingers to
    // hold the total up (the pre-fix defect left it monotonic).
    const after = await brain.storage.getNounsWithPagination({ limit: 100, offset: 0 })
    expect(after.totalCount).toBe(before.totalCount - 2)
    const remaining = after.items.map((n: any) => n.id)
    expect(remaining).toEqual(expect.arrayContaining([ids[2], ids[3]]))
    expect(remaining).not.toContain(ids[0])
    expect(remaining).not.toContain(ids[1])
  })
})

describe('repairIndex() prunes orphan containers left by the pre-fix partial delete', () => {
  let dir: string
  let brain: any

  beforeEach(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-orphan-'))
    brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir }, silent: true, dimensions: 384 })
    await brain.init()
  })
  afterEach(async () => {
    await brain.close?.().catch(() => {})
    fs.rmSync(dir, { recursive: true, force: true })
  })

  it('a vector-only "ghost" (metadata leg deleted out-of-band) is pruned', async () => {
    const ids: string[] = []
    for (let i = 0; i < 3; i++) ids.push(await brain.add({ data: `g${i}`, type: 'document', metadata: { i } }))
    await brain.flush()

    // Simulate the pre-fix defect on ids[0]: remove ONLY its metadata leg,
    // leaving vectors.json + the directory (the exact ghost shape).
    const ghostDir = entityDir(dir, 'nouns', ids[0])!
    for (const f of fs.readdirSync(ghostDir)) {
      if (f.startsWith('metadata.json')) fs.rmSync(path.join(ghostDir, f))
    }
    // The ghost dir (vectors.json, no metadata content leg) still sits on disk.
    expect(entityDir(dir, 'nouns', ids[0])).not.toBeNull()

    await brain.repairIndex()

    // The sweep removes the ghost container; the two healthy entities survive.
    expect(entityDir(dir, 'nouns', ids[0])).toBeNull()
    expect(entityDir(dir, 'nouns', ids[1])).not.toBeNull()
    expect(entityDir(dir, 'nouns', ids[2])).not.toBeNull()
  })

  it('an empty "scar" directory is pruned', async () => {
    const id = await brain.add({ data: 'lonely', type: 'document', metadata: {} })
    await brain.flush()
    const scarDir = entityDir(dir, 'nouns', id)!
    for (const f of fs.readdirSync(scarDir)) fs.rmSync(path.join(scarDir, f)) // empty the dir, keep it
    expect(fs.existsSync(scarDir)).toBe(true)

    await brain.repairIndex()

    expect(fs.existsSync(scarDir)).toBe(false)
  })

  it('a healthy entity (both legs present) is NEVER pruned', async () => {
    const id = await brain.add({ data: 'keep me', type: 'document', metadata: { keep: true } })
    await brain.flush()

    await brain.repairIndex()

    expect(entityDir(dir, 'nouns', id)).not.toBeNull()
    expect(await brain.get(id)).not.toBeNull()
  })
})