brainy/tests/integration/counter-recount.test.ts

/**
 * @module tests/integration/counter-recount
 * @description Counter honesty. Two laws under test:
 * (1) REMOVAL NEVER REQUIRES RE-READING THE REMOVED RECORD — the count
 *     decrement falls back to the caller's pre-delete read when the canonical
 *     metadata re-read returns null (replace race / ghost), instead of being
 *     silently skipped. The skip minted permanent inflation: adds counted,
 *     paired removals not decremented, and Math.max(totalNounCount, scanned)
 *     pinned the inflated scalar forever.
 * (2) THE SANCTIONED RECOUNT — repairIndex() unconditionally recomputes and
 *     PERSISTS every counter rollup (scalar totals + per-type maps +
 *     type-statistics) from one canonical walk, so an already-inflated brain
 *     is permanently corrected (survives reopen).
 */
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy } from '../../src/index.js'

/** Absolute path of one entity's `<id>` directory (or null if not present). */
function entityDir(root: string, id: string): string | null {
  const base = path.join(root, 'entities', 'nouns')
  if (!fs.existsSync(base)) return null
  for (const shard of fs.readdirSync(base)) {
    const candidate = path.join(base, shard, id)
    if (fs.existsSync(candidate)) return candidate
  }
  return null
}

describe('counter honesty — removal without re-reading + the sanctioned recount', () => {
  let dir: string
  let brain: any

  const open = async () => {
    const b: any = new Brainy({
      requireSubtype: false,
      storage: { type: 'filesystem', path: dir },
      silent: true,
      dimensions: 384
    })
    await b.init()
    return b
  }

  beforeEach(async () => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-recount-'))
    brain = await open()
  })
  afterEach(async () => {
    await brain.close?.().catch(() => {})
    fs.rmSync(dir, { recursive: true, force: true })
  })

  it('the counter returns to baseline across add→remove→re-create cycles (no drift)', async () => {
    const baseline = await brain.storage.getNounCount()
    for (let i = 0; i < 4; i++) {
      const id = await brain.add({ data: `cycle ${i}`, type: 'document', metadata: { i } })
      expect(await brain.storage.getNounCount()).toBe(baseline + 1)
      await brain.remove(id)
      expect(await brain.storage.getNounCount()).toBe(baseline)
    }
  })

  it('deleteNoun decrements from the provided prior record when the canonical re-read is null', async () => {
    const id = await brain.add({ data: 'to be ghosted', type: 'document', metadata: { g: 1 } })
    await brain.flush()
    const baseline = await brain.storage.getNounCount()

    // Capture the pre-delete read (what remove() holds), then simulate the
    // replace-race / ghost shape: the metadata leg vanishes before the delete's
    // internal re-read.
    const prior = await brain.storage.getNounMetadata(id)
    expect(prior).not.toBeNull()
    const eDir = entityDir(dir, id)!
    for (const f of fs.readdirSync(eDir)) {
      if (f.startsWith('metadata.json')) fs.rmSync(path.join(eDir, f))
    }

    // Without the prior record this decrement used to be silently skipped.
    await brain.storage.deleteNoun(id, prior)
    expect(await brain.storage.getNounCount()).toBe(baseline - 1)
    // Full removal still holds: nothing left on disk.
    expect(entityDir(dir, id)).toBeNull()
  })

  it('repairIndex() recounts an inflated persisted scalar over CLEAN shelves — and it survives reopen', async () => {
    for (let i = 0; i < 3; i++) {
      await brain.add({ data: `real ${i}`, type: 'document', metadata: { i } })
    }
    await brain.flush()
    const honest = await brain.storage.getNounCount()
    // Pagination's totalCount has its own baseline: it enumerates EVERYTHING
    // (including the internal VFS root), while the user-facing scalar counts
    // only public entities — so the two legitimately differ by the internals.
    const pageHonest = (await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 })).totalCount

    // Simulate the historical drift: an inflated persisted scalar (deletes whose
    // decrement was skipped). Persist it so a reopen rehydrates the lie.
    ;(brain.storage as any).totalNounCount = honest + 52
    await (brain.storage as any).persistCounts()
    await brain.close()
    brain = await open()
    expect(await brain.storage.getNounCount()).toBe(honest + 52) // the lie survived reopen

    // The sanctioned recount — unconditional in repairIndex (no orphans needed).
    await brain.repairIndex()
    expect(await brain.storage.getNounCount()).toBe(honest)

    // Permanently: the corrected counter survives another reopen.
    await brain.close()
    brain = await open()
    expect(await brain.storage.getNounCount()).toBe(honest)

    // And the paginated totalCount (the Math.max consumer) is honest too —
    // back to ITS baseline, no longer pinned high by the inflated scalar.
    const page = await brain.storage.getNounsWithPagination({ limit: 1000, offset: 0 })
    expect(page.totalCount).toBe(pageHonest)
  })
})