open-brainy/tests/integration/ledger-derivation-identity.test.ts

/**
 * @module tests/integration/ledger-derivation-identity
 * @description The ALL-visibility ledger scalars are an IDENTITY-RECORD
 * count, never a container count. A pre-8.3.1 partial-delete defect can
 * leave a "ghost" container (a stale `vectors.json` with no metadata content
 * leg) or a "scar" container (an empty `entities/<kind>/<shard>/<id>/`
 * directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need
 * the metadata content leg — yet the legacy derivation counted one entity
 * per id DIRECTORY, so orphaned containers inflated the ALL scalars forever
 * (they were never clamped and never re-derived). Laws under test:
 * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per
 *     metadata content leg (`metadata.json` or `.json.gz`), the same test
 *     `pruneOrphanedEntities()` uses, so the two agree by construction.
 * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AND THE OPEN NEVER WALKS — a
 *     counts.json that carries the ALL scalars but no
 *     `allCountsDerivedBy: 'identity-record'` stamp predates this fix;
 *     loading it marks `suspect = true` from a single field read alone and
 *     warns exactly once naming the cause. The open itself never pays a
 *     directory walk.
 * (2b) AND IT HEALS ITSELF. The ledger used to stay wrong for the life of the
 *     store, waiting for an operator to run `repairIndex()` — and a
 *     downstream index heal subtracted against the inflated denominator and
 *     reported work that did not exist. An honest derivation now runs in the
 *     BACKGROUND after the open (never blocking it, observable via
 *     `whenCountLedgerSettled()`), and refuses to stamp a number it derived
 *     while writes were landing.
 * (3) THE SANCTIONED RECOUNT ALSO CLEARS IT — `repairIndex()` prunes the
 *     orphaned containers, recounts from the canonical metadata.json walk,
 *     and re-stamps — the ALL scalar is exact and the containers are gone.
 * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store
 *     with no counts.json stamps as it writes, so a brand-new store never
 *     carries the legacy signature.
 */
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import * as fs from 'node:fs'
import * as os from 'node:os'
import * as path from 'node:path'
import { Brainy, FileSystemStorage } from '../../src/index.js'
import { prodLog } from '../../src/utils/logger.js'

const countsPath = (root: string) => path.join(root, '_system', 'counts.json')

/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */
function plantGhost(root: string, shard: string, id: string): void {
  const idDir = path.join(root, 'entities', 'nouns', shard, id)
  fs.mkdirSync(idDir, { recursive: true })
  fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] }))
}

/** Plant a scar container: an empty id directory, no legs at all. */
function plantScar(root: string, shard: string, id: string): void {
  fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true })
}

describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => {
  let dir: string

  const open = async () => {
    const b: any = new Brainy({
      requireSubtype: false,
      storage: { type: 'filesystem', path: dir },
      silent: true,
      dimensions: 384
    })
    await b.init()
    return b
  }

  beforeEach(() => {
    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-'))
  })
  afterEach(() => {
    vi.restoreAllMocks()
    fs.rmSync(dir, { recursive: true, force: true })
  })

  it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => {
    let brain = await open()
    const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone
    for (let i = 0; i < 3; i++) {
      await brain.add({ data: `real ${i}`, type: 'document' })
    }
    await brain.flush()
    const realTotal = baseline + 3
    await brain.close()

    // 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) —
    // neither is a live entity.
    for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`)
    for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`)

    // Remove counts.json so open() re-derives from scratch (the one-time
    // legacy/lost-file derivation path).
    fs.rmSync(countsPath(dir), { force: true })

    brain = await open()
    const ledger = await brain.storage.getCanonicalCounts()
    expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing
    expect(ledger.suspect).toBe(false)

    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
    expect(raw.totalNounCountAll).toBe(realTotal)
    expect(raw.allCountsDerivedBy).toBe('identity-record')

    await brain.close()
  })

  it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => {
    let brain = await open()
    await brain.add({ data: 'one', type: 'document' })
    await brain.add({ data: 'two', type: 'document' })
    await brain.flush()
    await brain.close()

    // Confirm a normal close under the fix DOES stamp — then strip the stamp
    // to simulate a counts.json produced before this fix existed.
    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
    expect(raw.allCountsDerivedBy).toBe('identity-record')
    expect(typeof raw.totalNounCountAll).toBe('number')
    expect(typeof raw.totalVerbCountAll).toBe('number')
    expect(typeof raw.totalVectoredNounCount).toBe('number')
    delete raw.allCountsDerivedBy
    fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))

    const narrateSpy = vi.spyOn(prodLog, 'narrate')
    // The derivation walks live on FileSystemStorage's prototype. Slow them
    // deliberately: the OPEN must not wait for them, and on a two-row store a
    // real walk finishes too fast to tell "not awaited" from "instant".
    const proto = FileSystemStorage.prototype as any
    const realScanEntities = proto.scanCanonicalEntities
    let scanEntitiesCalls = 0
    proto.scanCanonicalEntities = async function slow(this: any, ...args: any[]) {
      scanEntitiesCalls++
      await new Promise((r) => setTimeout(r, 1_200))
      return realScanEntities.apply(this, args)
    }
    try {
      const openStarted = Date.now()
      brain = await open()
      const openMs = Date.now() - openStarted

      // THE OPEN DID NOT WALK: two slowed walks would have added 2.4s to it.
      expect(openMs).toBeLessThan(2_000)

      // The stamp check itself is an O(1) field read, and it names the cause.
      const atOpen = await brain.storage.getCanonicalCounts()
      expect(atOpen.suspect).toBe(true)
      const stampWarnings = narrateSpy.mock.calls.filter(
        ([msg]: any[]) => String(msg).includes('legacy') && String(msg).includes('container rule')
      )
      expect(stampWarnings.length).toBe(1) // exactly one, loud

      // ...and the honest derivation is already running behind the open.
      await brain.storage.whenCountLedgerSettled()
      expect(scanEntitiesCalls).toBeGreaterThan(0)
      const healed = await brain.storage.getCanonicalCounts()
      expect(healed.suspect).toBe(false)
      expect(healed.nouns.all).toBe(raw.totalNounCountAll)
    } finally {
      proto.scanCanonicalEntities = realScanEntities
    }

    await brain.close()
  })

  it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => {
    let brain = await open()
    const baseline = (await brain.storage.getCanonicalCounts()).nouns.all
    for (let i = 0; i < 3; i++) {
      await brain.add({ data: `real ${i}`, type: 'document' })
    }
    await brain.flush()
    const realTotal = baseline + 3
    await brain.close()

    for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`)
    for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`)

    // Force the legacy (unstamped, container-rule-inflated) shape directly —
    // the shape a pre-existing production store actually carries.
    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
    raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars
    delete raw.allCountsDerivedBy
    fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))

    brain = await open()
    // Named suspect at load, then healed in the background WITHOUT the
    // operator asking — the inflated container count is corrected to the
    // identity-record population, though the orphaned containers themselves
    // are still on disk (only repairIndex() removes those).
    await brain.storage.whenCountLedgerSettled()
    let healed = await brain.storage.getCanonicalCounts()
    expect(healed.suspect).toBe(false)
    expect(healed.nouns.all).toBe(realTotal)

    await brain.repairIndex()

    let ledger = await brain.storage.getCanonicalCounts()
    expect(ledger.suspect).toBe(false)
    expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again

    const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
    expect(persisted.allCountsDerivedBy).toBe('identity-record')
    expect(persisted.allCountsSuspect).toBe(false)
    expect(persisted.totalNounCountAll).toBe(realTotal)

    await brain.close()
    brain = await open()
    ledger = await brain.storage.getCanonicalCounts()
    expect(ledger.suspect).toBe(false)
    expect(ledger.nouns.all).toBe(realTotal)
    await brain.close()
  })

  it('(d) a fresh store derives with the stamp and is never suspect', async () => {
    const brain = await open()
    const ledger = await brain.storage.getCanonicalCounts()
    expect(ledger.suspect).toBe(false)
    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
    expect(raw.allCountsDerivedBy).toBe('identity-record')
    await brain.close()
  })
})