open-brainy/tests/integration/count-ledger-identity-record.test.ts

/**
 * @module tests/integration/count-ledger-identity-record
 * @description THE COUNT LEDGER COUNTS RECORDS, NOT DIRECTORIES — and heals
 * itself when it was derived the other way.
 *
 * Measured on a real store: the ALL-visibility ledger read 14,231 nouns
 * against 14,056 identity records, and 72,729 verbs against 72,679 — exactly
 * that store's 25 noun and 50 verb SCAR directories (empty `<id>/` containers
 * left by a pre-8.3.1 partial delete). Two copies of the SAME archive derived
 * different numbers, because each had been persisted at a different moment
 * under the old container rule. A downstream index heal subtracted against
 * those denominators and reported remaining work that did not exist.
 *
 * The membership predicate is the IDENTITY RECORD (the metadata content leg).
 * The scan already applies it; what is pinned here is that a ledger persisted
 * under the OLD rule does not go on lying — it is corrected in the background,
 * without blocking the open, and two copies of one archive agree.
 */

import { describe, it, expect, afterEach } from 'vitest'
import {
  mkdtempSync,
  mkdirSync,
  rmSync,
  writeFileSync,
  readFileSync,
  cpSync,
  existsSync
} from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Brainy } from '../../src/brainy.js'
import { NounType } from '../../src/types/graphTypes.js'
import { FileSystemStorage as FileSystemStorageClass } from '../../src/storage/adapters/fileSystemStorage.js'
import type { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'

const NOUN_COUNT = 6
const NOUN_SCARS = 3
const VERB_SCARS = 2
/** A REAL two-hex shard — the scan skips any directory that is not one. */
const SCAR_SHARD = 'ab'

function makeTempDir(): string {
  return mkdtempSync(join(tmpdir(), 'brainy-count-ledger-'))
}

/** The FileSystemStorage behind a brain. */
function storageOf(brain: Brainy): FileSystemStorage {
  return (brain as unknown as { storage: FileSystemStorage }).storage
}

/**
 * Add `count` empty `<id>/` container directories under
 * `entities/<kind>/<shard>/` — scars, exactly as a partial delete leaves them.
 */
function addScarContainers(dir: string, kind: 'nouns' | 'verbs', count: number): void {
  for (let i = 0; i < count; i++) {
    const id = `${SCAR_SHARD}5ca4000-0000-0000-0000-00000000000${i}`
    mkdirSync(join(dir, 'entities', kind, SCAR_SHARD, id), { recursive: true })
  }
}

/** Add one GHOST container: a `vectors.json` leg with no identity record. */
function addGhostContainer(dir: string): void {
  const id = `${SCAR_SHARD}9405700-0000-0000-0000-000000000000`
  const idDir = join(dir, 'entities', 'nouns', SCAR_SHARD, id)
  mkdirSync(idDir, { recursive: true })
  writeFileSync(join(idDir, 'vectors.json'), JSON.stringify({ id, vector: [0.1, 0.2] }))
}

/**
 * Rewrite counts.json into the LEGACY shape: ALL scalars inflated by the
 * containers, and no `allCountsDerivedBy` stamp — exactly what a store carried
 * when it was last written by a build that counted directories.
 */
function writeLegacyCountsLedger(dir: string, inflateNouns: number, inflateVerbs: number): void {
  const file = join(dir, '_system', 'counts.json')
  const counts = JSON.parse(readFileSync(file, 'utf-8'))
  counts.totalNounCountAll = (counts.totalNounCountAll ?? 0) + inflateNouns
  counts.totalVerbCountAll = (counts.totalVerbCountAll ?? 0) + inflateVerbs
  delete counts.allCountsDerivedBy
  delete counts.allCountsSuspect
  writeFileSync(file, JSON.stringify(counts, null, 2))
}

/**
 * Seed a store and return the HONEST ledger it holds when freshly written —
 * the baseline the correction must return to. Read from the engine rather than
 * hardcoded: an open creates its own rows (the VFS root), and a pin that
 * asserts a literal would be pinning that incidental fact instead of the rule.
 */
async function seedStore(dir: string): Promise<{ nouns: number; verbs: number }> {
  const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
  await brain.init()
  const ids: string[] = []
  for (let i = 0; i < NOUN_COUNT; i++) {
    ids.push(await brain.add({ data: `entity number ${i}`, type: NounType.Concept }))
  }
  await brain.relate({ from: ids[0], to: ids[1], type: 'relatedTo' } as never)
  await brain.relate({ from: ids[1], to: ids[2], type: 'relatedTo' } as never)
  await brain.flush()
  const ledger = await storageOf(brain).getCanonicalCounts()
  const baseline = { nouns: ledger.nouns.all, verbs: ledger.verbs.all }
  await brain.close()
  return baseline
}

/**
 * Make the ledger walk take `ms` so a test can observe the open completing
 * WITHOUT it. Patches the prototype before any brain is constructed; returns
 * the restore function.
 */
function slowTheLedgerWalk(ms: number): () => void {
  const proto = (
    FileSystemStorageClass as unknown as {
      prototype: Record<string, (...args: unknown[]) => Promise<unknown>>
    }
  ).prototype
  const real = proto.scanCanonicalEntities
  proto.scanCanonicalEntities = async function slow(this: unknown, ...args: unknown[]) {
    await new Promise((r) => setTimeout(r, ms))
    return real.apply(this, args)
  }
  return () => { proto.scanCanonicalEntities = real }
}

describe('the canonical count ledger', () => {
  const dirs: string[] = []

  afterEach(() => {
    for (const d of dirs.splice(0)) {
      try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ }
    }
  })

  function trackDir(): string {
    const dir = makeTempDir()
    dirs.push(dir)
    return dir
  }

  it('corrects a legacy container-rule ledger in the background, counting identity records', async () => {
    const dir = trackDir()
    const baseline = await seedStore(dir)

    // Scars and a ghost: containers with no identity record.
    addScarContainers(dir, 'nouns', NOUN_SCARS)
    addScarContainers(dir, 'verbs', VERB_SCARS)
    addGhostContainer(dir)
    // The ledger as the old rule left it: every container counted.
    writeLegacyCountsLedger(dir, NOUN_SCARS + 1, VERB_SCARS)

    const restore = slowTheLedgerWalk(1_500)
    let brain: Brainy
    try {
      const openStarted = Date.now()
      brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
      await brain.init()
      const openMs = Date.now() - openStarted
      const storage = storageOf(brain)

      // THE OPEN DID NOT WAIT. Two walks of 1.5s each would have added 3s.
      expect(openMs).toBeLessThan(2_500)
      // And while it runs, the scalars say so instead of being subtracted against.
      const atOpen = await storage.getCanonicalCounts()
      expect(atOpen.suspect).toBe(true)
      expect(atOpen.nouns.all).toBe(baseline.nouns + NOUN_SCARS + 1)

      await storage.whenCountLedgerSettled()
    } finally {
      restore()
    }
    const storage = storageOf(brain!)

    const healed = await storage.getCanonicalCounts()
    expect(healed.nouns.all).toBe(baseline.nouns)
    expect(healed.verbs.all).toBe(baseline.verbs)
    expect(healed.suspect).toBe(false)

    // And it is PERSISTED with the honest stamp — the correction survives a
    // reopen instead of being re-derived (or re-lost) every time.
    await brain!.close()
    const persisted = JSON.parse(readFileSync(join(dir, '_system', 'counts.json'), 'utf-8'))
    expect(persisted.totalNounCountAll).toBe(baseline.nouns)
    expect(persisted.totalVerbCountAll).toBe(baseline.verbs)
    expect(persisted.allCountsDerivedBy).toBe('identity-record')

    const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
    await reopened.init()
    const afterReopen = await storageOf(reopened).getCanonicalCounts()
    expect(afterReopen.nouns.all).toBe(baseline.nouns)
    expect(afterReopen.suspect).toBe(false)
    await reopened.close()
  }, 180_000)

  it('derives the same number from two copies of one archive', async () => {
    const source = trackDir()
    const baseline = await seedStore(source)
    addScarContainers(source, 'nouns', NOUN_SCARS)
    addGhostContainer(source)

    // Two copies of the SAME bytes, each carrying a DIFFERENT legacy ledger —
    // the situation that made one archive report 14,231 and its twin 14,081.
    const copyA = trackDir()
    const copyB = trackDir()
    cpSync(source, copyA, { recursive: true })
    cpSync(source, copyB, { recursive: true })
    writeLegacyCountsLedger(copyA, NOUN_SCARS + 1, 0)
    writeLegacyCountsLedger(copyB, 1, 0)

    const derived: number[] = []
    for (const dir of [copyA, copyB]) {
      const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
      await brain.init()
      const storage = storageOf(brain)
      await storage.whenCountLedgerSettled()
      derived.push((await storage.getCanonicalCounts()).nouns.all)
      await brain.close()
    }
    expect(derived[0]).toBe(derived[1])
    expect(derived[0]).toBe(baseline.nouns)
  }, 180_000)

  it('writes counts.json atomically — no reader ever sees it empty', async () => {
    const dir = trackDir()
    await seedStore(dir)
    const file = join(dir, '_system', 'counts.json')
    expect(existsSync(file)).toBe(true)

    const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
    await brain.init()
    const storage = storageOf(brain)

    // Watch the ledger across many persists. A truncating write leaves a
    // window in which the file parses as nothing; a temp+rename never does.
    let sawUnparseable = 0
    const watcher = setInterval(() => {
      try {
        JSON.parse(readFileSync(file, 'utf-8'))
      } catch {
        sawUnparseable++
      }
    }, 1)
    for (let i = 0; i < 40; i++) {
      await (storage as unknown as { persistCounts: () => Promise<void> }).persistCounts()
    }
    clearInterval(watcher)
    await brain.close()
    expect(sawUnparseable).toBe(0)
  }, 180_000)
})