fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load

tests/integration/ledger-derivation-identity.test.ts

@@ -0,0 +1,195 @@
+/**
+ * @module tests/integration/ledger-derivation-identity
+ * @description The ALL-visibility ledger scalars are an IDENTITY-RECORD
+ * count, never a container count. A pre-8.3.1 partial-delete defect can
+ * leave a "ghost" container (a stale `vectors.json` with no metadata content
+ * leg) or a "scar" container (an empty `entities/<kind>/<shard>/<id>/`
+ * directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need
+ * the metadata content leg — yet the legacy derivation counted one entity
+ * per id DIRECTORY, so orphaned containers inflated the ALL scalars forever
+ * (they were never clamped and never re-derived). Laws under test:
+ * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per
+ *     metadata content leg (`metadata.json` or `.json.gz`), the same test
+ *     `pruneOrphanedEntities()` uses, so the two agree by construction.
+ * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that
+ *     carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'`
+ *     stamp predates this fix; loading it marks `suspect = true` from a
+ *     single field read alone, never a directory walk, and warns exactly
+ *     once naming the cause.
+ * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned
+ *     containers, recounts from the canonical metadata.json walk, and
+ *     re-stamps — suspect clears and the ALL scalar is exact again.
+ * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store
+ *     with no counts.json stamps as it writes, so a brand-new store never
+ *     carries the legacy signature.
+ */
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy, FileSystemStorage } from '../../src/index.js'
+import { prodLog } from '../../src/utils/logger.js'
+
+const countsPath = (root: string) => path.join(root, '_system', 'counts.json')
+
+/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */
+function plantGhost(root: string, shard: string, id: string): void {
+  const idDir = path.join(root, 'entities', 'nouns', shard, id)
+  fs.mkdirSync(idDir, { recursive: true })
+  fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] }))
+}
+
+/** Plant a scar container: an empty id directory, no legs at all. */
+function plantScar(root: string, shard: string, id: string): void {
+  fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true })
+}
+
+describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => {
+  let dir: string
+
+  const open = async () => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(() => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-'))
+  })
+  afterEach(() => {
+    vi.restoreAllMocks()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => {
+    let brain = await open()
+    const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone
+    for (let i = 0; i < 3; i++) {
+      await brain.add({ data: `real ${i}`, type: 'document' })
+    }
+    await brain.flush()
+    const realTotal = baseline + 3
+    await brain.close()
+
+    // 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) —
+    // neither is a live entity.
+    for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`)
+    for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`)
+
+    // Remove counts.json so open() re-derives from scratch (the one-time
+    // legacy/lost-file derivation path).
+    fs.rmSync(countsPath(dir), { force: true })
+
+    brain = await open()
+    const ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing
+    expect(ledger.suspect).toBe(false)
+
+    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    expect(raw.totalNounCountAll).toBe(realTotal)
+    expect(raw.allCountsDerivedBy).toBe('identity-record')
+
+    await brain.close()
+  })
+
+  it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => {
+    let brain = await open()
+    await brain.add({ data: 'one', type: 'document' })
+    await brain.add({ data: 'two', type: 'document' })
+    await brain.flush()
+    await brain.close()
+
+    // Confirm a normal close under the fix DOES stamp — then strip the stamp
+    // to simulate a counts.json produced before this fix existed.
+    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    expect(raw.allCountsDerivedBy).toBe('identity-record')
+    expect(typeof raw.totalNounCountAll).toBe('number')
+    expect(typeof raw.totalVerbCountAll).toBe('number')
+    expect(typeof raw.totalVectoredNounCount).toBe('number')
+    delete raw.allCountsDerivedBy
+    fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
+
+    const warnSpy = vi.spyOn(prodLog, 'warn')
+    // The two derivation walks live on FileSystemStorage's prototype —
+    // spying here (rather than on fs.promises.readdir globally) isolates
+    // THIS code path's behavior from unrelated walks elsewhere in the open
+    // sequence (a separate, pre-existing engine's own O(store) cost — not
+    // this fix's concern, and not something this pin should be sensitive
+    // to). Neither derivation method may run: the stamp check is a field
+    // read on the already-parsed counts.json, nothing more.
+    const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities')
+    const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount')
+
+    brain = await open()
+
+    const ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.suspect).toBe(true)
+
+    const stampWarnings = warnSpy.mock.calls.filter(
+      ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule')
+    )
+    expect(stampWarnings.length).toBe(1) // exactly one, loud
+
+    expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk
+    expect(scanVectoredSpy).not.toHaveBeenCalled()
+
+    await brain.close()
+  })
+
+  it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => {
+    let brain = await open()
+    const baseline = (await brain.storage.getCanonicalCounts()).nouns.all
+    for (let i = 0; i < 3; i++) {
+      await brain.add({ data: `real ${i}`, type: 'document' })
+    }
+    await brain.flush()
+    const realTotal = baseline + 3
+    await brain.close()
+
+    for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`)
+    for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`)
+
+    // Force the legacy (unstamped, container-rule-inflated) shape directly —
+    // the shape a pre-existing production store actually carries.
+    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars
+    delete raw.allCountsDerivedBy
+    fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
+
+    brain = await open()
+    expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load
+
+    await brain.repairIndex()
+
+    let ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.suspect).toBe(false)
+    expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again
+
+    const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    expect(persisted.allCountsDerivedBy).toBe('identity-record')
+    expect(persisted.allCountsSuspect).toBe(false)
+    expect(persisted.totalNounCountAll).toBe(realTotal)
+
+    await brain.close()
+    brain = await open()
+    ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.suspect).toBe(false)
+    expect(ledger.nouns.all).toBe(realTotal)
+    await brain.close()
+  })
+
+  it('(d) a fresh store derives with the stamp and is never suspect', async () => {
+    const brain = await open()
+    const ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.suspect).toBe(false)
+    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    expect(raw.allCountsDerivedBy).toBe('identity-record')
+    await brain.close()
+  })
+})