195 lines
8.5 KiB
TypeScript
195 lines
8.5 KiB
TypeScript
/**
|
|
* @module tests/integration/ledger-derivation-identity
|
|
* @description The ALL-visibility ledger scalars are an IDENTITY-RECORD
|
|
* count, never a container count. A pre-8.3.1 partial-delete defect can
|
|
* leave a "ghost" container (a stale `vectors.json` with no metadata content
|
|
* leg) or a "scar" container (an empty `entities/<kind>/<shard>/<id>/`
|
|
* directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need
|
|
* the metadata content leg — yet the legacy derivation counted one entity
|
|
* per id DIRECTORY, so orphaned containers inflated the ALL scalars forever
|
|
* (they were never clamped and never re-derived). Laws under test:
|
|
* (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per
|
|
* metadata content leg (`metadata.json` or `.json.gz`), the same test
|
|
* `pruneOrphanedEntities()` uses, so the two agree by construction.
|
|
* (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that
|
|
* carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'`
|
|
* stamp predates this fix; loading it marks `suspect = true` from a
|
|
* single field read alone, never a directory walk, and warns exactly
|
|
* once naming the cause.
|
|
* (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned
|
|
* containers, recounts from the canonical metadata.json walk, and
|
|
* re-stamps — suspect clears and the ALL scalar is exact again.
|
|
* (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store
|
|
* with no counts.json stamps as it writes, so a brand-new store never
|
|
* carries the legacy signature.
|
|
*/
|
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
import * as fs from 'node:fs'
|
|
import * as os from 'node:os'
|
|
import * as path from 'node:path'
|
|
import { Brainy, FileSystemStorage } from '../../src/index.js'
|
|
import { prodLog } from '../../src/utils/logger.js'
|
|
|
|
const countsPath = (root: string) => path.join(root, '_system', 'counts.json')
|
|
|
|
/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */
|
|
function plantGhost(root: string, shard: string, id: string): void {
|
|
const idDir = path.join(root, 'entities', 'nouns', shard, id)
|
|
fs.mkdirSync(idDir, { recursive: true })
|
|
fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] }))
|
|
}
|
|
|
|
/** Plant a scar container: an empty id directory, no legs at all. */
|
|
function plantScar(root: string, shard: string, id: string): void {
|
|
fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true })
|
|
}
|
|
|
|
describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => {
|
|
let dir: string
|
|
|
|
const open = async () => {
|
|
const b: any = new Brainy({
|
|
requireSubtype: false,
|
|
storage: { type: 'filesystem', path: dir },
|
|
silent: true,
|
|
dimensions: 384
|
|
})
|
|
await b.init()
|
|
return b
|
|
}
|
|
|
|
beforeEach(() => {
|
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-'))
|
|
})
|
|
afterEach(() => {
|
|
vi.restoreAllMocks()
|
|
fs.rmSync(dir, { recursive: true, force: true })
|
|
})
|
|
|
|
it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => {
|
|
let brain = await open()
|
|
const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone
|
|
for (let i = 0; i < 3; i++) {
|
|
await brain.add({ data: `real ${i}`, type: 'document' })
|
|
}
|
|
await brain.flush()
|
|
const realTotal = baseline + 3
|
|
await brain.close()
|
|
|
|
// 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) —
|
|
// neither is a live entity.
|
|
for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`)
|
|
for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`)
|
|
|
|
// Remove counts.json so open() re-derives from scratch (the one-time
|
|
// legacy/lost-file derivation path).
|
|
fs.rmSync(countsPath(dir), { force: true })
|
|
|
|
brain = await open()
|
|
const ledger = await brain.storage.getCanonicalCounts()
|
|
expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing
|
|
expect(ledger.suspect).toBe(false)
|
|
|
|
const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
|
expect(raw.totalNounCountAll).toBe(realTotal)
|
|
expect(raw.allCountsDerivedBy).toBe('identity-record')
|
|
|
|
await brain.close()
|
|
})
|
|
|
|
it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => {
|
|
let brain = await open()
|
|
await brain.add({ data: 'one', type: 'document' })
|
|
await brain.add({ data: 'two', type: 'document' })
|
|
await brain.flush()
|
|
await brain.close()
|
|
|
|
// Confirm a normal close under the fix DOES stamp — then strip the stamp
|
|
// to simulate a counts.json produced before this fix existed.
|
|
const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
|
expect(raw.allCountsDerivedBy).toBe('identity-record')
|
|
expect(typeof raw.totalNounCountAll).toBe('number')
|
|
expect(typeof raw.totalVerbCountAll).toBe('number')
|
|
expect(typeof raw.totalVectoredNounCount).toBe('number')
|
|
delete raw.allCountsDerivedBy
|
|
fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
|
|
|
|
const warnSpy = vi.spyOn(prodLog, 'warn')
|
|
// The two derivation walks live on FileSystemStorage's prototype —
|
|
// spying here (rather than on fs.promises.readdir globally) isolates
|
|
// THIS code path's behavior from unrelated walks elsewhere in the open
|
|
// sequence (a separate, pre-existing engine's own O(store) cost — not
|
|
// this fix's concern, and not something this pin should be sensitive
|
|
// to). Neither derivation method may run: the stamp check is a field
|
|
// read on the already-parsed counts.json, nothing more.
|
|
const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities')
|
|
const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount')
|
|
|
|
brain = await open()
|
|
|
|
const ledger = await brain.storage.getCanonicalCounts()
|
|
expect(ledger.suspect).toBe(true)
|
|
|
|
const stampWarnings = warnSpy.mock.calls.filter(
|
|
([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule')
|
|
)
|
|
expect(stampWarnings.length).toBe(1) // exactly one, loud
|
|
|
|
expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk
|
|
expect(scanVectoredSpy).not.toHaveBeenCalled()
|
|
|
|
await brain.close()
|
|
})
|
|
|
|
it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => {
|
|
let brain = await open()
|
|
const baseline = (await brain.storage.getCanonicalCounts()).nouns.all
|
|
for (let i = 0; i < 3; i++) {
|
|
await brain.add({ data: `real ${i}`, type: 'document' })
|
|
}
|
|
await brain.flush()
|
|
const realTotal = baseline + 3
|
|
await brain.close()
|
|
|
|
for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`)
|
|
for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`)
|
|
|
|
// Force the legacy (unstamped, container-rule-inflated) shape directly —
|
|
// the shape a pre-existing production store actually carries.
|
|
const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
|
raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars
|
|
delete raw.allCountsDerivedBy
|
|
fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
|
|
|
|
brain = await open()
|
|
expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load
|
|
|
|
await brain.repairIndex()
|
|
|
|
let ledger = await brain.storage.getCanonicalCounts()
|
|
expect(ledger.suspect).toBe(false)
|
|
expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again
|
|
|
|
const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
|
expect(persisted.allCountsDerivedBy).toBe('identity-record')
|
|
expect(persisted.allCountsSuspect).toBe(false)
|
|
expect(persisted.totalNounCountAll).toBe(realTotal)
|
|
|
|
await brain.close()
|
|
brain = await open()
|
|
ledger = await brain.storage.getCanonicalCounts()
|
|
expect(ledger.suspect).toBe(false)
|
|
expect(ledger.nouns.all).toBe(realTotal)
|
|
await brain.close()
|
|
})
|
|
|
|
it('(d) a fresh store derives with the stamp and is never suspect', async () => {
|
|
const brain = await open()
|
|
const ledger = await brain.storage.getCanonicalCounts()
|
|
expect(ledger.suspect).toBe(false)
|
|
const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
|
expect(raw.allCountsDerivedBy).toBe('identity-record')
|
|
await brain.close()
|
|
})
|
|
})
|