fix(storage): a suspect count ledger heals itself, and counts.json is written atomically
MEASURED on a real store: the ALL-visibility ledger read 14,231 nouns against 14,056 identity records and 72,729 verbs against 72,679 — exactly that store's 25 noun and 50 verb SCAR directories. Two copies of the same archive derived different numbers (14,231 and 14,081), because each had been persisted at a different moment under the old rule that counted one entity per id DIRECTORY. A downstream index heal subtracted against that denominator and reported remaining work that did not exist. The scan already applies the right predicate — one entity per IDENTITY RECORD (the metadata content leg), shared with pruneOrphanedEntities so the two agree by construction. What was missing is that a ledger persisted under the old rule was only FLAGGED suspect and then went on serving its wrong numbers for the life of the store, waiting for an operator to run repairIndex. - The ledger now derives itself honestly in the BACKGROUND after the open, narrating start and finish with the correction it made. Background because these scalars are denominators — no read is served from them — and because walks exactly like these are how a 24,898-id store spent minutes of a restart in silence. Observable via whenCountLedgerSettled(); nothing in the read path waits on it. - A derivation that raced a write refuses to stamp its number "exact": one retry on a quiet store, then the ledger stays SUSPECT and says so, naming repairIndex as the door that recounts under a barrier. - The one derivation that CANNOT leave the foreground says why it cannot: getNounCount()/getVerbCount() are served from it, and a background walk would make a populated store answer "0 entities" — a wrong answer, not a slow one. It narrates its start and its wall instead. - counts.json is written temp+rename. A truncating write left a window — measured at roughly 750ms after a flush or close — in which a concurrent reader saw the file EMPTY; an unparseable ledger sends the next open down the full-rescan path, so the cheapest file in the store was buying the most expensive recovery. - The writer lock's clean-close record is now consulted before the same-process branch too: a restart reported "Re-acquiring writer lock ... this is a bug" immediately after a clean close, sending an operator after a leak that did not exist. Pins: tests/integration/count-ledger-identity-record.test.ts (background correction with scar and ghost fixtures, two copies of one archive agreeing, counts.json never observed unparseable across 40 persists); tests/integration/ledger-derivation-identity.test.ts updated to the new law — the OPEN still never walks (proved by slowing the walk 1.2s and timing the open), and the ledger heals behind it.
This commit is contained in:
parent
afe08a1ff9
commit
f4e2d34b4e
3 changed files with 519 additions and 72 deletions
251
tests/integration/count-ledger-identity-record.test.ts
Normal file
251
tests/integration/count-ledger-identity-record.test.ts
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
/**
|
||||
* @module tests/integration/count-ledger-identity-record
|
||||
* @description THE COUNT LEDGER COUNTS RECORDS, NOT DIRECTORIES — and heals
|
||||
* itself when it was derived the other way.
|
||||
*
|
||||
* Measured on a real store: the ALL-visibility ledger read 14,231 nouns
|
||||
* against 14,056 identity records, and 72,729 verbs against 72,679 — exactly
|
||||
* that store's 25 noun and 50 verb SCAR directories (empty `<id>/` containers
|
||||
* left by a pre-8.3.1 partial delete). Two copies of the SAME archive derived
|
||||
* different numbers, because each had been persisted at a different moment
|
||||
* under the old container rule. A downstream index heal subtracted against
|
||||
* those denominators and reported remaining work that did not exist.
|
||||
*
|
||||
* The membership predicate is the IDENTITY RECORD (the metadata content leg).
|
||||
* The scan already applies it; what is pinned here is that a ledger persisted
|
||||
* under the OLD rule does not go on lying — it is corrected in the background,
|
||||
* without blocking the open, and two copies of one archive agree.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, afterEach } from 'vitest'
|
||||
import {
|
||||
mkdtempSync,
|
||||
mkdirSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
readFileSync,
|
||||
cpSync,
|
||||
existsSync
|
||||
} from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { Brainy } from '../../src/brainy.js'
|
||||
import { NounType } from '../../src/types/graphTypes.js'
|
||||
import { FileSystemStorage as FileSystemStorageClass } from '../../src/storage/adapters/fileSystemStorage.js'
|
||||
import type { FileSystemStorage } from '../../src/storage/adapters/fileSystemStorage.js'
|
||||
|
||||
const NOUN_COUNT = 6
|
||||
const NOUN_SCARS = 3
|
||||
const VERB_SCARS = 2
|
||||
/** A REAL two-hex shard — the scan skips any directory that is not one. */
|
||||
const SCAR_SHARD = 'ab'
|
||||
|
||||
function makeTempDir(): string {
|
||||
return mkdtempSync(join(tmpdir(), 'brainy-count-ledger-'))
|
||||
}
|
||||
|
||||
/** The FileSystemStorage behind a brain. */
|
||||
function storageOf(brain: Brainy): FileSystemStorage {
|
||||
return (brain as unknown as { storage: FileSystemStorage }).storage
|
||||
}
|
||||
|
||||
/**
|
||||
* Add `count` empty `<id>/` container directories under
|
||||
* `entities/<kind>/<shard>/` — scars, exactly as a partial delete leaves them.
|
||||
*/
|
||||
function addScarContainers(dir: string, kind: 'nouns' | 'verbs', count: number): void {
|
||||
for (let i = 0; i < count; i++) {
|
||||
const id = `${SCAR_SHARD}5ca4000-0000-0000-0000-00000000000${i}`
|
||||
mkdirSync(join(dir, 'entities', kind, SCAR_SHARD, id), { recursive: true })
|
||||
}
|
||||
}
|
||||
|
||||
/** Add one GHOST container: a `vectors.json` leg with no identity record. */
|
||||
function addGhostContainer(dir: string): void {
|
||||
const id = `${SCAR_SHARD}9405700-0000-0000-0000-000000000000`
|
||||
const idDir = join(dir, 'entities', 'nouns', SCAR_SHARD, id)
|
||||
mkdirSync(idDir, { recursive: true })
|
||||
writeFileSync(join(idDir, 'vectors.json'), JSON.stringify({ id, vector: [0.1, 0.2] }))
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite counts.json into the LEGACY shape: ALL scalars inflated by the
|
||||
* containers, and no `allCountsDerivedBy` stamp — exactly what a store carried
|
||||
* when it was last written by a build that counted directories.
|
||||
*/
|
||||
function writeLegacyCountsLedger(dir: string, inflateNouns: number, inflateVerbs: number): void {
|
||||
const file = join(dir, '_system', 'counts.json')
|
||||
const counts = JSON.parse(readFileSync(file, 'utf-8'))
|
||||
counts.totalNounCountAll = (counts.totalNounCountAll ?? 0) + inflateNouns
|
||||
counts.totalVerbCountAll = (counts.totalVerbCountAll ?? 0) + inflateVerbs
|
||||
delete counts.allCountsDerivedBy
|
||||
delete counts.allCountsSuspect
|
||||
writeFileSync(file, JSON.stringify(counts, null, 2))
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed a store and return the HONEST ledger it holds when freshly written —
|
||||
* the baseline the correction must return to. Read from the engine rather than
|
||||
* hardcoded: an open creates its own rows (the VFS root), and a pin that
|
||||
* asserts a literal would be pinning that incidental fact instead of the rule.
|
||||
*/
|
||||
async function seedStore(dir: string): Promise<{ nouns: number; verbs: number }> {
|
||||
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||
await brain.init()
|
||||
const ids: string[] = []
|
||||
for (let i = 0; i < NOUN_COUNT; i++) {
|
||||
ids.push(await brain.add({ data: `entity number ${i}`, type: NounType.Concept }))
|
||||
}
|
||||
await brain.relate({ from: ids[0], to: ids[1], type: 'relatedTo' } as never)
|
||||
await brain.relate({ from: ids[1], to: ids[2], type: 'relatedTo' } as never)
|
||||
await brain.flush()
|
||||
const ledger = await storageOf(brain).getCanonicalCounts()
|
||||
const baseline = { nouns: ledger.nouns.all, verbs: ledger.verbs.all }
|
||||
await brain.close()
|
||||
return baseline
|
||||
}
|
||||
|
||||
/**
|
||||
* Make the ledger walk take `ms` so a test can observe the open completing
|
||||
* WITHOUT it. Patches the prototype before any brain is constructed; returns
|
||||
* the restore function.
|
||||
*/
|
||||
function slowTheLedgerWalk(ms: number): () => void {
|
||||
const proto = (
|
||||
FileSystemStorageClass as unknown as {
|
||||
prototype: Record<string, (...args: unknown[]) => Promise<unknown>>
|
||||
}
|
||||
).prototype
|
||||
const real = proto.scanCanonicalEntities
|
||||
proto.scanCanonicalEntities = async function slow(this: unknown, ...args: unknown[]) {
|
||||
await new Promise((r) => setTimeout(r, ms))
|
||||
return real.apply(this, args)
|
||||
}
|
||||
return () => { proto.scanCanonicalEntities = real }
|
||||
}
|
||||
|
||||
describe('the canonical count ledger', () => {
|
||||
const dirs: string[] = []
|
||||
|
||||
afterEach(() => {
|
||||
for (const d of dirs.splice(0)) {
|
||||
try { rmSync(d, { recursive: true, force: true }) } catch { /* ignore */ }
|
||||
}
|
||||
})
|
||||
|
||||
function trackDir(): string {
|
||||
const dir = makeTempDir()
|
||||
dirs.push(dir)
|
||||
return dir
|
||||
}
|
||||
|
||||
it('corrects a legacy container-rule ledger in the background, counting identity records', async () => {
|
||||
const dir = trackDir()
|
||||
const baseline = await seedStore(dir)
|
||||
|
||||
// Scars and a ghost: containers with no identity record.
|
||||
addScarContainers(dir, 'nouns', NOUN_SCARS)
|
||||
addScarContainers(dir, 'verbs', VERB_SCARS)
|
||||
addGhostContainer(dir)
|
||||
// The ledger as the old rule left it: every container counted.
|
||||
writeLegacyCountsLedger(dir, NOUN_SCARS + 1, VERB_SCARS)
|
||||
|
||||
const restore = slowTheLedgerWalk(1_500)
|
||||
let brain: Brainy
|
||||
try {
|
||||
const openStarted = Date.now()
|
||||
brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||
await brain.init()
|
||||
const openMs = Date.now() - openStarted
|
||||
const storage = storageOf(brain)
|
||||
|
||||
// THE OPEN DID NOT WAIT. Two walks of 1.5s each would have added 3s.
|
||||
expect(openMs).toBeLessThan(2_500)
|
||||
// And while it runs, the scalars say so instead of being subtracted against.
|
||||
const atOpen = await storage.getCanonicalCounts()
|
||||
expect(atOpen.suspect).toBe(true)
|
||||
expect(atOpen.nouns.all).toBe(baseline.nouns + NOUN_SCARS + 1)
|
||||
|
||||
await storage.whenCountLedgerSettled()
|
||||
} finally {
|
||||
restore()
|
||||
}
|
||||
const storage = storageOf(brain!)
|
||||
|
||||
const healed = await storage.getCanonicalCounts()
|
||||
expect(healed.nouns.all).toBe(baseline.nouns)
|
||||
expect(healed.verbs.all).toBe(baseline.verbs)
|
||||
expect(healed.suspect).toBe(false)
|
||||
|
||||
// And it is PERSISTED with the honest stamp — the correction survives a
|
||||
// reopen instead of being re-derived (or re-lost) every time.
|
||||
await brain!.close()
|
||||
const persisted = JSON.parse(readFileSync(join(dir, '_system', 'counts.json'), 'utf-8'))
|
||||
expect(persisted.totalNounCountAll).toBe(baseline.nouns)
|
||||
expect(persisted.totalVerbCountAll).toBe(baseline.verbs)
|
||||
expect(persisted.allCountsDerivedBy).toBe('identity-record')
|
||||
|
||||
const reopened = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||
await reopened.init()
|
||||
const afterReopen = await storageOf(reopened).getCanonicalCounts()
|
||||
expect(afterReopen.nouns.all).toBe(baseline.nouns)
|
||||
expect(afterReopen.suspect).toBe(false)
|
||||
await reopened.close()
|
||||
}, 180_000)
|
||||
|
||||
it('derives the same number from two copies of one archive', async () => {
|
||||
const source = trackDir()
|
||||
const baseline = await seedStore(source)
|
||||
addScarContainers(source, 'nouns', NOUN_SCARS)
|
||||
addGhostContainer(source)
|
||||
|
||||
// Two copies of the SAME bytes, each carrying a DIFFERENT legacy ledger —
|
||||
// the situation that made one archive report 14,231 and its twin 14,081.
|
||||
const copyA = trackDir()
|
||||
const copyB = trackDir()
|
||||
cpSync(source, copyA, { recursive: true })
|
||||
cpSync(source, copyB, { recursive: true })
|
||||
writeLegacyCountsLedger(copyA, NOUN_SCARS + 1, 0)
|
||||
writeLegacyCountsLedger(copyB, 1, 0)
|
||||
|
||||
const derived: number[] = []
|
||||
for (const dir of [copyA, copyB]) {
|
||||
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||
await brain.init()
|
||||
const storage = storageOf(brain)
|
||||
await storage.whenCountLedgerSettled()
|
||||
derived.push((await storage.getCanonicalCounts()).nouns.all)
|
||||
await brain.close()
|
||||
}
|
||||
expect(derived[0]).toBe(derived[1])
|
||||
expect(derived[0]).toBe(baseline.nouns)
|
||||
}, 180_000)
|
||||
|
||||
it('writes counts.json atomically — no reader ever sees it empty', async () => {
|
||||
const dir = trackDir()
|
||||
await seedStore(dir)
|
||||
const file = join(dir, '_system', 'counts.json')
|
||||
expect(existsSync(file)).toBe(true)
|
||||
|
||||
const brain = new Brainy({ requireSubtype: false, storage: { type: 'filesystem', path: dir } })
|
||||
await brain.init()
|
||||
const storage = storageOf(brain)
|
||||
|
||||
// Watch the ledger across many persists. A truncating write leaves a
|
||||
// window in which the file parses as nothing; a temp+rename never does.
|
||||
let sawUnparseable = 0
|
||||
const watcher = setInterval(() => {
|
||||
try {
|
||||
JSON.parse(readFileSync(file, 'utf-8'))
|
||||
} catch {
|
||||
sawUnparseable++
|
||||
}
|
||||
}, 1)
|
||||
for (let i = 0; i < 40; i++) {
|
||||
await (storage as unknown as { persistCounts: () => Promise<void> }).persistCounts()
|
||||
}
|
||||
clearInterval(watcher)
|
||||
await brain.close()
|
||||
expect(sawUnparseable).toBe(0)
|
||||
}, 180_000)
|
||||
})
|
||||
|
|
@ -11,14 +11,22 @@
|
|||
* (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per
|
||||
* metadata content leg (`metadata.json` or `.json.gz`), the same test
|
||||
* `pruneOrphanedEntities()` uses, so the two agree by construction.
|
||||
* (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that
|
||||
* carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'`
|
||||
* stamp predates this fix; loading it marks `suspect = true` from a
|
||||
* single field read alone, never a directory walk, and warns exactly
|
||||
* once naming the cause.
|
||||
* (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned
|
||||
* containers, recounts from the canonical metadata.json walk, and
|
||||
* re-stamps — suspect clears and the ALL scalar is exact again.
|
||||
* (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AND THE OPEN NEVER WALKS — a
|
||||
* counts.json that carries the ALL scalars but no
|
||||
* `allCountsDerivedBy: 'identity-record'` stamp predates this fix;
|
||||
* loading it marks `suspect = true` from a single field read alone and
|
||||
* warns exactly once naming the cause. The open itself never pays a
|
||||
* directory walk.
|
||||
* (2b) AND IT HEALS ITSELF. The ledger used to stay wrong for the life of the
|
||||
* store, waiting for an operator to run `repairIndex()` — and a
|
||||
* downstream index heal subtracted against the inflated denominator and
|
||||
* reported work that did not exist. An honest derivation now runs in the
|
||||
* BACKGROUND after the open (never blocking it, observable via
|
||||
* `whenCountLedgerSettled()`), and refuses to stamp a number it derived
|
||||
* while writes were landing.
|
||||
* (3) THE SANCTIONED RECOUNT ALSO CLEARS IT — `repairIndex()` prunes the
|
||||
* orphaned containers, recounts from the canonical metadata.json walk,
|
||||
* and re-stamps — the ALL scalar is exact and the containers are gone.
|
||||
* (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store
|
||||
* with no counts.json stamps as it writes, so a brand-new store never
|
||||
* carries the legacy signature.
|
||||
|
|
@ -115,29 +123,43 @@ describe('ledger derivation identity — the ALL scalar is the identity-record p
|
|||
delete raw.allCountsDerivedBy
|
||||
fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
|
||||
|
||||
const warnSpy = vi.spyOn(prodLog, 'warn')
|
||||
// The two derivation walks live on FileSystemStorage's prototype —
|
||||
// spying here (rather than on fs.promises.readdir globally) isolates
|
||||
// THIS code path's behavior from unrelated walks elsewhere in the open
|
||||
// sequence (a separate, pre-existing engine's own O(store) cost — not
|
||||
// this fix's concern, and not something this pin should be sensitive
|
||||
// to). Neither derivation method may run: the stamp check is a field
|
||||
// read on the already-parsed counts.json, nothing more.
|
||||
const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities')
|
||||
const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount')
|
||||
const narrateSpy = vi.spyOn(prodLog, 'narrate')
|
||||
// The derivation walks live on FileSystemStorage's prototype. Slow them
|
||||
// deliberately: the OPEN must not wait for them, and on a two-row store a
|
||||
// real walk finishes too fast to tell "not awaited" from "instant".
|
||||
const proto = FileSystemStorage.prototype as any
|
||||
const realScanEntities = proto.scanCanonicalEntities
|
||||
let scanEntitiesCalls = 0
|
||||
proto.scanCanonicalEntities = async function slow(this: any, ...args: any[]) {
|
||||
scanEntitiesCalls++
|
||||
await new Promise((r) => setTimeout(r, 1_200))
|
||||
return realScanEntities.apply(this, args)
|
||||
}
|
||||
try {
|
||||
const openStarted = Date.now()
|
||||
brain = await open()
|
||||
const openMs = Date.now() - openStarted
|
||||
|
||||
brain = await open()
|
||||
// THE OPEN DID NOT WALK: two slowed walks would have added 2.4s to it.
|
||||
expect(openMs).toBeLessThan(2_000)
|
||||
|
||||
const ledger = await brain.storage.getCanonicalCounts()
|
||||
expect(ledger.suspect).toBe(true)
|
||||
// The stamp check itself is an O(1) field read, and it names the cause.
|
||||
const atOpen = await brain.storage.getCanonicalCounts()
|
||||
expect(atOpen.suspect).toBe(true)
|
||||
const stampWarnings = narrateSpy.mock.calls.filter(
|
||||
([msg]: any[]) => String(msg).includes('legacy') && String(msg).includes('container rule')
|
||||
)
|
||||
expect(stampWarnings.length).toBe(1) // exactly one, loud
|
||||
|
||||
const stampWarnings = warnSpy.mock.calls.filter(
|
||||
([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule')
|
||||
)
|
||||
expect(stampWarnings.length).toBe(1) // exactly one, loud
|
||||
|
||||
expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk
|
||||
expect(scanVectoredSpy).not.toHaveBeenCalled()
|
||||
// ...and the honest derivation is already running behind the open.
|
||||
await brain.storage.whenCountLedgerSettled()
|
||||
expect(scanEntitiesCalls).toBeGreaterThan(0)
|
||||
const healed = await brain.storage.getCanonicalCounts()
|
||||
expect(healed.suspect).toBe(false)
|
||||
expect(healed.nouns.all).toBe(raw.totalNounCountAll)
|
||||
} finally {
|
||||
proto.scanCanonicalEntities = realScanEntities
|
||||
}
|
||||
|
||||
await brain.close()
|
||||
})
|
||||
|
|
@ -163,7 +185,14 @@ describe('ledger derivation identity — the ALL scalar is the identity-record p
|
|||
fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
|
||||
|
||||
brain = await open()
|
||||
expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load
|
||||
// Named suspect at load, then healed in the background WITHOUT the
|
||||
// operator asking — the inflated container count is corrected to the
|
||||
// identity-record population, though the orphaned containers themselves
|
||||
// are still on disk (only repairIndex() removes those).
|
||||
await brain.storage.whenCountLedgerSettled()
|
||||
let healed = await brain.storage.getCanonicalCounts()
|
||||
expect(healed.suspect).toBe(false)
|
||||
expect(healed.nouns.all).toBe(realTotal)
|
||||
|
||||
await brain.repairIndex()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue