From fd6b4ce4ffc4194df1311dc8cb7900b2e9f67dd0 Mon Sep 17 00:00:00 2001 From: David Snelling Date: Thu, 27 Aug 2026 13:00:44 -0700 Subject: [PATCH] fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load --- src/storage/adapters/baseStorageAdapter.ts | 12 ++ src/storage/adapters/fileSystemStorage.ts | 81 +++++++- src/storage/baseStorage.ts | 4 + .../ledger-derivation-identity.test.ts | 195 ++++++++++++++++++ 4 files changed, 285 insertions(+), 7 deletions(-) create mode 100644 tests/integration/ledger-derivation-identity.test.ts diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 88018a66..cabe2e30 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1066,6 +1066,18 @@ export abstract class BaseStorageAdapter implements StorageAdapter { protected allCountsSuspect = false /** One narration per session for the suspect transition (never per delete). */ private allCountsSuspectNarrated = false + /** + * Which rule produced the ALL scalars currently in memory. `'identity-record'` + * means one counted entity per metadata content leg — the honest rule: a + * bare id-directory (a ghost or scar left by a partial-delete defect, no + * content leg) counts zero. Set by the one-time derivation and by the + * sanctioned recount, alongside `allCountsSuspect = false`; left `undefined` + * when a loaded counts.json carries the ALL scalars but no stamp — the + * legacy container-rule derivation, which forces `allCountsSuspect = true` + * at load instead. A filesystem concern: `MemoryStorage` has no counts.json + * and never sets this. + */ + protected allCountsDerivedBy?: 'identity-record' protected entityCounts: Map = new Map() // type -> count protected verbCounts: Map = new Map() // verb type -> count protected countCache: Map = new Map() diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index fd9dbb4c..817e5e36 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -18,6 +18,7 @@ import { } from '../baseStorage.js' import { getBrainyVersion } from '../../utils/index.js' import { isAbsentError } from '../../utils/errorClassification.js' +import { prodLog } from '../../utils/logger.js' import { TornRecordError, isUnparseablePayloadError, @@ -602,6 +603,20 @@ export class FileSystemStorage extends BaseStorage { * automatically. Returns the pruned container ids so the caller can recompute * counts. */ + /** + * @description Whether an id directory's file legs include the metadata + * CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single + * test that decides whether an `entities////` container is + * a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete + * defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune + * and {@link scanCanonicalEntities} so the two agree by construction — one + * counted entity per identity record, never per bare container. + * @param legs - File names in one `entities////` directory. + */ + private hasMetadataContentLeg(legs: string[]): boolean { + return legs.some((f) => f.startsWith('metadata.json')) + } + public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> { await this.ensureInitialized() const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] } @@ -641,7 +656,7 @@ export class FileSystemStorage extends BaseStorage { } // A live entity has its metadata content leg. No content leg → a // vector-only ghost or an empty scar → prune the whole container. - if (legs.some((f) => f.startsWith('metadata.json'))) continue + if (this.hasMetadataContentLeg(legs)) continue await fs.promises.rm(idAbs, { recursive: true, force: true }) pruned[kind].push(entry.name) console.warn( @@ -2590,13 +2605,36 @@ export class FileSystemStorage extends BaseStorage { ) { this.totalNounCountAll = counts.totalNounCountAll this.totalVerbCountAll = counts.totalVerbCountAll - this.allCountsSuspect = counts.allCountsSuspect === true + if (counts.allCountsDerivedBy === 'identity-record') { + // Derived (or recounted) under the honest rule — one counted + // entity per metadata content leg. Trust the persisted suspect + // flag as-is; an unprovable delete since may still have set it. + this.allCountsDerivedBy = 'identity-record' + this.allCountsSuspect = counts.allCountsSuspect === true + } else { + // The ALL scalars exist but predate the identity-record stamp — + // they were derived under the legacy rule that counted one + // entity per id DIRECTORY, so orphaned ghost/scar containers (a + // pre-8.3.1 partial-delete defect — see pruneOrphanedEntities()) + // were counted as entities too. O(1) field read, NEVER a walk + // here: force suspect and name it loudly. A sanctioned recount + // (repairIndex) restores exact denominators and clears this. + this.allCountsDerivedBy = undefined + this.allCountsSuspect = true + needsPersist = true + prodLog.warn( + '[FileSystemStorage] canonical count ledger was derived under the legacy ' + + 'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' + + 'exact denominators' + ) + } } else { const nouns = await this.scanCanonicalEntities('nouns') const verbs = await this.scanCanonicalEntities('verbs') this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false + this.allCountsDerivedBy = 'identity-record' console.warn( `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` + `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` + @@ -2667,6 +2705,7 @@ export class FileSystemStorage extends BaseStorage { this.totalNounCountAll = nouns.count this.totalVerbCountAll = verbs.count this.allCountsSuspect = false + this.allCountsDerivedBy = 'identity-record' // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT // (a deferred-embed noun's file exists but holds an empty vector until // its embed lands), so this is a full O(nouns) content scan — see @@ -2704,10 +2743,19 @@ export class FileSystemStorage extends BaseStorage { /** * Walk the canonical `entities//<2-hex-shard>//` tree, counting - * one entity per id directory (the layout `getNounVectorPath`/`getNouns` - * use). Returns up to 100 sampled entity directories (absolute paths) — - * nouns feed the type-distribution estimate above. An absent tree (fresh - * store) counts zero. + * one entity per id directory that holds the metadata CONTENT leg + * (`metadata.json` or its `.json.gz` variant — see + * {@link hasMetadataContentLeg}). A bare container — a ghost (a stale + * `vectors.json` left with no metadata leg) or a scar (an empty directory), + * both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the + * identity record IS the population (ADR-008 G1), never the directory. + * This is the ONE-TIME legacy derivation walk (see callers); a prior + * version of this scan counted every id directory regardless of content, + * over-counting any store carrying orphaned containers — see + * `allCountsDerivedBy` for how a counts.json derived under that old rule is + * marked suspect on load. Returns up to 100 sampled *counted* entity + * directories (absolute paths) — nouns feed the type-distribution estimate + * above. An absent tree (fresh store) counts zero. */ private async scanCanonicalEntities( kind: 'nouns' | 'verbs' @@ -2724,9 +2772,21 @@ export class FileSystemStorage extends BaseStorage { const ids = await fs.promises.readdir(shardPath, { withFileTypes: true }) for (const entry of ids) { if (!entry.isDirectory()) continue + const idAbs = path.join(shardPath, entry.name) + let legs: string[] + try { + legs = await fs.promises.readdir(idAbs) + } catch (error: any) { + if (error?.code === 'ENOENT') continue + throw error + } + // No metadata content leg → a ghost or scar container → not an + // entity. Same test pruneOrphanedEntities() uses, so the two agree + // by construction. + if (!this.hasMetadataContentLeg(legs)) continue count++ if (sampleDirs.length < SAMPLE_MAX) { - sampleDirs.push(path.join(shardPath, entry.name)) + sampleDirs.push(idAbs) } } } @@ -2834,6 +2894,13 @@ export class FileSystemStorage extends BaseStorage { // scanVectoredNounCount()'s JSDoc). totalVectoredNounCount: this.totalVectoredNounCount, allCountsSuspect: this.allCountsSuspect, + // Derivation-rule stamp for the ALL scalars above — 'identity-record' + // when they were counted one-per-metadata-content-leg (the honest + // rule); omitted (JSON.stringify drops `undefined`) when the current + // in-memory scalars came from a legacy container-rule counts.json + // that hasn't been through a sanctioned recount yet, so a future load + // keeps naming them suspect rather than trusting an unproven value. + allCountsDerivedBy: this.allCountsDerivedBy, lastUpdated: new Date().toISOString() } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index ea88921c..5510f93b 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -4820,6 +4820,10 @@ export abstract class BaseStorage extends BaseStorageAdapter { this.totalVerbCountAll = allVerbs this.totalVectoredNounCount = allVectoredNouns this.allCountsSuspect = false + // This walk counts one entity per metadata.json record (never per bare + // container) — the identity-record rule. Stamp it so a future load + // trusts these scalars instead of naming them suspect at open. + this.allCountsDerivedBy = 'identity-record' this.countCache.clear() await this.persistCounts() diff --git a/tests/integration/ledger-derivation-identity.test.ts b/tests/integration/ledger-derivation-identity.test.ts new file mode 100644 index 00000000..cb19af4a --- /dev/null +++ b/tests/integration/ledger-derivation-identity.test.ts @@ -0,0 +1,195 @@ +/** + * @module tests/integration/ledger-derivation-identity + * @description The ALL-visibility ledger scalars are an IDENTITY-RECORD + * count, never a container count. A pre-8.3.1 partial-delete defect can + * leave a "ghost" container (a stale `vectors.json` with no metadata content + * leg) or a "scar" container (an empty `entities////` + * directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need + * the metadata content leg — yet the legacy derivation counted one entity + * per id DIRECTORY, so orphaned containers inflated the ALL scalars forever + * (they were never clamped and never re-derived). Laws under test: + * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per + * metadata content leg (`metadata.json` or `.json.gz`), the same test + * `pruneOrphanedEntities()` uses, so the two agree by construction. + * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that + * carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'` + * stamp predates this fix; loading it marks `suspect = true` from a + * single field read alone, never a directory walk, and warns exactly + * once naming the cause. + * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned + * containers, recounts from the canonical metadata.json walk, and + * re-stamps — suspect clears and the ALL scalar is exact again. + * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store + * with no counts.json stamps as it writes, so a brand-new store never + * carries the legacy signature. + */ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import * as fs from 'node:fs' +import * as os from 'node:os' +import * as path from 'node:path' +import { Brainy, FileSystemStorage } from '../../src/index.js' +import { prodLog } from '../../src/utils/logger.js' + +const countsPath = (root: string) => path.join(root, '_system', 'counts.json') + +/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */ +function plantGhost(root: string, shard: string, id: string): void { + const idDir = path.join(root, 'entities', 'nouns', shard, id) + fs.mkdirSync(idDir, { recursive: true }) + fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] })) +} + +/** Plant a scar container: an empty id directory, no legs at all. */ +function plantScar(root: string, shard: string, id: string): void { + fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true }) +} + +describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => { + let dir: string + + const open = async () => { + const b: any = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', path: dir }, + silent: true, + dimensions: 384 + }) + await b.init() + return b + } + + beforeEach(() => { + process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true' + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-')) + }) + afterEach(() => { + vi.restoreAllMocks() + fs.rmSync(dir, { recursive: true, force: true }) + }) + + it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => { + let brain = await open() + const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone + for (let i = 0; i < 3; i++) { + await brain.add({ data: `real ${i}`, type: 'document' }) + } + await brain.flush() + const realTotal = baseline + 3 + await brain.close() + + // 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) — + // neither is a live entity. + for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) + for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) + + // Remove counts.json so open() re-derives from scratch (the one-time + // legacy/lost-file derivation path). + fs.rmSync(countsPath(dir), { force: true }) + + brain = await open() + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing + expect(ledger.suspect).toBe(false) + + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.totalNounCountAll).toBe(realTotal) + expect(raw.allCountsDerivedBy).toBe('identity-record') + + await brain.close() + }) + + it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => { + let brain = await open() + await brain.add({ data: 'one', type: 'document' }) + await brain.add({ data: 'two', type: 'document' }) + await brain.flush() + await brain.close() + + // Confirm a normal close under the fix DOES stamp — then strip the stamp + // to simulate a counts.json produced before this fix existed. + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.allCountsDerivedBy).toBe('identity-record') + expect(typeof raw.totalNounCountAll).toBe('number') + expect(typeof raw.totalVerbCountAll).toBe('number') + expect(typeof raw.totalVectoredNounCount).toBe('number') + delete raw.allCountsDerivedBy + fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) + + const warnSpy = vi.spyOn(prodLog, 'warn') + // The two derivation walks live on FileSystemStorage's prototype — + // spying here (rather than on fs.promises.readdir globally) isolates + // THIS code path's behavior from unrelated walks elsewhere in the open + // sequence (a separate, pre-existing engine's own O(store) cost — not + // this fix's concern, and not something this pin should be sensitive + // to). Neither derivation method may run: the stamp check is a field + // read on the already-parsed counts.json, nothing more. + const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities') + const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount') + + brain = await open() + + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(true) + + const stampWarnings = warnSpy.mock.calls.filter( + ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule') + ) + expect(stampWarnings.length).toBe(1) // exactly one, loud + + expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk + expect(scanVectoredSpy).not.toHaveBeenCalled() + + await brain.close() + }) + + it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => { + let brain = await open() + const baseline = (await brain.storage.getCanonicalCounts()).nouns.all + for (let i = 0; i < 3; i++) { + await brain.add({ data: `real ${i}`, type: 'document' }) + } + await brain.flush() + const realTotal = baseline + 3 + await brain.close() + + for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`) + for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`) + + // Force the legacy (unstamped, container-rule-inflated) shape directly — + // the shape a pre-existing production store actually carries. + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars + delete raw.allCountsDerivedBy + fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2)) + + brain = await open() + expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load + + await brain.repairIndex() + + let ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again + + const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(persisted.allCountsDerivedBy).toBe('identity-record') + expect(persisted.allCountsSuspect).toBe(false) + expect(persisted.totalNounCountAll).toBe(realTotal) + + await brain.close() + brain = await open() + ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + expect(ledger.nouns.all).toBe(realTotal) + await brain.close() + }) + + it('(d) a fresh store derives with the stamp and is never suspect', async () => { + const brain = await open() + const ledger = await brain.storage.getCanonicalCounts() + expect(ledger.suspect).toBe(false) + const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8')) + expect(raw.allCountsDerivedBy).toBe('identity-record') + await brain.close() + }) +})