fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load

src/storage/adapters/baseStorageAdapter.ts

@@ -1066,6 +1066,18 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
   protected allCountsSuspect = false
   /** One narration per session for the suspect transition (never per delete). */
   private allCountsSuspectNarrated = false
+  /**
+   * Which rule produced the ALL scalars currently in memory. `'identity-record'`
+   * means one counted entity per metadata content leg — the honest rule: a
+   * bare id-directory (a ghost or scar left by a partial-delete defect, no
+   * content leg) counts zero. Set by the one-time derivation and by the
+   * sanctioned recount, alongside `allCountsSuspect = false`; left `undefined`
+   * when a loaded counts.json carries the ALL scalars but no stamp — the
+   * legacy container-rule derivation, which forces `allCountsSuspect = true`
+   * at load instead. A filesystem concern: `MemoryStorage` has no counts.json
+   * and never sets this.
+   */
+  protected allCountsDerivedBy?: 'identity-record'
   protected entityCounts: Map<string, number> = new Map() // type -> count
   protected verbCounts: Map<string, number> = new Map() // verb type -> count
   protected countCache: Map<string, { count: number; timestamp: number }> = new Map()

src/storage/adapters/fileSystemStorage.ts

@@ -18,6 +18,7 @@ import {
 } from '../baseStorage.js'
 import { getBrainyVersion } from '../../utils/index.js'
 import { isAbsentError } from '../../utils/errorClassification.js'
+import { prodLog } from '../../utils/logger.js'
 import {
   TornRecordError,
   isUnparseablePayloadError,
@@ -602,6 +603,20 @@ export class FileSystemStorage extends BaseStorage {
    * automatically. Returns the pruned container ids so the caller can recompute
    * counts.
    */
+  /**
+   * @description Whether an id directory's file legs include the metadata
+   * CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single
+   * test that decides whether an `entities/<kind>/<shard>/<id>/` container is
+   * a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete
+   * defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune
+   * and {@link scanCanonicalEntities} so the two agree by construction — one
+   * counted entity per identity record, never per bare container.
+   * @param legs - File names in one `entities/<kind>/<shard>/<id>/` directory.
+   */
+  private hasMetadataContentLeg(legs: string[]): boolean {
+    return legs.some((f) => f.startsWith('metadata.json'))
+  }
+
   public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
     await this.ensureInitialized()
     const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
@@ -641,7 +656,7 @@ export class FileSystemStorage extends BaseStorage {
           }
           // A live entity has its metadata content leg. No content leg → a
           // vector-only ghost or an empty scar → prune the whole container.
-          if (legs.some((f) => f.startsWith('metadata.json'))) continue
+          if (this.hasMetadataContentLeg(legs)) continue
           await fs.promises.rm(idAbs, { recursive: true, force: true })
           pruned[kind].push(entry.name)
           console.warn(
@@ -2590,13 +2605,36 @@ export class FileSystemStorage extends BaseStorage {
         ) {
           this.totalNounCountAll = counts.totalNounCountAll
           this.totalVerbCountAll = counts.totalVerbCountAll
-          this.allCountsSuspect = counts.allCountsSuspect === true
+          if (counts.allCountsDerivedBy === 'identity-record') {
+            // Derived (or recounted) under the honest rule — one counted
+            // entity per metadata content leg. Trust the persisted suspect
+            // flag as-is; an unprovable delete since may still have set it.
+            this.allCountsDerivedBy = 'identity-record'
+            this.allCountsSuspect = counts.allCountsSuspect === true
+          } else {
+            // The ALL scalars exist but predate the identity-record stamp —
+            // they were derived under the legacy rule that counted one
+            // entity per id DIRECTORY, so orphaned ghost/scar containers (a
+            // pre-8.3.1 partial-delete defect — see pruneOrphanedEntities())
+            // were counted as entities too. O(1) field read, NEVER a walk
+            // here: force suspect and name it loudly. A sanctioned recount
+            // (repairIndex) restores exact denominators and clears this.
+            this.allCountsDerivedBy = undefined
+            this.allCountsSuspect = true
+            needsPersist = true
+            prodLog.warn(
+              '[FileSystemStorage] canonical count ledger was derived under the legacy ' +
+                'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' +
+                'exact denominators'
+            )
+          }
         } else {
           const nouns = await this.scanCanonicalEntities('nouns')
           const verbs = await this.scanCanonicalEntities('verbs')
           this.totalNounCountAll = nouns.count
           this.totalVerbCountAll = verbs.count
           this.allCountsSuspect = false
+          this.allCountsDerivedBy = 'identity-record'
           console.warn(
             `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
               `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
@@ -2667,6 +2705,7 @@ export class FileSystemStorage extends BaseStorage {
       this.totalNounCountAll = nouns.count
       this.totalVerbCountAll = verbs.count
       this.allCountsSuspect = false
+      this.allCountsDerivedBy = 'identity-record'
       // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
       // (a deferred-embed noun's file exists but holds an empty vector until
       // its embed lands), so this is a full O(nouns) content scan — see
@@ -2704,10 +2743,19 @@ export class FileSystemStorage extends BaseStorage {
 
   /**
    * Walk the canonical `entities/<kind>/<2-hex-shard>/<id>/` tree, counting
-   * one entity per id directory (the layout `getNounVectorPath`/`getNouns`
-   * use). Returns up to 100 sampled entity directories (absolute paths) —
-   * nouns feed the type-distribution estimate above. An absent tree (fresh
-   * store) counts zero.
+   * one entity per id directory that holds the metadata CONTENT leg
+   * (`metadata.json` or its `.json.gz` variant — see
+   * {@link hasMetadataContentLeg}). A bare container — a ghost (a stale
+   * `vectors.json` left with no metadata leg) or a scar (an empty directory),
+   * both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the
+   * identity record IS the population (ADR-008 G1), never the directory.
+   * This is the ONE-TIME legacy derivation walk (see callers); a prior
+   * version of this scan counted every id directory regardless of content,
+   * over-counting any store carrying orphaned containers — see
+   * `allCountsDerivedBy` for how a counts.json derived under that old rule is
+   * marked suspect on load. Returns up to 100 sampled *counted* entity
+   * directories (absolute paths) — nouns feed the type-distribution estimate
+   * above. An absent tree (fresh store) counts zero.
    */
   private async scanCanonicalEntities(
     kind: 'nouns' | 'verbs'
@@ -2724,9 +2772,21 @@ export class FileSystemStorage extends BaseStorage {
         const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
         for (const entry of ids) {
           if (!entry.isDirectory()) continue
+          const idAbs = path.join(shardPath, entry.name)
+          let legs: string[]
+          try {
+            legs = await fs.promises.readdir(idAbs)
+          } catch (error: any) {
+            if (error?.code === 'ENOENT') continue
+            throw error
+          }
+          // No metadata content leg → a ghost or scar container → not an
+          // entity. Same test pruneOrphanedEntities() uses, so the two agree
+          // by construction.
+          if (!this.hasMetadataContentLeg(legs)) continue
           count++
           if (sampleDirs.length < SAMPLE_MAX) {
-            sampleDirs.push(path.join(shardPath, entry.name))
+            sampleDirs.push(idAbs)
           }
         }
       }
@@ -2834,6 +2894,13 @@ export class FileSystemStorage extends BaseStorage {
         // scanVectoredNounCount()'s JSDoc).
         totalVectoredNounCount: this.totalVectoredNounCount,
         allCountsSuspect: this.allCountsSuspect,
+        // Derivation-rule stamp for the ALL scalars above — 'identity-record'
+        // when they were counted one-per-metadata-content-leg (the honest
+        // rule); omitted (JSON.stringify drops `undefined`) when the current
+        // in-memory scalars came from a legacy container-rule counts.json
+        // that hasn't been through a sanctioned recount yet, so a future load
+        // keeps naming them suspect rather than trusting an unproven value.
+        allCountsDerivedBy: this.allCountsDerivedBy,
         lastUpdated: new Date().toISOString()
       }
 

src/storage/baseStorage.ts

@@ -4820,6 +4820,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     this.totalVerbCountAll = allVerbs
     this.totalVectoredNounCount = allVectoredNouns
     this.allCountsSuspect = false
+    // This walk counts one entity per metadata.json record (never per bare
+    // container) — the identity-record rule. Stamp it so a future load
+    // trusts these scalars instead of naming them suspect at open.
+    this.allCountsDerivedBy = 'identity-record'
     this.countCache.clear()
     await this.persistCounts()
 

tests/integration/ledger-derivation-identity.test.ts

@@ -0,0 +1,195 @@
+/**
+ * @module tests/integration/ledger-derivation-identity
+ * @description The ALL-visibility ledger scalars are an IDENTITY-RECORD
+ * count, never a container count. A pre-8.3.1 partial-delete defect can
+ * leave a "ghost" container (a stale `vectors.json` with no metadata content
+ * leg) or a "scar" container (an empty `entities/<kind>/<shard>/<id>/`
+ * directory) on disk. Neither is a live entity — `getNoun`/`getVerb` need
+ * the metadata content leg — yet the legacy derivation counted one entity
+ * per id DIRECTORY, so orphaned containers inflated the ALL scalars forever
+ * (they were never clamped and never re-derived). Laws under test:
+ * (1) IDENTITY, NOT CONTAINER — the derivation counts one entity per
+ *     metadata content leg (`metadata.json` or `.json.gz`), the same test
+ *     `pruneOrphanedEntities()` uses, so the two agree by construction.
+ * (2) THE STAMP NAMES SUSPECT COUNTS LOUDLY, AT O(1) — a counts.json that
+ *     carries the ALL scalars but no `allCountsDerivedBy: 'identity-record'`
+ *     stamp predates this fix; loading it marks `suspect = true` from a
+ *     single field read alone, never a directory walk, and warns exactly
+ *     once naming the cause.
+ * (3) THE SANCTIONED RECOUNT CLEARS IT — `repairIndex()` prunes the orphaned
+ *     containers, recounts from the canonical metadata.json walk, and
+ *     re-stamps — suspect clears and the ALL scalar is exact again.
+ * (4) A FRESH STORE IS NEVER SUSPECT — the one-time derivation for a store
+ *     with no counts.json stamps as it writes, so a brand-new store never
+ *     carries the legacy signature.
+ */
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+import { Brainy, FileSystemStorage } from '../../src/index.js'
+import { prodLog } from '../../src/utils/logger.js'
+
+const countsPath = (root: string) => path.join(root, '_system', 'counts.json')
+
+/** Plant a ghost container: a stale `vectors.json` leg, no metadata leg. */
+function plantGhost(root: string, shard: string, id: string): void {
+  const idDir = path.join(root, 'entities', 'nouns', shard, id)
+  fs.mkdirSync(idDir, { recursive: true })
+  fs.writeFileSync(path.join(idDir, 'vectors.json'), JSON.stringify({ vector: [0.1, 0.2, 0.3] }))
+}
+
+/** Plant a scar container: an empty id directory, no legs at all. */
+function plantScar(root: string, shard: string, id: string): void {
+  fs.mkdirSync(path.join(root, 'entities', 'nouns', shard, id), { recursive: true })
+}
+
+describe('ledger derivation identity — the ALL scalar is the identity-record population, never the container count', () => {
+  let dir: string
+
+  const open = async () => {
+    const b: any = new Brainy({
+      requireSubtype: false,
+      storage: { type: 'filesystem', path: dir },
+      silent: true,
+      dimensions: 384
+    })
+    await b.init()
+    return b
+  }
+
+  beforeEach(() => {
+    process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-identity-'))
+  })
+  afterEach(() => {
+    vi.restoreAllMocks()
+    fs.rmSync(dir, { recursive: true, force: true })
+  })
+
+  it('(a) ghost + scar containers count ZERO; the fresh derivation stamps counts.json', async () => {
+    let brain = await open()
+    const baseline = (await brain.storage.getCanonicalCounts()).nouns.all // the VFS root alone
+    for (let i = 0; i < 3; i++) {
+      await brain.add({ data: `real ${i}`, type: 'document' })
+    }
+    await brain.flush()
+    const realTotal = baseline + 3
+    await brain.close()
+
+    // 3 ghosts (stale vectors.json, no metadata leg) + 2 scars (empty dirs) —
+    // neither is a live entity.
+    for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`)
+    for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`)
+
+    // Remove counts.json so open() re-derives from scratch (the one-time
+    // legacy/lost-file derivation path).
+    fs.rmSync(countsPath(dir), { force: true })
+
+    brain = await open()
+    const ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars contribute nothing
+    expect(ledger.suspect).toBe(false)
+
+    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    expect(raw.totalNounCountAll).toBe(realTotal)
+    expect(raw.allCountsDerivedBy).toBe('identity-record')
+
+    await brain.close()
+  })
+
+  it('(b) a counts.json with the ALL scalars but no stamp is marked suspect at open — an O(1) field read, never a walk', async () => {
+    let brain = await open()
+    await brain.add({ data: 'one', type: 'document' })
+    await brain.add({ data: 'two', type: 'document' })
+    await brain.flush()
+    await brain.close()
+
+    // Confirm a normal close under the fix DOES stamp — then strip the stamp
+    // to simulate a counts.json produced before this fix existed.
+    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    expect(raw.allCountsDerivedBy).toBe('identity-record')
+    expect(typeof raw.totalNounCountAll).toBe('number')
+    expect(typeof raw.totalVerbCountAll).toBe('number')
+    expect(typeof raw.totalVectoredNounCount).toBe('number')
+    delete raw.allCountsDerivedBy
+    fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
+
+    const warnSpy = vi.spyOn(prodLog, 'warn')
+    // The two derivation walks live on FileSystemStorage's prototype —
+    // spying here (rather than on fs.promises.readdir globally) isolates
+    // THIS code path's behavior from unrelated walks elsewhere in the open
+    // sequence (a separate, pre-existing engine's own O(store) cost — not
+    // this fix's concern, and not something this pin should be sensitive
+    // to). Neither derivation method may run: the stamp check is a field
+    // read on the already-parsed counts.json, nothing more.
+    const scanEntitiesSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanCanonicalEntities')
+    const scanVectoredSpy = vi.spyOn(FileSystemStorage.prototype as any, 'scanVectoredNounCount')
+
+    brain = await open()
+
+    const ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.suspect).toBe(true)
+
+    const stampWarnings = warnSpy.mock.calls.filter(
+      ([msg]) => String(msg).includes('legacy') && String(msg).includes('container rule')
+    )
+    expect(stampWarnings.length).toBe(1) // exactly one, loud
+
+    expect(scanEntitiesSpy).not.toHaveBeenCalled() // O(1) field read only, no re-derivation walk
+    expect(scanVectoredSpy).not.toHaveBeenCalled()
+
+    await brain.close()
+  })
+
+  it('(c) repairIndex() prunes the orphans, recounts, and re-stamps — suspect clears, the ALL scalar is exact, and it survives reopen', async () => {
+    let brain = await open()
+    const baseline = (await brain.storage.getCanonicalCounts()).nouns.all
+    for (let i = 0; i < 3; i++) {
+      await brain.add({ data: `real ${i}`, type: 'document' })
+    }
+    await brain.flush()
+    const realTotal = baseline + 3
+    await brain.close()
+
+    for (let i = 0; i < 3; i++) plantGhost(dir, 'fe', `ghost-${i}`)
+    for (let i = 0; i < 2; i++) plantScar(dir, 'fd', `scar-${i}`)
+
+    // Force the legacy (unstamped, container-rule-inflated) shape directly —
+    // the shape a pre-existing production store actually carries.
+    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    raw.totalNounCountAll = realTotal + 5 // the old rule: +3 ghosts +2 scars
+    delete raw.allCountsDerivedBy
+    fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
+
+    brain = await open()
+    expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // named suspect at load
+
+    await brain.repairIndex()
+
+    let ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.suspect).toBe(false)
+    expect(ledger.nouns.all).toBe(realTotal) // ghosts + scars pruned; exact again
+
+    const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    expect(persisted.allCountsDerivedBy).toBe('identity-record')
+    expect(persisted.allCountsSuspect).toBe(false)
+    expect(persisted.totalNounCountAll).toBe(realTotal)
+
+    await brain.close()
+    brain = await open()
+    ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.suspect).toBe(false)
+    expect(ledger.nouns.all).toBe(realTotal)
+    await brain.close()
+  })
+
+  it('(d) a fresh store derives with the stamp and is never suspect', async () => {
+    const brain = await open()
+    const ledger = await brain.storage.getCanonicalCounts()
+    expect(ledger.suspect).toBe(false)
+    const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
+    expect(raw.allCountsDerivedBy).toBe('identity-record')
+    await brain.close()
+  })
+})