fix(storage): derive the canonical count ledger from identity records, stamp the derivation rule, and mark legacy-derived ledgers suspect at load

src/storage/adapters/baseStorageAdapter.ts

@@ -1066,6 +1066,18 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
   protected allCountsSuspect = false
   /** One narration per session for the suspect transition (never per delete). */
   private allCountsSuspectNarrated = false
+  /**
+   * Which rule produced the ALL scalars currently in memory. `'identity-record'`
+   * means one counted entity per metadata content leg — the honest rule: a
+   * bare id-directory (a ghost or scar left by a partial-delete defect, no
+   * content leg) counts zero. Set by the one-time derivation and by the
+   * sanctioned recount, alongside `allCountsSuspect = false`; left `undefined`
+   * when a loaded counts.json carries the ALL scalars but no stamp — the
+   * legacy container-rule derivation, which forces `allCountsSuspect = true`
+   * at load instead. A filesystem concern: `MemoryStorage` has no counts.json
+   * and never sets this.
+   */
+  protected allCountsDerivedBy?: 'identity-record'
   protected entityCounts: Map<string, number> = new Map() // type -> count
   protected verbCounts: Map<string, number> = new Map() // verb type -> count
   protected countCache: Map<string, { count: number; timestamp: number }> = new Map()

src/storage/adapters/fileSystemStorage.ts

@@ -18,6 +18,7 @@ import {
 } from '../baseStorage.js'
 import { getBrainyVersion } from '../../utils/index.js'
 import { isAbsentError } from '../../utils/errorClassification.js'
+import { prodLog } from '../../utils/logger.js'
 import {
   TornRecordError,
   isUnparseablePayloadError,
@@ -602,6 +603,20 @@ export class FileSystemStorage extends BaseStorage {
    * automatically. Returns the pruned container ids so the caller can recompute
    * counts.
    */
+  /**
+   * @description Whether an id directory's file legs include the metadata
+   * CONTENT leg (`metadata.json` or its `.json.gz` variant) — the single
+   * test that decides whether an `entities/<kind>/<shard>/<id>/` container is
+   * a live entity or a ghost/scar orphan left by the pre-8.3.1 partial-delete
+   * defect (see {@link pruneOrphanedEntities}). Shared by the orphan prune
+   * and {@link scanCanonicalEntities} so the two agree by construction — one
+   * counted entity per identity record, never per bare container.
+   * @param legs - File names in one `entities/<kind>/<shard>/<id>/` directory.
+   */
+  private hasMetadataContentLeg(legs: string[]): boolean {
+    return legs.some((f) => f.startsWith('metadata.json'))
+  }
+
   public async pruneOrphanedEntities(): Promise<{ nouns: string[]; verbs: string[] }> {
     await this.ensureInitialized()
     const pruned: { nouns: string[]; verbs: string[] } = { nouns: [], verbs: [] }
@@ -641,7 +656,7 @@ export class FileSystemStorage extends BaseStorage {
           }
           // A live entity has its metadata content leg. No content leg → a
           // vector-only ghost or an empty scar → prune the whole container.
-          if (legs.some((f) => f.startsWith('metadata.json'))) continue
+          if (this.hasMetadataContentLeg(legs)) continue
           await fs.promises.rm(idAbs, { recursive: true, force: true })
           pruned[kind].push(entry.name)
           console.warn(
@@ -2590,13 +2605,36 @@ export class FileSystemStorage extends BaseStorage {
         ) {
           this.totalNounCountAll = counts.totalNounCountAll
           this.totalVerbCountAll = counts.totalVerbCountAll
-          this.allCountsSuspect = counts.allCountsSuspect === true
+          if (counts.allCountsDerivedBy === 'identity-record') {
+            // Derived (or recounted) under the honest rule — one counted
+            // entity per metadata content leg. Trust the persisted suspect
+            // flag as-is; an unprovable delete since may still have set it.
+            this.allCountsDerivedBy = 'identity-record'
+            this.allCountsSuspect = counts.allCountsSuspect === true
+          } else {
+            // The ALL scalars exist but predate the identity-record stamp —
+            // they were derived under the legacy rule that counted one
+            // entity per id DIRECTORY, so orphaned ghost/scar containers (a
+            // pre-8.3.1 partial-delete defect — see pruneOrphanedEntities())
+            // were counted as entities too. O(1) field read, NEVER a walk
+            // here: force suspect and name it loudly. A sanctioned recount
+            // (repairIndex) restores exact denominators and clears this.
+            this.allCountsDerivedBy = undefined
+            this.allCountsSuspect = true
+            needsPersist = true
+            prodLog.warn(
+              '[FileSystemStorage] canonical count ledger was derived under the legacy ' +
+                'container rule — marked suspect; a sanctioned recount (repairIndex) restores ' +
+                'exact denominators'
+            )
+          }
         } else {
           const nouns = await this.scanCanonicalEntities('nouns')
           const verbs = await this.scanCanonicalEntities('verbs')
           this.totalNounCountAll = nouns.count
           this.totalVerbCountAll = verbs.count
           this.allCountsSuspect = false
+          this.allCountsDerivedBy = 'identity-record'
           console.warn(
             `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
               `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
@@ -2667,6 +2705,7 @@ export class FileSystemStorage extends BaseStorage {
       this.totalNounCountAll = nouns.count
       this.totalVerbCountAll = verbs.count
       this.allCountsSuspect = false
+      this.allCountsDerivedBy = 'identity-record'
       // Vectored-noun scalar: presence needs each noun's vectors.json CONTENT
       // (a deferred-embed noun's file exists but holds an empty vector until
       // its embed lands), so this is a full O(nouns) content scan — see
@@ -2704,10 +2743,19 @@ export class FileSystemStorage extends BaseStorage {
 
   /**
    * Walk the canonical `entities/<kind>/<2-hex-shard>/<id>/` tree, counting
-   * one entity per id directory (the layout `getNounVectorPath`/`getNouns`
-   * use). Returns up to 100 sampled entity directories (absolute paths) —
-   * nouns feed the type-distribution estimate above. An absent tree (fresh
-   * store) counts zero.
+   * one entity per id directory that holds the metadata CONTENT leg
+   * (`metadata.json` or its `.json.gz` variant — see
+   * {@link hasMetadataContentLeg}). A bare container — a ghost (a stale
+   * `vectors.json` left with no metadata leg) or a scar (an empty directory),
+   * both artifacts of the pre-8.3.1 partial-delete defect — counts ZERO: the
+   * identity record IS the population (ADR-008 G1), never the directory.
+   * This is the ONE-TIME legacy derivation walk (see callers); a prior
+   * version of this scan counted every id directory regardless of content,
+   * over-counting any store carrying orphaned containers — see
+   * `allCountsDerivedBy` for how a counts.json derived under that old rule is
+   * marked suspect on load. Returns up to 100 sampled *counted* entity
+   * directories (absolute paths) — nouns feed the type-distribution estimate
+   * above. An absent tree (fresh store) counts zero.
    */
   private async scanCanonicalEntities(
     kind: 'nouns' | 'verbs'
@@ -2724,9 +2772,21 @@ export class FileSystemStorage extends BaseStorage {
         const ids = await fs.promises.readdir(shardPath, { withFileTypes: true })
         for (const entry of ids) {
           if (!entry.isDirectory()) continue
+          const idAbs = path.join(shardPath, entry.name)
+          let legs: string[]
+          try {
+            legs = await fs.promises.readdir(idAbs)
+          } catch (error: any) {
+            if (error?.code === 'ENOENT') continue
+            throw error
+          }
+          // No metadata content leg → a ghost or scar container → not an
+          // entity. Same test pruneOrphanedEntities() uses, so the two agree
+          // by construction.
+          if (!this.hasMetadataContentLeg(legs)) continue
           count++
           if (sampleDirs.length < SAMPLE_MAX) {
-            sampleDirs.push(path.join(shardPath, entry.name))
+            sampleDirs.push(idAbs)
           }
         }
       }
@@ -2834,6 +2894,13 @@ export class FileSystemStorage extends BaseStorage {
         // scanVectoredNounCount()'s JSDoc).
         totalVectoredNounCount: this.totalVectoredNounCount,
         allCountsSuspect: this.allCountsSuspect,
+        // Derivation-rule stamp for the ALL scalars above — 'identity-record'
+        // when they were counted one-per-metadata-content-leg (the honest
+        // rule); omitted (JSON.stringify drops `undefined`) when the current
+        // in-memory scalars came from a legacy container-rule counts.json
+        // that hasn't been through a sanctioned recount yet, so a future load
+        // keeps naming them suspect rather than trusting an unproven value.
+        allCountsDerivedBy: this.allCountsDerivedBy,
         lastUpdated: new Date().toISOString()
       }
 

src/storage/baseStorage.ts

@@ -4820,6 +4820,10 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     this.totalVerbCountAll = allVerbs
     this.totalVectoredNounCount = allVectoredNouns
     this.allCountsSuspect = false
+    // This walk counts one entity per metadata.json record (never per bare
+    // container) — the identity-record rule. Stamp it so a future load
+    // trusts these scalars instead of naming them suspect at open.
+    this.allCountsDerivedBy = 'identity-record'
     this.countCache.clear()
     await this.persistCounts()