feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete

The storage-level unfiltered getNouns()/getVerbs() walks enumerate every
tier, but their totalCount reported the user-facing scalar, which skips
system/internal records on the write path — so a derived-index coverage
ledger comparing its posted count against that total would read
"over-posted by N" on every store with a VFS. This adds the ledger's real
denominators:

- totalNounCountAll / totalVerbCountAll: +1 for every new canonical record
  regardless of tier, −1 for every PROVEN delete (record read, or the
  caller's prior image), persisted in counts.json beside the counted
  scalars, recomputed by the sanctioned recount (rebuildTypeCounts).
- The unfiltered storage-level totalCount is now the ALL scalar and is
  never clamped: Math.max(scalar, scanned) could only move a scalar up, so
  an inflated counter hid forever; a divergence is now visible and healed
  by repairIndex().
- A delete that cannot prove the record existed never decrements on faith:
  it marks the ledger SUSPECT (persisted, narrated once per session) and
  the recount clears the flag with proof.
- getCanonicalCounts() on StorageAdapter (optional) exposes {counted, all}
  per family plus the suspect flag — O(1), no I/O.
- A counts.json written before the ledger existed derives both scalars
  once from the canonical id tree at open and persists them; absent keys
  are a legacy file, never a zero.

User-facing getNounCount()/getVerbCount() are unchanged.

Pinned in tests/integration/canonical-count-ledger.test.ts (5 laws).

src/coreTypes.ts

@@ -792,6 +792,19 @@ export interface DerivedFamilyDeclaration {
   rebuildable?: boolean
 }
 
+/**
+ * @description The canonical count ledger a storage adapter maintains on its
+ * write path: per family, the user-facing `counted` scalar and the
+ * ALL-visibility `all` scalar (every tier — the coverage-ledger denominator).
+ * See {@link StorageAdapter.getCanonicalCounts}.
+ */
+export interface CanonicalCounts {
+  nouns: { counted: number; all: number }
+  verbs: { counted: number; all: number }
+  /** An unprovable delete has left the `all` scalars unverified since the last recount. */
+  suspect: boolean
+}
+
 export interface StorageAdapter {
   init(): Promise<void>
 
@@ -1293,6 +1306,19 @@ export interface StorageAdapter {
    */
   getVerbCount(): Promise<number>
 
+  /**
+   * The canonical count ledger — O(1), no I/O. `counted` mirrors
+   * `getNounCount()` / `getVerbCount()` (public + internal tiers); `all` is
+   * the ALL-visibility scalar every unfiltered storage walk is measured
+   * against — the denominator a derived-index provider's coverage ledger
+   * subtracts from. `suspect` is `true` when an unprovable delete has left
+   * `all` unverified since the last sanctioned recount (`repairIndex()`).
+   * Optional: adapters without the ledger omit it; a consumer treats absence
+   * as "no denominator", never as zero.
+   * @returns Both scalars per family plus the suspect flag.
+   */
+  getCanonicalCounts?(): Promise<CanonicalCounts>
+
   /**
    * OPTIONAL — create a pre-upgrade backup of the whole store and return its
    * location, or `null` when there is nothing to back up (empty store). On the

src/storage/adapters/baseStorageAdapter.ts

@@ -12,7 +12,8 @@ import {
   HNSWNounWithMetadata,
   HNSWVerbWithMetadata,
   NounMetadata,
-  VerbMetadata
+  VerbMetadata,
+  CanonicalCounts,
 } from '../../coreTypes.js'
 import { StorageBatchConfig } from '../baseStorage.js'
 import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
@@ -1028,6 +1029,28 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
   // Universal count tracking - O(1) operations
   protected totalNounCount = 0
   protected totalVerbCount = 0
+  /**
+   * The ALL-visibility canonical scalars — every noun / verb the unfiltered
+   * storage walk yields, system and internal tiers included. These are the
+   * denominators a derived-index provider's coverage ledger subtracts from
+   * (`posted === all` is the whole-store coverage verdict); the user-facing
+   * `totalNounCount` / `totalVerbCount` skip hidden tiers by design and can
+   * never serve as a ledger denominator. Maintained on the write path
+   * (every new record +1, every proven delete −1), persisted beside the
+   * counted scalars, recomputed by the sanctioned recount. Never clamped.
+   */
+  protected totalNounCountAll = 0
+  protected totalVerbCountAll = 0
+  /**
+   * `true` when a delete could not prove whether the record existed (no
+   * canonical read, no caller-provided prior) — the ALL scalar may be off by
+   * the unprovable deletes since. Loud, persisted, and cleared only by the
+   * sanctioned recount; a consumer reading the scalar as a ledger denominator
+   * must treat a suspect scalar as unverified, never as exact.
+   */
+  protected allCountsSuspect = false
+  /** One narration per session for the suspect transition (never per delete). */
+  private allCountsSuspectNarrated = false
   protected entityCounts: Map<string, number> = new Map() // type -> count
   protected verbCounts: Map<string, number> = new Map() // verb type -> count
   protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
@@ -1056,6 +1079,43 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
     return this.totalVerbCount
   }
 
+  /**
+   * The canonical count ledger — O(1), no I/O. `counted` is the user-facing
+   * scalar (public/internal tiers, what `getNounCount()` returns); `all` is
+   * the ALL-visibility scalar every unfiltered storage walk is measured
+   * against (the coverage-ledger denominator for derived-index providers).
+   * `suspect` is `true` when an unprovable delete has made `all` unverified
+   * since the last sanctioned recount (`rebuildTypeCounts`).
+   * @returns Both scalars per family plus the suspect flag.
+   */
+  async getCanonicalCounts(): Promise<CanonicalCounts> {
+    return {
+      nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
+      verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
+      suspect: this.allCountsSuspect
+    }
+  }
+
+  /**
+   * Mark the ALL scalars unverified after a delete that could not prove the
+   * record existed. Narrates ONCE per session (the flag is what persists);
+   * the sanctioned recount clears it.
+   * @param family - Which family's delete was unprovable.
+   * @param id - The id whose existence could not be established.
+   */
+  protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void {
+    this.allCountsSuspect = true
+    if (!this.allCountsSuspectNarrated) {
+      this.allCountsSuspectNarrated = true
+      console.warn(
+        `[Storage] ${family} delete of ${id} could not prove the record existed ` +
+          `(no canonical read, no prior record) — the ALL-visibility count ledger is ` +
+          `SUSPECT until brain.repairIndex() recounts. Further unprovable deletes ` +
+          `this session are counted silently under the same flag.`
+      )
+    }
+  }
+
   /**
    * Increment count for entity type - O(1) operation.
    * Concurrency is handled by the process-global mutex

src/storage/adapters/fileSystemStorage.ts

@@ -2561,6 +2561,33 @@ export class FileSystemStorage extends BaseStorage {
         this.totalNounCount = counts.totalNounCount || 0
         this.totalVerbCount = counts.totalVerbCount || 0
 
+        // The ALL-visibility scalars (ledger denominators). A counts.json
+        // written before they existed carries neither key: derive both ONCE
+        // from the canonical id tree (an id-directory listing — O(ids), no
+        // record reads), persist, and never scan again. Absent keys are a
+        // legacy file, not a zero — a zero here would make every provider's
+        // coverage ledger read "over-posted" on a populated store.
+        if (
+          typeof counts.totalNounCountAll === 'number' &&
+          typeof counts.totalVerbCountAll === 'number'
+        ) {
+          this.totalNounCountAll = counts.totalNounCountAll
+          this.totalVerbCountAll = counts.totalVerbCountAll
+          this.allCountsSuspect = counts.allCountsSuspect === true
+        } else {
+          const nouns = await this.scanCanonicalEntities('nouns')
+          const verbs = await this.scanCanonicalEntities('verbs')
+          this.totalNounCountAll = nouns.count
+          this.totalVerbCountAll = verbs.count
+          this.allCountsSuspect = false
+          console.warn(
+            `[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
+              `derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
+              `every tier) and persisted; no further scan.`
+          )
+          await this.persistCounts()
+        }
+
         // Also populate the cache for backward compatibility
         this.countCache.set('nouns_count', {
           count: this.totalNounCount,
@@ -2596,6 +2623,10 @@ export class FileSystemStorage extends BaseStorage {
       this.totalNounCount = nouns.count
       const verbs = await this.scanCanonicalEntities('verbs')
       this.totalVerbCount = verbs.count
+      // The id-tree scan counts every tier — it IS the ALL-visibility ledger.
+      this.totalNounCountAll = nouns.count
+      this.totalVerbCountAll = verbs.count
+      this.allCountsSuspect = false
 
       // Sample some entities for the type distribution (don't read all).
       // Read the metadata files DIRECTLY with fs — this runs inside init(),
@@ -2693,6 +2724,11 @@ export class FileSystemStorage extends BaseStorage {
         verbCounts: Object.fromEntries(this.verbCounts),
         totalNounCount: this.totalNounCount,
         totalVerbCount: this.totalVerbCount,
+        // ALL-visibility ledger scalars (+ the suspect flag) — absent in files
+        // written before the ledger existed; initializeCounts() derives them once.
+        totalNounCountAll: this.totalNounCountAll,
+        totalVerbCountAll: this.totalVerbCountAll,
+        allCountsSuspect: this.allCountsSuspect,
         lastUpdated: new Date().toISOString()
       }
 

src/storage/adapters/memoryStorage.ts

@@ -540,6 +540,10 @@ export class MemoryStorage extends BaseStorage {
 
     this.totalNounCount = totalNouns
     this.totalVerbCount = totalVerbs
+    // A scan of every canonical record IS the ALL-visibility count.
+    this.totalNounCountAll = totalNouns
+    this.totalVerbCountAll = totalVerbs
+    this.allCountsSuspect = false
   }
 
   /**

src/storage/baseStorage.ts

@@ -2273,11 +2273,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
     // totalCount must be the TRUE dataset total, not this peeked page. For the
     // unfiltered case the authoritative total is the O(1) counter maintained on
-    // every add/delete (rehydrated on init); `Math.max` guards a stale counter. A
-    // filtered scan has no cheap exact total, so it keeps the collected length.
-    const totalCount = filter
-      ? collected.length
-      : Math.max(this.totalNounCount, collected.length)
+    // every add/delete (rehydrated on init) — the ALL-visibility scalar, because
+    // this walk is unfiltered by tier (system/internal records are in `collected`).
+    // Never clamped: `Math.max(scalar, scanned)` could only ever move the scalar
+    // UP, so an inflated counter could never correct itself and a divergence was
+    // hidden instead of reported. A scalar that disagrees with the walk is the
+    // canonical-count-ledger invariant's job, healed by the sanctioned recount.
+    // A filtered scan has no cheap exact total, so it keeps the collected length.
+    const totalCount = filter ? collected.length : this.totalNounCountAll
 
     // nextCursor = the (shard, id) of the last RETURNED noun, so the next call
     // resumes immediately after it (works for both cursor and offset callers).
@@ -2409,7 +2412,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     const pagePairs = collected.slice(windowStart, windowStart + limit)
     const ids = pagePairs.map((p) => p.id)
     const hasMore = collected.length > windowStart + limit
-    const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length)
+    // ALL-visibility scalar, unclamped — same law as getNouns() above.
+    const totalCount = filter ? collected.length : this.totalNounCountAll
 
     let nextCursor: string | undefined = undefined
     if (hasMore && pagePairs.length > 0) {
@@ -2641,13 +2645,13 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     const hasMore = collected.length > windowStart + limit
 
     // totalCount must be the TRUE dataset total, not this peeked page. For the
-    // unfiltered scan the authoritative total is the O(1) `totalVerbCount` counter
-    // (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` guards a
-    // stale counter from under-reporting. A filtered scan has no cheap exact total,
-    // so it keeps the collected length (a lower bound).
-    const totalCount = filter
-      ? collected.length
-      : Math.max(this.totalVerbCount, collected.length)
+    // unfiltered scan the authoritative total is the O(1) ALL-visibility counter
+    // (`totalVerbCountAll`: isNew-gated, EVERY tier, rehydrated on init) — the walk
+    // itself is unfiltered by tier, so the user-facing `totalVerbCount` (which skips
+    // system/internal edges) would undercount it on every store with a VFS. Never
+    // clamped (see getNouns): a divergence is reported, not hidden. A filtered scan
+    // has no cheap exact total, so it keeps the collected length (a lower bound).
+    const totalCount = filter ? collected.length : this.totalVerbCountAll
 
     // nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
     // resumes immediately after it — for both cursor and offset callers (an offset
@@ -3455,6 +3459,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
     const isCounted = isCountedVisibility(newVisibility)
 
+    // ALL-visibility ledger: every NEW canonical record is +1 regardless of tier
+    // (the unfiltered walk yields it, so the denominator must count it). The
+    // counted branch below persists for public/internal records; a hidden new
+    // record persists here so the ALL scalar never lags the tree.
+    if (isNew) {
+      this.totalNounCountAll++
+      if (!(metadata.noun && isCounted)) {
+        this.scheduleCountPersist().catch(() => {
+          // Ignore persist errors — the in-memory count is authoritative; a later op retries.
+        })
+      }
+    }
+
     // CRITICAL FIX: Increment count for new entities
     // This runs AFTER metadata is saved, guaranteeing type information is available
     // Uses synchronous increment since storage operations are already serialized
@@ -3858,6 +3875,20 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     await this.deleteCanonicalObject(path)
     const record = read ?? priorRecord
 
+    // ALL-visibility ledger: a PROVEN delete (the record was read, or the caller
+    // carried its prior image) is −1 regardless of tier. A delete that can prove
+    // nothing never guesses — it marks the ledger suspect (loud, persisted) and the
+    // sanctioned recount restores exactness.
+    if (record) {
+      if (this.totalNounCountAll > 0) this.totalNounCountAll--
+      else this.markAllCountsSuspect('noun', id)
+    } else {
+      this.markAllCountsSuspect('noun', id)
+    }
+    this.scheduleCountPersist().catch(() => {
+      // Ignore persist errors — the in-memory count is authoritative; a later op retries.
+    })
+
     const priorType = record?.noun as NounType | undefined
     // 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
     // (gated in `saveNounMetadata_internal()`), so it must not be decremented here either.
@@ -3991,6 +4022,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     // Fixes Bug #2: Count synchronization failure during relate() and import()
     // 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
     if (isNew) {
+      // ALL-visibility ledger: every new edge is +1 regardless of tier (the
+      // unfiltered walk yields VFS/system edges too; the denominator must count them).
+      this.totalVerbCountAll++
       if (isCounted) {
         this.incrementVerbCount(verbType)
       } else {
@@ -4052,6 +4086,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     await this.deleteCanonicalObject(path)
     const record = read ?? priorRecord
 
+    // ALL-visibility ledger: proven delete −1 regardless of tier; an unprovable
+    // delete marks the ledger suspect instead of guessing (see deleteNounMetadata).
+    if (record) {
+      if (this.totalVerbCountAll > 0) this.totalVerbCountAll--
+      else this.markAllCountsSuspect('verb', id)
+    } else {
+      this.markAllCountsSuspect('verb', id)
+    }
+    this.scheduleCountPersist().catch(() => {
+      // Ignore persist errors — in-memory count is authoritative; a later op retries.
+    })
+
     const priorVerb = record?.verb as VerbType | undefined
     // Symmetric count decrement (previously OMITTED — verb deletes touched neither the
     // scalar total nor the per-type bucket, so both inflated permanently). A COUNTED
@@ -4497,6 +4543,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     // walk, every counter rollup rebuilt and persisted from it.
     const countedNouns = new Map<string, number>()
     const countedVerbs = new Map<string, number>()
+    // ALL-visibility scalars: one per canonical record the walk yields, every
+    // tier, readable or not — the same population the unfiltered getNouns()/
+    // getVerbs() walks enumerate, so `totalCount` and this recount agree by
+    // construction.
+    let allNouns = 0
+    let allVerbs = 0
 
     // Scan noun shards
     for (let shard = 0; shard < 256; shard++) {
@@ -4508,6 +4560,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
         for (const path of paths) {
           if (!path.includes('/metadata.json')) continue
+          allNouns++
 
           try {
             const metadata = await this.readCanonicalObject(path)
@@ -4540,6 +4593,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
         for (const path of paths) {
           if (!path.includes('/metadata.json')) continue
+          allVerbs++
 
           try {
             const metadata = await this.readCanonicalObject(path)
@@ -4576,10 +4630,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     this.verbCounts = countedVerbs
     this.totalNounCount = totalNouns
     this.totalVerbCount = totalVerbs
+    // The ALL scalars are exact again and the suspect flag clears — this walk
+    // IS the proof an unprovable delete could not give.
+    const nounsAllBefore = this.totalNounCountAll
+    const verbsAllBefore = this.totalVerbCountAll
+    this.totalNounCountAll = allNouns
+    this.totalVerbCountAll = allVerbs
+    this.allCountsSuspect = false
     this.countCache.clear()
     await this.persistCounts()
 
-    prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`)
+    prodLog.info(
+      `[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
+        `ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
+        (nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
+          ? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
+          : ' (unchanged)') +
+        ` — scalar + per-type persisted`
+    )
   }
 
   /**