feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete

The storage-level unfiltered getNouns()/getVerbs() walks enumerate every
tier, but their totalCount reported the user-facing scalar, which skips
system/internal records on the write path — so a derived-index coverage
ledger comparing its posted count against that total would read
"over-posted by N" on every store with a VFS. This adds the ledger's real
denominators:

- totalNounCountAll / totalVerbCountAll: +1 for every new canonical record
  regardless of tier, −1 for every PROVEN delete (record read, or the
  caller's prior image), persisted in counts.json beside the counted
  scalars, recomputed by the sanctioned recount (rebuildTypeCounts).
- The unfiltered storage-level totalCount is now the ALL scalar and is
  never clamped: Math.max(scalar, scanned) could only move a scalar up, so
  an inflated counter hid forever; a divergence is now visible and healed
  by repairIndex().
- A delete that cannot prove the record existed never decrements on faith:
  it marks the ledger SUSPECT (persisted, narrated once per session) and
  the recount clears the flag with proof.
- getCanonicalCounts() on StorageAdapter (optional) exposes {counted, all}
  per family plus the suspect flag — O(1), no I/O.
- A counts.json written before the ledger existed derives both scalars
  once from the canonical id tree at open and persists them; absent keys
  are a legacy file, never a zero.

User-facing getNounCount()/getVerbCount() are unchanged.

Pinned in tests/integration/canonical-count-ledger.test.ts (5 laws).

src/storage/baseStorage.ts

@@ -2273,11 +2273,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
     // totalCount must be the TRUE dataset total, not this peeked page. For the
     // unfiltered case the authoritative total is the O(1) counter maintained on
-    // every add/delete (rehydrated on init); `Math.max` guards a stale counter. A
-    // filtered scan has no cheap exact total, so it keeps the collected length.
-    const totalCount = filter
-      ? collected.length
-      : Math.max(this.totalNounCount, collected.length)
+    // every add/delete (rehydrated on init) — the ALL-visibility scalar, because
+    // this walk is unfiltered by tier (system/internal records are in `collected`).
+    // Never clamped: `Math.max(scalar, scanned)` could only ever move the scalar
+    // UP, so an inflated counter could never correct itself and a divergence was
+    // hidden instead of reported. A scalar that disagrees with the walk is the
+    // canonical-count-ledger invariant's job, healed by the sanctioned recount.
+    // A filtered scan has no cheap exact total, so it keeps the collected length.
+    const totalCount = filter ? collected.length : this.totalNounCountAll
 
     // nextCursor = the (shard, id) of the last RETURNED noun, so the next call
     // resumes immediately after it (works for both cursor and offset callers).
@@ -2409,7 +2412,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     const pagePairs = collected.slice(windowStart, windowStart + limit)
     const ids = pagePairs.map((p) => p.id)
     const hasMore = collected.length > windowStart + limit
-    const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length)
+    // ALL-visibility scalar, unclamped — same law as getNouns() above.
+    const totalCount = filter ? collected.length : this.totalNounCountAll
 
     let nextCursor: string | undefined = undefined
     if (hasMore && pagePairs.length > 0) {
@@ -2641,13 +2645,13 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     const hasMore = collected.length > windowStart + limit
 
     // totalCount must be the TRUE dataset total, not this peeked page. For the
-    // unfiltered scan the authoritative total is the O(1) `totalVerbCount` counter
-    // (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` guards a
-    // stale counter from under-reporting. A filtered scan has no cheap exact total,
-    // so it keeps the collected length (a lower bound).
-    const totalCount = filter
-      ? collected.length
-      : Math.max(this.totalVerbCount, collected.length)
+    // unfiltered scan the authoritative total is the O(1) ALL-visibility counter
+    // (`totalVerbCountAll`: isNew-gated, EVERY tier, rehydrated on init) — the walk
+    // itself is unfiltered by tier, so the user-facing `totalVerbCount` (which skips
+    // system/internal edges) would undercount it on every store with a VFS. Never
+    // clamped (see getNouns): a divergence is reported, not hidden. A filtered scan
+    // has no cheap exact total, so it keeps the collected length (a lower bound).
+    const totalCount = filter ? collected.length : this.totalVerbCountAll
 
     // nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
     // resumes immediately after it — for both cursor and offset callers (an offset
@@ -3455,6 +3459,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
     const isCounted = isCountedVisibility(newVisibility)
 
+    // ALL-visibility ledger: every NEW canonical record is +1 regardless of tier
+    // (the unfiltered walk yields it, so the denominator must count it). The
+    // counted branch below persists for public/internal records; a hidden new
+    // record persists here so the ALL scalar never lags the tree.
+    if (isNew) {
+      this.totalNounCountAll++
+      if (!(metadata.noun && isCounted)) {
+        this.scheduleCountPersist().catch(() => {
+          // Ignore persist errors — the in-memory count is authoritative; a later op retries.
+        })
+      }
+    }
+
     // CRITICAL FIX: Increment count for new entities
     // This runs AFTER metadata is saved, guaranteeing type information is available
     // Uses synchronous increment since storage operations are already serialized
@@ -3858,6 +3875,20 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     await this.deleteCanonicalObject(path)
     const record = read ?? priorRecord
 
+    // ALL-visibility ledger: a PROVEN delete (the record was read, or the caller
+    // carried its prior image) is −1 regardless of tier. A delete that can prove
+    // nothing never guesses — it marks the ledger suspect (loud, persisted) and the
+    // sanctioned recount restores exactness.
+    if (record) {
+      if (this.totalNounCountAll > 0) this.totalNounCountAll--
+      else this.markAllCountsSuspect('noun', id)
+    } else {
+      this.markAllCountsSuspect('noun', id)
+    }
+    this.scheduleCountPersist().catch(() => {
+      // Ignore persist errors — the in-memory count is authoritative; a later op retries.
+    })
+
     const priorType = record?.noun as NounType | undefined
     // 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
     // (gated in `saveNounMetadata_internal()`), so it must not be decremented here either.
@@ -3991,6 +4022,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     // Fixes Bug #2: Count synchronization failure during relate() and import()
     // 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
     if (isNew) {
+      // ALL-visibility ledger: every new edge is +1 regardless of tier (the
+      // unfiltered walk yields VFS/system edges too; the denominator must count them).
+      this.totalVerbCountAll++
       if (isCounted) {
         this.incrementVerbCount(verbType)
       } else {
@@ -4052,6 +4086,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     await this.deleteCanonicalObject(path)
     const record = read ?? priorRecord
 
+    // ALL-visibility ledger: proven delete −1 regardless of tier; an unprovable
+    // delete marks the ledger suspect instead of guessing (see deleteNounMetadata).
+    if (record) {
+      if (this.totalVerbCountAll > 0) this.totalVerbCountAll--
+      else this.markAllCountsSuspect('verb', id)
+    } else {
+      this.markAllCountsSuspect('verb', id)
+    }
+    this.scheduleCountPersist().catch(() => {
+      // Ignore persist errors — in-memory count is authoritative; a later op retries.
+    })
+
     const priorVerb = record?.verb as VerbType | undefined
     // Symmetric count decrement (previously OMITTED — verb deletes touched neither the
     // scalar total nor the per-type bucket, so both inflated permanently). A COUNTED
@@ -4497,6 +4543,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     // walk, every counter rollup rebuilt and persisted from it.
     const countedNouns = new Map<string, number>()
     const countedVerbs = new Map<string, number>()
+    // ALL-visibility scalars: one per canonical record the walk yields, every
+    // tier, readable or not — the same population the unfiltered getNouns()/
+    // getVerbs() walks enumerate, so `totalCount` and this recount agree by
+    // construction.
+    let allNouns = 0
+    let allVerbs = 0
 
     // Scan noun shards
     for (let shard = 0; shard < 256; shard++) {
@@ -4508,6 +4560,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
         for (const path of paths) {
           if (!path.includes('/metadata.json')) continue
+          allNouns++
 
           try {
             const metadata = await this.readCanonicalObject(path)
@@ -4540,6 +4593,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
 
         for (const path of paths) {
           if (!path.includes('/metadata.json')) continue
+          allVerbs++
 
           try {
             const metadata = await this.readCanonicalObject(path)
@@ -4576,10 +4630,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
     this.verbCounts = countedVerbs
     this.totalNounCount = totalNouns
     this.totalVerbCount = totalVerbs
+    // The ALL scalars are exact again and the suspect flag clears — this walk
+    // IS the proof an unprovable delete could not give.
+    const nounsAllBefore = this.totalNounCountAll
+    const verbsAllBefore = this.totalVerbCountAll
+    this.totalNounCountAll = allNouns
+    this.totalVerbCountAll = allVerbs
+    this.allCountsSuspect = false
     this.countCache.clear()
     await this.persistCounts()
 
-    prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`)
+    prodLog.info(
+      `[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
+        `ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
+        (nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
+          ? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
+          : ' (unchanged)') +
+        ` — scalar + per-type persisted`
+    )
   }
 
   /**