feat(storage): the canonical count ledger — ALL-visibility scalars, unclamped totals, suspect-on-unprovable-delete
The storage-level unfiltered getNouns()/getVerbs() walks enumerate every
tier, but their totalCount reported the user-facing scalar, which skips
system/internal records on the write path — so a derived-index coverage
ledger comparing its posted count against that total would read
"over-posted by N" on every store with a VFS. This adds the ledger's real
denominators:
- totalNounCountAll / totalVerbCountAll: +1 for every new canonical record
regardless of tier, −1 for every PROVEN delete (record read, or the
caller's prior image), persisted in counts.json beside the counted
scalars, recomputed by the sanctioned recount (rebuildTypeCounts).
- The unfiltered storage-level totalCount is now the ALL scalar and is
never clamped: Math.max(scalar, scanned) could only move a scalar up, so
an inflated counter hid forever; a divergence is now visible and healed
by repairIndex().
- A delete that cannot prove the record existed never decrements on faith:
it marks the ledger SUSPECT (persisted, narrated once per session) and
the recount clears the flag with proof.
- getCanonicalCounts() on StorageAdapter (optional) exposes {counted, all}
per family plus the suspect flag — O(1), no I/O.
- A counts.json written before the ledger existed derives both scalars
once from the canonical id tree at open and persists them; absent keys
are a legacy file, never a zero.
User-facing getNounCount()/getVerbCount() are unchanged.
Pinned in tests/integration/canonical-count-ledger.test.ts (5 laws).
This commit is contained in:
parent
607e9f5492
commit
7c8c8be30c
6 changed files with 391 additions and 15 deletions
|
|
@ -792,6 +792,19 @@ export interface DerivedFamilyDeclaration {
|
||||||
rebuildable?: boolean
|
rebuildable?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @description The canonical count ledger a storage adapter maintains on its
|
||||||
|
* write path: per family, the user-facing `counted` scalar and the
|
||||||
|
* ALL-visibility `all` scalar (every tier — the coverage-ledger denominator).
|
||||||
|
* See {@link StorageAdapter.getCanonicalCounts}.
|
||||||
|
*/
|
||||||
|
export interface CanonicalCounts {
|
||||||
|
nouns: { counted: number; all: number }
|
||||||
|
verbs: { counted: number; all: number }
|
||||||
|
/** An unprovable delete has left the `all` scalars unverified since the last recount. */
|
||||||
|
suspect: boolean
|
||||||
|
}
|
||||||
|
|
||||||
export interface StorageAdapter {
|
export interface StorageAdapter {
|
||||||
init(): Promise<void>
|
init(): Promise<void>
|
||||||
|
|
||||||
|
|
@ -1293,6 +1306,19 @@ export interface StorageAdapter {
|
||||||
*/
|
*/
|
||||||
getVerbCount(): Promise<number>
|
getVerbCount(): Promise<number>
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The canonical count ledger — O(1), no I/O. `counted` mirrors
|
||||||
|
* `getNounCount()` / `getVerbCount()` (public + internal tiers); `all` is
|
||||||
|
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||||
|
* against — the denominator a derived-index provider's coverage ledger
|
||||||
|
* subtracts from. `suspect` is `true` when an unprovable delete has left
|
||||||
|
* `all` unverified since the last sanctioned recount (`repairIndex()`).
|
||||||
|
* Optional: adapters without the ledger omit it; a consumer treats absence
|
||||||
|
* as "no denominator", never as zero.
|
||||||
|
* @returns Both scalars per family plus the suspect flag.
|
||||||
|
*/
|
||||||
|
getCanonicalCounts?(): Promise<CanonicalCounts>
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* OPTIONAL — create a pre-upgrade backup of the whole store and return its
|
* OPTIONAL — create a pre-upgrade backup of the whole store and return its
|
||||||
* location, or `null` when there is nothing to back up (empty store). On the
|
* location, or `null` when there is nothing to back up (empty store). On the
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,8 @@ import {
|
||||||
HNSWNounWithMetadata,
|
HNSWNounWithMetadata,
|
||||||
HNSWVerbWithMetadata,
|
HNSWVerbWithMetadata,
|
||||||
NounMetadata,
|
NounMetadata,
|
||||||
VerbMetadata
|
VerbMetadata,
|
||||||
|
CanonicalCounts,
|
||||||
} from '../../coreTypes.js'
|
} from '../../coreTypes.js'
|
||||||
import { StorageBatchConfig } from '../baseStorage.js'
|
import { StorageBatchConfig } from '../baseStorage.js'
|
||||||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
||||||
|
|
@ -1028,6 +1029,28 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
// Universal count tracking - O(1) operations
|
// Universal count tracking - O(1) operations
|
||||||
protected totalNounCount = 0
|
protected totalNounCount = 0
|
||||||
protected totalVerbCount = 0
|
protected totalVerbCount = 0
|
||||||
|
/**
|
||||||
|
* The ALL-visibility canonical scalars — every noun / verb the unfiltered
|
||||||
|
* storage walk yields, system and internal tiers included. These are the
|
||||||
|
* denominators a derived-index provider's coverage ledger subtracts from
|
||||||
|
* (`posted === all` is the whole-store coverage verdict); the user-facing
|
||||||
|
* `totalNounCount` / `totalVerbCount` skip hidden tiers by design and can
|
||||||
|
* never serve as a ledger denominator. Maintained on the write path
|
||||||
|
* (every new record +1, every proven delete −1), persisted beside the
|
||||||
|
* counted scalars, recomputed by the sanctioned recount. Never clamped.
|
||||||
|
*/
|
||||||
|
protected totalNounCountAll = 0
|
||||||
|
protected totalVerbCountAll = 0
|
||||||
|
/**
|
||||||
|
* `true` when a delete could not prove whether the record existed (no
|
||||||
|
* canonical read, no caller-provided prior) — the ALL scalar may be off by
|
||||||
|
* the unprovable deletes since. Loud, persisted, and cleared only by the
|
||||||
|
* sanctioned recount; a consumer reading the scalar as a ledger denominator
|
||||||
|
* must treat a suspect scalar as unverified, never as exact.
|
||||||
|
*/
|
||||||
|
protected allCountsSuspect = false
|
||||||
|
/** One narration per session for the suspect transition (never per delete). */
|
||||||
|
private allCountsSuspectNarrated = false
|
||||||
protected entityCounts: Map<string, number> = new Map() // type -> count
|
protected entityCounts: Map<string, number> = new Map() // type -> count
|
||||||
protected verbCounts: Map<string, number> = new Map() // verb type -> count
|
protected verbCounts: Map<string, number> = new Map() // verb type -> count
|
||||||
protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
|
protected countCache: Map<string, { count: number; timestamp: number }> = new Map()
|
||||||
|
|
@ -1056,6 +1079,43 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
||||||
return this.totalVerbCount
|
return this.totalVerbCount
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The canonical count ledger — O(1), no I/O. `counted` is the user-facing
|
||||||
|
* scalar (public/internal tiers, what `getNounCount()` returns); `all` is
|
||||||
|
* the ALL-visibility scalar every unfiltered storage walk is measured
|
||||||
|
* against (the coverage-ledger denominator for derived-index providers).
|
||||||
|
* `suspect` is `true` when an unprovable delete has made `all` unverified
|
||||||
|
* since the last sanctioned recount (`rebuildTypeCounts`).
|
||||||
|
* @returns Both scalars per family plus the suspect flag.
|
||||||
|
*/
|
||||||
|
async getCanonicalCounts(): Promise<CanonicalCounts> {
|
||||||
|
return {
|
||||||
|
nouns: { counted: this.totalNounCount, all: this.totalNounCountAll },
|
||||||
|
verbs: { counted: this.totalVerbCount, all: this.totalVerbCountAll },
|
||||||
|
suspect: this.allCountsSuspect
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Mark the ALL scalars unverified after a delete that could not prove the
|
||||||
|
* record existed. Narrates ONCE per session (the flag is what persists);
|
||||||
|
* the sanctioned recount clears it.
|
||||||
|
* @param family - Which family's delete was unprovable.
|
||||||
|
* @param id - The id whose existence could not be established.
|
||||||
|
*/
|
||||||
|
protected markAllCountsSuspect(family: 'noun' | 'verb', id: string): void {
|
||||||
|
this.allCountsSuspect = true
|
||||||
|
if (!this.allCountsSuspectNarrated) {
|
||||||
|
this.allCountsSuspectNarrated = true
|
||||||
|
console.warn(
|
||||||
|
`[Storage] ${family} delete of ${id} could not prove the record existed ` +
|
||||||
|
`(no canonical read, no prior record) — the ALL-visibility count ledger is ` +
|
||||||
|
`SUSPECT until brain.repairIndex() recounts. Further unprovable deletes ` +
|
||||||
|
`this session are counted silently under the same flag.`
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Increment count for entity type - O(1) operation.
|
* Increment count for entity type - O(1) operation.
|
||||||
* Concurrency is handled by the process-global mutex
|
* Concurrency is handled by the process-global mutex
|
||||||
|
|
|
||||||
|
|
@ -2561,6 +2561,33 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.totalNounCount = counts.totalNounCount || 0
|
this.totalNounCount = counts.totalNounCount || 0
|
||||||
this.totalVerbCount = counts.totalVerbCount || 0
|
this.totalVerbCount = counts.totalVerbCount || 0
|
||||||
|
|
||||||
|
// The ALL-visibility scalars (ledger denominators). A counts.json
|
||||||
|
// written before they existed carries neither key: derive both ONCE
|
||||||
|
// from the canonical id tree (an id-directory listing — O(ids), no
|
||||||
|
// record reads), persist, and never scan again. Absent keys are a
|
||||||
|
// legacy file, not a zero — a zero here would make every provider's
|
||||||
|
// coverage ledger read "over-posted" on a populated store.
|
||||||
|
if (
|
||||||
|
typeof counts.totalNounCountAll === 'number' &&
|
||||||
|
typeof counts.totalVerbCountAll === 'number'
|
||||||
|
) {
|
||||||
|
this.totalNounCountAll = counts.totalNounCountAll
|
||||||
|
this.totalVerbCountAll = counts.totalVerbCountAll
|
||||||
|
this.allCountsSuspect = counts.allCountsSuspect === true
|
||||||
|
} else {
|
||||||
|
const nouns = await this.scanCanonicalEntities('nouns')
|
||||||
|
const verbs = await this.scanCanonicalEntities('verbs')
|
||||||
|
this.totalNounCountAll = nouns.count
|
||||||
|
this.totalVerbCountAll = verbs.count
|
||||||
|
this.allCountsSuspect = false
|
||||||
|
console.warn(
|
||||||
|
`[FileSystemStorage] counts.json predates the ALL-visibility count ledger — ` +
|
||||||
|
`derived once from the canonical id tree (${nouns.count} nouns, ${verbs.count} verbs, ` +
|
||||||
|
`every tier) and persisted; no further scan.`
|
||||||
|
)
|
||||||
|
await this.persistCounts()
|
||||||
|
}
|
||||||
|
|
||||||
// Also populate the cache for backward compatibility
|
// Also populate the cache for backward compatibility
|
||||||
this.countCache.set('nouns_count', {
|
this.countCache.set('nouns_count', {
|
||||||
count: this.totalNounCount,
|
count: this.totalNounCount,
|
||||||
|
|
@ -2596,6 +2623,10 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
this.totalNounCount = nouns.count
|
this.totalNounCount = nouns.count
|
||||||
const verbs = await this.scanCanonicalEntities('verbs')
|
const verbs = await this.scanCanonicalEntities('verbs')
|
||||||
this.totalVerbCount = verbs.count
|
this.totalVerbCount = verbs.count
|
||||||
|
// The id-tree scan counts every tier — it IS the ALL-visibility ledger.
|
||||||
|
this.totalNounCountAll = nouns.count
|
||||||
|
this.totalVerbCountAll = verbs.count
|
||||||
|
this.allCountsSuspect = false
|
||||||
|
|
||||||
// Sample some entities for the type distribution (don't read all).
|
// Sample some entities for the type distribution (don't read all).
|
||||||
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
// Read the metadata files DIRECTLY with fs — this runs inside init(),
|
||||||
|
|
@ -2693,6 +2724,11 @@ export class FileSystemStorage extends BaseStorage {
|
||||||
verbCounts: Object.fromEntries(this.verbCounts),
|
verbCounts: Object.fromEntries(this.verbCounts),
|
||||||
totalNounCount: this.totalNounCount,
|
totalNounCount: this.totalNounCount,
|
||||||
totalVerbCount: this.totalVerbCount,
|
totalVerbCount: this.totalVerbCount,
|
||||||
|
// ALL-visibility ledger scalars (+ the suspect flag) — absent in files
|
||||||
|
// written before the ledger existed; initializeCounts() derives them once.
|
||||||
|
totalNounCountAll: this.totalNounCountAll,
|
||||||
|
totalVerbCountAll: this.totalVerbCountAll,
|
||||||
|
allCountsSuspect: this.allCountsSuspect,
|
||||||
lastUpdated: new Date().toISOString()
|
lastUpdated: new Date().toISOString()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -540,6 +540,10 @@ export class MemoryStorage extends BaseStorage {
|
||||||
|
|
||||||
this.totalNounCount = totalNouns
|
this.totalNounCount = totalNouns
|
||||||
this.totalVerbCount = totalVerbs
|
this.totalVerbCount = totalVerbs
|
||||||
|
// A scan of every canonical record IS the ALL-visibility count.
|
||||||
|
this.totalNounCountAll = totalNouns
|
||||||
|
this.totalVerbCountAll = totalVerbs
|
||||||
|
this.allCountsSuspect = false
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -2273,11 +2273,14 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
||||||
// unfiltered case the authoritative total is the O(1) counter maintained on
|
// unfiltered case the authoritative total is the O(1) counter maintained on
|
||||||
// every add/delete (rehydrated on init); `Math.max` guards a stale counter. A
|
// every add/delete (rehydrated on init) — the ALL-visibility scalar, because
|
||||||
// filtered scan has no cheap exact total, so it keeps the collected length.
|
// this walk is unfiltered by tier (system/internal records are in `collected`).
|
||||||
const totalCount = filter
|
// Never clamped: `Math.max(scalar, scanned)` could only ever move the scalar
|
||||||
? collected.length
|
// UP, so an inflated counter could never correct itself and a divergence was
|
||||||
: Math.max(this.totalNounCount, collected.length)
|
// hidden instead of reported. A scalar that disagrees with the walk is the
|
||||||
|
// canonical-count-ledger invariant's job, healed by the sanctioned recount.
|
||||||
|
// A filtered scan has no cheap exact total, so it keeps the collected length.
|
||||||
|
const totalCount = filter ? collected.length : this.totalNounCountAll
|
||||||
|
|
||||||
// nextCursor = the (shard, id) of the last RETURNED noun, so the next call
|
// nextCursor = the (shard, id) of the last RETURNED noun, so the next call
|
||||||
// resumes immediately after it (works for both cursor and offset callers).
|
// resumes immediately after it (works for both cursor and offset callers).
|
||||||
|
|
@ -2409,7 +2412,8 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const pagePairs = collected.slice(windowStart, windowStart + limit)
|
const pagePairs = collected.slice(windowStart, windowStart + limit)
|
||||||
const ids = pagePairs.map((p) => p.id)
|
const ids = pagePairs.map((p) => p.id)
|
||||||
const hasMore = collected.length > windowStart + limit
|
const hasMore = collected.length > windowStart + limit
|
||||||
const totalCount = filter ? collected.length : Math.max(this.totalNounCount, collected.length)
|
// ALL-visibility scalar, unclamped — same law as getNouns() above.
|
||||||
|
const totalCount = filter ? collected.length : this.totalNounCountAll
|
||||||
|
|
||||||
let nextCursor: string | undefined = undefined
|
let nextCursor: string | undefined = undefined
|
||||||
if (hasMore && pagePairs.length > 0) {
|
if (hasMore && pagePairs.length > 0) {
|
||||||
|
|
@ -2641,13 +2645,13 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const hasMore = collected.length > windowStart + limit
|
const hasMore = collected.length > windowStart + limit
|
||||||
|
|
||||||
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
// totalCount must be the TRUE dataset total, not this peeked page. For the
|
||||||
// unfiltered scan the authoritative total is the O(1) `totalVerbCount` counter
|
// unfiltered scan the authoritative total is the O(1) ALL-visibility counter
|
||||||
// (isNew-gated, visibility-filtered, rehydrated on init); `Math.max` guards a
|
// (`totalVerbCountAll`: isNew-gated, EVERY tier, rehydrated on init) — the walk
|
||||||
// stale counter from under-reporting. A filtered scan has no cheap exact total,
|
// itself is unfiltered by tier, so the user-facing `totalVerbCount` (which skips
|
||||||
// so it keeps the collected length (a lower bound).
|
// system/internal edges) would undercount it on every store with a VFS. Never
|
||||||
const totalCount = filter
|
// clamped (see getNouns): a divergence is reported, not hidden. A filtered scan
|
||||||
? collected.length
|
// has no cheap exact total, so it keeps the collected length (a lower bound).
|
||||||
: Math.max(this.totalVerbCount, collected.length)
|
const totalCount = filter ? collected.length : this.totalVerbCountAll
|
||||||
|
|
||||||
// nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
|
// nextCursor encodes the (shard, id) of the LAST RETURNED verb so the next call
|
||||||
// resumes immediately after it — for both cursor and offset callers (an offset
|
// resumes immediately after it — for both cursor and offset callers (an offset
|
||||||
|
|
@ -3455,6 +3459,19 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
|
const wasCounted = isNew ? false : isCountedVisibility(existingMetadata?.visibility)
|
||||||
const isCounted = isCountedVisibility(newVisibility)
|
const isCounted = isCountedVisibility(newVisibility)
|
||||||
|
|
||||||
|
// ALL-visibility ledger: every NEW canonical record is +1 regardless of tier
|
||||||
|
// (the unfiltered walk yields it, so the denominator must count it). The
|
||||||
|
// counted branch below persists for public/internal records; a hidden new
|
||||||
|
// record persists here so the ALL scalar never lags the tree.
|
||||||
|
if (isNew) {
|
||||||
|
this.totalNounCountAll++
|
||||||
|
if (!(metadata.noun && isCounted)) {
|
||||||
|
this.scheduleCountPersist().catch(() => {
|
||||||
|
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// CRITICAL FIX: Increment count for new entities
|
// CRITICAL FIX: Increment count for new entities
|
||||||
// This runs AFTER metadata is saved, guaranteeing type information is available
|
// This runs AFTER metadata is saved, guaranteeing type information is available
|
||||||
// Uses synchronous increment since storage operations are already serialized
|
// Uses synchronous increment since storage operations are already serialized
|
||||||
|
|
@ -3858,6 +3875,20 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
await this.deleteCanonicalObject(path)
|
await this.deleteCanonicalObject(path)
|
||||||
const record = read ?? priorRecord
|
const record = read ?? priorRecord
|
||||||
|
|
||||||
|
// ALL-visibility ledger: a PROVEN delete (the record was read, or the caller
|
||||||
|
// carried its prior image) is −1 regardless of tier. A delete that can prove
|
||||||
|
// nothing never guesses — it marks the ledger suspect (loud, persisted) and the
|
||||||
|
// sanctioned recount restores exactness.
|
||||||
|
if (record) {
|
||||||
|
if (this.totalNounCountAll > 0) this.totalNounCountAll--
|
||||||
|
else this.markAllCountsSuspect('noun', id)
|
||||||
|
} else {
|
||||||
|
this.markAllCountsSuspect('noun', id)
|
||||||
|
}
|
||||||
|
this.scheduleCountPersist().catch(() => {
|
||||||
|
// Ignore persist errors — the in-memory count is authoritative; a later op retries.
|
||||||
|
})
|
||||||
|
|
||||||
const priorType = record?.noun as NounType | undefined
|
const priorType = record?.noun as NounType | undefined
|
||||||
// 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
|
// 8.0 visibility: an internal/system entity was never added to `nounCountsByType`
|
||||||
// (gated in `saveNounMetadata_internal()`), so it must not be decremented here either.
|
// (gated in `saveNounMetadata_internal()`), so it must not be decremented here either.
|
||||||
|
|
@ -3991,6 +4022,9 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// Fixes Bug #2: Count synchronization failure during relate() and import()
|
// Fixes Bug #2: Count synchronization failure during relate() and import()
|
||||||
// 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
|
// 8.0: skip the user-facing total for internal/system edges (counts.json + getVerbCount()).
|
||||||
if (isNew) {
|
if (isNew) {
|
||||||
|
// ALL-visibility ledger: every new edge is +1 regardless of tier (the
|
||||||
|
// unfiltered walk yields VFS/system edges too; the denominator must count them).
|
||||||
|
this.totalVerbCountAll++
|
||||||
if (isCounted) {
|
if (isCounted) {
|
||||||
this.incrementVerbCount(verbType)
|
this.incrementVerbCount(verbType)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -4052,6 +4086,18 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
await this.deleteCanonicalObject(path)
|
await this.deleteCanonicalObject(path)
|
||||||
const record = read ?? priorRecord
|
const record = read ?? priorRecord
|
||||||
|
|
||||||
|
// ALL-visibility ledger: proven delete −1 regardless of tier; an unprovable
|
||||||
|
// delete marks the ledger suspect instead of guessing (see deleteNounMetadata).
|
||||||
|
if (record) {
|
||||||
|
if (this.totalVerbCountAll > 0) this.totalVerbCountAll--
|
||||||
|
else this.markAllCountsSuspect('verb', id)
|
||||||
|
} else {
|
||||||
|
this.markAllCountsSuspect('verb', id)
|
||||||
|
}
|
||||||
|
this.scheduleCountPersist().catch(() => {
|
||||||
|
// Ignore persist errors — in-memory count is authoritative; a later op retries.
|
||||||
|
})
|
||||||
|
|
||||||
const priorVerb = record?.verb as VerbType | undefined
|
const priorVerb = record?.verb as VerbType | undefined
|
||||||
// Symmetric count decrement (previously OMITTED — verb deletes touched neither the
|
// Symmetric count decrement (previously OMITTED — verb deletes touched neither the
|
||||||
// scalar total nor the per-type bucket, so both inflated permanently). A COUNTED
|
// scalar total nor the per-type bucket, so both inflated permanently). A COUNTED
|
||||||
|
|
@ -4497,6 +4543,12 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
// walk, every counter rollup rebuilt and persisted from it.
|
// walk, every counter rollup rebuilt and persisted from it.
|
||||||
const countedNouns = new Map<string, number>()
|
const countedNouns = new Map<string, number>()
|
||||||
const countedVerbs = new Map<string, number>()
|
const countedVerbs = new Map<string, number>()
|
||||||
|
// ALL-visibility scalars: one per canonical record the walk yields, every
|
||||||
|
// tier, readable or not — the same population the unfiltered getNouns()/
|
||||||
|
// getVerbs() walks enumerate, so `totalCount` and this recount agree by
|
||||||
|
// construction.
|
||||||
|
let allNouns = 0
|
||||||
|
let allVerbs = 0
|
||||||
|
|
||||||
// Scan noun shards
|
// Scan noun shards
|
||||||
for (let shard = 0; shard < 256; shard++) {
|
for (let shard = 0; shard < 256; shard++) {
|
||||||
|
|
@ -4508,6 +4560,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
if (!path.includes('/metadata.json')) continue
|
if (!path.includes('/metadata.json')) continue
|
||||||
|
allNouns++
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const metadata = await this.readCanonicalObject(path)
|
const metadata = await this.readCanonicalObject(path)
|
||||||
|
|
@ -4540,6 +4593,7 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
|
|
||||||
for (const path of paths) {
|
for (const path of paths) {
|
||||||
if (!path.includes('/metadata.json')) continue
|
if (!path.includes('/metadata.json')) continue
|
||||||
|
allVerbs++
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const metadata = await this.readCanonicalObject(path)
|
const metadata = await this.readCanonicalObject(path)
|
||||||
|
|
@ -4576,10 +4630,24 @@ export abstract class BaseStorage extends BaseStorageAdapter {
|
||||||
this.verbCounts = countedVerbs
|
this.verbCounts = countedVerbs
|
||||||
this.totalNounCount = totalNouns
|
this.totalNounCount = totalNouns
|
||||||
this.totalVerbCount = totalVerbs
|
this.totalVerbCount = totalVerbs
|
||||||
|
// The ALL scalars are exact again and the suspect flag clears — this walk
|
||||||
|
// IS the proof an unprovable delete could not give.
|
||||||
|
const nounsAllBefore = this.totalNounCountAll
|
||||||
|
const verbsAllBefore = this.totalVerbCountAll
|
||||||
|
this.totalNounCountAll = allNouns
|
||||||
|
this.totalVerbCountAll = allVerbs
|
||||||
|
this.allCountsSuspect = false
|
||||||
this.countCache.clear()
|
this.countCache.clear()
|
||||||
await this.persistCounts()
|
await this.persistCounts()
|
||||||
|
|
||||||
prodLog.info(`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (scalar + per-type persisted)`)
|
prodLog.info(
|
||||||
|
`[BaseStorage] Rebuilt counts: ${totalNouns} nouns, ${totalVerbs} verbs (user-facing); ` +
|
||||||
|
`ALL-visibility ledger ${allNouns} nouns / ${allVerbs} verbs` +
|
||||||
|
(nounsAllBefore !== allNouns || verbsAllBefore !== allVerbs
|
||||||
|
? ` (corrected from ${nounsAllBefore} / ${verbsAllBefore})`
|
||||||
|
: ' (unchanged)') +
|
||||||
|
` — scalar + per-type persisted`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
182
tests/integration/canonical-count-ledger.test.ts
Normal file
182
tests/integration/canonical-count-ledger.test.ts
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
/**
|
||||||
|
* @module tests/integration/canonical-count-ledger
|
||||||
|
* @description The canonical count ledger — the denominators a derived-index
|
||||||
|
* provider's coverage ledger subtracts from. Laws under test:
|
||||||
|
* (1) THE ALL-VISIBILITY SCALAR IS THE UNFILTERED WALK'S TOTAL — the
|
||||||
|
* storage-level `getNouns()` / `getVerbs()` `totalCount` counts EVERY tier
|
||||||
|
* (system, internal, public) because the walk yields every tier; the
|
||||||
|
* user-facing `getNounCount()` / `getVerbCount()` keep skipping hidden
|
||||||
|
* tiers. A ledger built on the user-facing scalar would read "over-posted"
|
||||||
|
* on every store with a VFS — the mismatch this pin makes unbuildable.
|
||||||
|
* (2) NEVER CLAMPED — `Math.max(scalar, scanned)` could only move a scalar up,
|
||||||
|
* so an inflated counter hid forever. An inflated scalar is now VISIBLE
|
||||||
|
* (totalCount ≠ walk) and the sanctioned recount heals it, durably.
|
||||||
|
* (3) NEVER GUESSED — a delete that cannot prove the record existed marks the
|
||||||
|
* ledger SUSPECT (persisted) instead of decrementing on faith; the recount
|
||||||
|
* clears the flag with proof.
|
||||||
|
* (4) LEGACY FILES DERIVE ONCE — a counts.json written before the ledger is
|
||||||
|
* upgraded from the canonical id tree at open, then persisted.
|
||||||
|
*/
|
||||||
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||||
|
import * as fs from 'node:fs'
|
||||||
|
import * as os from 'node:os'
|
||||||
|
import * as path from 'node:path'
|
||||||
|
import { Brainy } from '../../src/index.js'
|
||||||
|
|
||||||
|
/** Count canonical `<root>/entities/<kind>/<shard>/<id>` directories — every tier. */
|
||||||
|
function countIdDirs(root: string, kind: 'nouns' | 'verbs'): number {
|
||||||
|
const base = path.join(root, 'entities', kind)
|
||||||
|
if (!fs.existsSync(base)) return 0
|
||||||
|
let n = 0
|
||||||
|
for (const shard of fs.readdirSync(base)) {
|
||||||
|
const shardDir = path.join(base, shard)
|
||||||
|
if (!fs.statSync(shardDir).isDirectory()) continue
|
||||||
|
for (const id of fs.readdirSync(shardDir)) {
|
||||||
|
if (fs.statSync(path.join(shardDir, id)).isDirectory()) n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
const countsPath = (root: string) => path.join(root, '_system', 'counts.json')
|
||||||
|
|
||||||
|
describe('canonical count ledger — ALL-visibility scalars, unclamped totals, recount heals', () => {
|
||||||
|
let dir: string
|
||||||
|
let brain: any
|
||||||
|
|
||||||
|
const open = async () => {
|
||||||
|
const b: any = new Brainy({
|
||||||
|
requireSubtype: false,
|
||||||
|
storage: { type: 'filesystem', path: dir },
|
||||||
|
silent: true,
|
||||||
|
dimensions: 384
|
||||||
|
})
|
||||||
|
await b.init()
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
process.env.BRAINY_DETERMINISTIC_EMBEDDINGS = 'true'
|
||||||
|
dir = fs.mkdtempSync(path.join(os.tmpdir(), 'brainy-ledger-'))
|
||||||
|
brain = await open()
|
||||||
|
})
|
||||||
|
afterEach(async () => {
|
||||||
|
await brain.close?.().catch(() => {})
|
||||||
|
fs.rmSync(dir, { recursive: true, force: true })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('the unfiltered walk totalCount is the ALL scalar (every tier); the user-facing count stays counted', async () => {
|
||||||
|
const a = await brain.add({ data: 'public a', type: 'document' })
|
||||||
|
const b = await brain.add({ data: 'internal b', type: 'document', visibility: 'internal' })
|
||||||
|
await brain.relate({ from: a, to: b, type: 'relatedTo', visibility: 'internal' })
|
||||||
|
await brain.vfs.writeFile('/docs/x.txt', 'hello') // VFS: system-tier nouns + Contains edges
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
const ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.suspect).toBe(false)
|
||||||
|
expect(ledger.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
||||||
|
expect(ledger.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
||||||
|
expect(ledger.nouns.counted).toBe(await brain.storage.getNounCount())
|
||||||
|
expect(ledger.verbs.counted).toBe(await brain.storage.getVerbCount())
|
||||||
|
// Hidden tiers exist (the VFS root at minimum, the internal noun, the internal edge):
|
||||||
|
expect(ledger.nouns.all).toBeGreaterThan(ledger.nouns.counted)
|
||||||
|
expect(ledger.verbs.all).toBeGreaterThan(ledger.verbs.counted)
|
||||||
|
|
||||||
|
// The storage-level unfiltered walks report the ALL scalar, and a full page equals it.
|
||||||
|
const nouns = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
|
||||||
|
expect(nouns.totalCount).toBe(ledger.nouns.all)
|
||||||
|
expect(nouns.items.length).toBe(ledger.nouns.all)
|
||||||
|
const verbs = await brain.storage.getVerbs({ pagination: { limit: 1000, offset: 0 } })
|
||||||
|
expect(verbs.totalCount).toBe(ledger.verbs.all)
|
||||||
|
expect(verbs.items.length).toBe(ledger.verbs.all)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('proven deletes move the ALL scalar for every tier and the ledger stays exact and unsuspect', async () => {
|
||||||
|
const p = await brain.add({ data: 'public p', type: 'document' })
|
||||||
|
const q = await brain.add({ data: 'internal q', type: 'document', visibility: 'internal' })
|
||||||
|
await brain.relate({ from: p, to: q, type: 'relatedTo' })
|
||||||
|
await brain.flush()
|
||||||
|
const before = await brain.storage.getCanonicalCounts()
|
||||||
|
|
||||||
|
await brain.remove(q) // cascades the edge
|
||||||
|
await brain.remove(p)
|
||||||
|
await brain.flush()
|
||||||
|
|
||||||
|
const after = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(after.nouns.all).toBe(before.nouns.all - 2)
|
||||||
|
expect(after.verbs.all).toBe(before.verbs.all - 1)
|
||||||
|
expect(after.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
||||||
|
expect(after.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
||||||
|
expect(after.nouns.counted).toBe(before.nouns.counted - 1)
|
||||||
|
expect(after.suspect).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a legacy counts.json without the ALL keys is derived once from the id tree and persisted', async () => {
|
||||||
|
await brain.add({ data: 'one', type: 'document' })
|
||||||
|
await brain.add({ data: 'two', type: 'document', visibility: 'internal' })
|
||||||
|
await brain.vfs.writeFile('/a.txt', 'x')
|
||||||
|
await brain.flush()
|
||||||
|
await brain.close()
|
||||||
|
|
||||||
|
const raw = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
||||||
|
expect(typeof raw.totalNounCountAll).toBe('number')
|
||||||
|
delete raw.totalNounCountAll
|
||||||
|
delete raw.totalVerbCountAll
|
||||||
|
delete raw.allCountsSuspect
|
||||||
|
fs.writeFileSync(countsPath(dir), JSON.stringify(raw, null, 2))
|
||||||
|
|
||||||
|
brain = await open()
|
||||||
|
const ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.nouns.all).toBe(countIdDirs(dir, 'nouns'))
|
||||||
|
expect(ledger.verbs.all).toBe(countIdDirs(dir, 'verbs'))
|
||||||
|
expect(ledger.suspect).toBe(false)
|
||||||
|
const persisted = JSON.parse(fs.readFileSync(countsPath(dir), 'utf-8'))
|
||||||
|
expect(persisted.totalNounCountAll).toBe(ledger.nouns.all)
|
||||||
|
expect(persisted.totalVerbCountAll).toBe(ledger.verbs.all)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an inflated ALL scalar is VISIBLE (unclamped) and healed by repairIndex(), surviving reopen', async () => {
|
||||||
|
for (let i = 0; i < 3; i++) await brain.add({ data: `real ${i}`, type: 'document' })
|
||||||
|
await brain.flush()
|
||||||
|
const truth = countIdDirs(dir, 'nouns')
|
||||||
|
|
||||||
|
;(brain.storage as any).totalNounCountAll = truth + 40
|
||||||
|
await (brain.storage as any).persistCounts()
|
||||||
|
await brain.close()
|
||||||
|
brain = await open()
|
||||||
|
|
||||||
|
// The lie survives reopen AND is observable: totalCount disagrees with the walk.
|
||||||
|
const page = await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })
|
||||||
|
expect(page.totalCount).toBe(truth + 40)
|
||||||
|
expect(page.items.length).toBe(truth)
|
||||||
|
|
||||||
|
await brain.repairIndex()
|
||||||
|
expect((await brain.storage.getCanonicalCounts()).nouns.all).toBe(truth)
|
||||||
|
expect((await brain.storage.getNouns({ pagination: { limit: 1000, offset: 0 } })).totalCount).toBe(truth)
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
brain = await open()
|
||||||
|
expect((await brain.storage.getCanonicalCounts()).nouns.all).toBe(truth)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an unprovable delete marks the ledger SUSPECT (persisted); the recount clears it with proof', async () => {
|
||||||
|
await brain.add({ data: 'anchor', type: 'document' })
|
||||||
|
await brain.flush()
|
||||||
|
const truth = countIdDirs(dir, 'nouns')
|
||||||
|
|
||||||
|
// A ghost: no canonical record, no prior image — nothing to prove existence with.
|
||||||
|
await brain.storage.deleteNounMetadata('00000000-dead-4dea-8dea-000000000000')
|
||||||
|
let ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.suspect).toBe(true)
|
||||||
|
expect(ledger.nouns.all).toBe(truth) // never decremented on faith
|
||||||
|
|
||||||
|
await brain.close()
|
||||||
|
brain = await open()
|
||||||
|
expect((await brain.storage.getCanonicalCounts()).suspect).toBe(true) // the flag persists
|
||||||
|
|
||||||
|
await brain.repairIndex()
|
||||||
|
ledger = await brain.storage.getCanonicalCounts()
|
||||||
|
expect(ledger.suspect).toBe(false)
|
||||||
|
expect(ledger.nouns.all).toBe(truth)
|
||||||
|
})
|
||||||
|
})
|
||||||
Loading…
Add table
Add a link
Reference in a new issue