fix(8.0): getNouns().totalCount reports true total, not page size (port of 7.32.1)

getNounsWithPagination returned collectedNouns.length as totalCount, but the
type-first shard scan early-terminates at offset+limit+1 (peeks one for hasMore)
— so getNouns({ pagination: { limit: 1 } }).totalCount was the page size for any
non-empty brain. The index-rebuild gate calls exactly that, so cold starts
mis-logged "Small dataset (N items)". Now reports the authoritative O(1) noun
counter (rehydrated on init) as the unfiltered total; 8.0's peek-based hasMore is
already correct and unchanged. Filtered scans unchanged.

Regression: tests/unit/storage/getNouns-totalCount.test.ts. Full 8.0 unit suite green (1453).
This commit is contained in:
David Snelling 2026-06-17 14:10:06 -07:00
parent 5eaf579937
commit b2005ff22a
2 changed files with 103 additions and 1 deletions

View file

@ -0,0 +1,88 @@
/**
* @module tests/unit/storage/getNouns-totalCount
* @description Regression for the rebuild-gate count (the 8.0 port of the 7.32.1
* fix; see handoff BRAINY-MMAP-VECTOR-HOOK). The index-rebuild gate reads
* `getNouns({ pagination: { limit: 1 } }).totalCount` to decide whether a brain
* is "small" enough to rebuild inline. The shard-scan pagination
* (`BaseStorage.getNounsWithPagination`) early-terminates at `offset + limit + 1`
* (it peeks one extra item to compute `hasMore`), then returned
* `collectedNouns.length` as `totalCount` i.e. the PAGE size, never the dataset
* total. So every non-empty brain reported a tiny `totalCount` and logged
* "Small dataset (N items)" on cold start regardless of the real corpus size.
* `totalCount` must be the authoritative O(1) noun counter (maintained on every
* add/delete, rehydrated from `counts.json` on init).
*/
import { describe, it, expect } from 'vitest'
import { mkdtempSync, rmSync } from 'fs'
import { tmpdir } from 'os'
import { join } from 'path'
import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js'
import type { NounMetadata } from '../../../src/coreTypes.js'
const DIM = 8
/** A deterministic non-zero vector so the noun record is well-formed. */
function vec(seed: number): number[] {
return Array.from({ length: DIM }, (_, i) => ((seed + i) % 7) / 7 - 0.5)
}
/** Storage shards by UUID, so ids must be 32 hex chars. */
function uuid(i: number): string {
return i.toString(16).padStart(32, '0')
}
/**
* Seed `n` unique nouns. Metadata is saved BEFORE the noun record: it is the
* recommended order (avoids stat drift) and it is also where the O(1) noun
* counter is incremented, so this mirrors the real add() path.
*/
async function seed(storage: FileSystemStorage, n: number): Promise<void> {
for (let i = 0; i < n; i++) {
const id = uuid(i)
await storage.saveNounMetadata(id, {
noun: 'thing',
createdAt: Date.now(),
updatedAt: Date.now()
} as NounMetadata)
await storage.saveNoun({ id, vector: vec(i), connections: new Map(), level: 0 })
}
}
describe('FileSystemStorage.getNouns totalCount (rebuild-gate count)', () => {
it('reports the TRUE total, not the page size, for a 1-item page', async () => {
const dir = mkdtempSync(join(tmpdir(), 'brainy-totalcount-'))
try {
const storage = new FileSystemStorage(dir)
await storage.init()
await seed(storage, 25)
// The exact call the index-rebuild gate makes.
const onePage = await storage.getNouns({ pagination: { limit: 1 } })
expect(onePage.items.length).toBe(1)
// Before the fix this was the peeked page size (limit + 1 == 2).
expect(onePage.totalCount).toBe(25)
expect(onePage.hasMore).toBe(true)
} finally {
rmSync(dir, { recursive: true, force: true })
}
})
it('still reports the true total after a cold reopen (counts rehydrate)', async () => {
const dir = mkdtempSync(join(tmpdir(), 'brainy-totalcount-reopen-'))
try {
const first = new FileSystemStorage(dir)
await first.init()
await seed(first, 25)
// Flush the count to counts.json so a fresh instance rehydrates it.
await (first as unknown as { persistCounts(): Promise<void> }).persistCounts()
// Cold start: a brand-new instance over the same directory.
const reopened = new FileSystemStorage(dir)
await reopened.init()
const page = await reopened.getNouns({ pagination: { limit: 1 } })
expect(page.totalCount).toBe(25)
} finally {
rmSync(dir, { recursive: true, force: true })
}
})
})