diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 0e7a444a..6ba42a7b 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1620,9 +1620,23 @@ export abstract class BaseStorage extends BaseStorageAdapter { const paginatedNouns = collectedNouns.slice(offset, offset + limit) const hasMore = collectedNouns.length > targetCount + // totalCount must be the TRUE dataset total, not the size of this (peeked) + // page. The shard scan early-terminates at `peekCount = offset + limit + 1` + // for memory efficiency, so `collectedNouns.length` only ever reaches the + // page size + 1 — returning it as `totalCount` made every non-empty brain + // look like it held ~`limit` items (e.g. `getNouns({ pagination: { limit: 1 } + // }).totalCount` was 2, tripping the index-rebuild gate's "Small dataset" + // path). For the unfiltered case the authoritative total is the O(1) counter + // maintained on every add/delete and rehydrated on init; `Math.max` guards a + // stale counter from under-reporting below what we collected. A filtered scan + // has no cheap exact total, so it keeps the collected length (a lower bound). + const totalCount = filter + ? collectedNouns.length + : Math.max(this.totalNounCount, collectedNouns.length) + return { items: paginatedNouns, - totalCount: collectedNouns.length, + totalCount, hasMore, nextCursor: hasMore && paginatedNouns.length > 0 ? paginatedNouns[paginatedNouns.length - 1].id diff --git a/tests/unit/storage/getNouns-totalCount.test.ts b/tests/unit/storage/getNouns-totalCount.test.ts new file mode 100644 index 00000000..748b5eec --- /dev/null +++ b/tests/unit/storage/getNouns-totalCount.test.ts @@ -0,0 +1,88 @@ +/** + * @module tests/unit/storage/getNouns-totalCount + * @description Regression for the rebuild-gate count (the 8.0 port of the 7.32.1 + * fix; see handoff BRAINY-MMAP-VECTOR-HOOK). The index-rebuild gate reads + * `getNouns({ pagination: { limit: 1 } }).totalCount` to decide whether a brain + * is "small" enough to rebuild inline. The shard-scan pagination + * (`BaseStorage.getNounsWithPagination`) early-terminates at `offset + limit + 1` + * (it peeks one extra item to compute `hasMore`), then returned + * `collectedNouns.length` as `totalCount` — i.e. the PAGE size, never the dataset + * total. So every non-empty brain reported a tiny `totalCount` and logged + * "Small dataset (N items)" on cold start regardless of the real corpus size. + * `totalCount` must be the authoritative O(1) noun counter (maintained on every + * add/delete, rehydrated from `counts.json` on init). + */ +import { describe, it, expect } from 'vitest' +import { mkdtempSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import type { NounMetadata } from '../../../src/coreTypes.js' + +const DIM = 8 + +/** A deterministic non-zero vector so the noun record is well-formed. */ +function vec(seed: number): number[] { + return Array.from({ length: DIM }, (_, i) => ((seed + i) % 7) / 7 - 0.5) +} + +/** Storage shards by UUID, so ids must be 32 hex chars. */ +function uuid(i: number): string { + return i.toString(16).padStart(32, '0') +} + +/** + * Seed `n` unique nouns. Metadata is saved BEFORE the noun record: it is the + * recommended order (avoids stat drift) and it is also where the O(1) noun + * counter is incremented, so this mirrors the real add() path. + */ +async function seed(storage: FileSystemStorage, n: number): Promise { + for (let i = 0; i < n; i++) { + const id = uuid(i) + await storage.saveNounMetadata(id, { + noun: 'thing', + createdAt: Date.now(), + updatedAt: Date.now() + } as NounMetadata) + await storage.saveNoun({ id, vector: vec(i), connections: new Map(), level: 0 }) + } +} + +describe('FileSystemStorage.getNouns totalCount (rebuild-gate count)', () => { + it('reports the TRUE total, not the page size, for a 1-item page', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-totalcount-')) + try { + const storage = new FileSystemStorage(dir) + await storage.init() + await seed(storage, 25) + + // The exact call the index-rebuild gate makes. + const onePage = await storage.getNouns({ pagination: { limit: 1 } }) + expect(onePage.items.length).toBe(1) + // Before the fix this was the peeked page size (limit + 1 == 2). + expect(onePage.totalCount).toBe(25) + expect(onePage.hasMore).toBe(true) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('still reports the true total after a cold reopen (counts rehydrate)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-totalcount-reopen-')) + try { + const first = new FileSystemStorage(dir) + await first.init() + await seed(first, 25) + // Flush the count to counts.json so a fresh instance rehydrates it. + await (first as unknown as { persistCounts(): Promise }).persistCounts() + + // Cold start: a brand-new instance over the same directory. + const reopened = new FileSystemStorage(dir) + await reopened.init() + const page = await reopened.getNouns({ pagination: { limit: 1 } }) + expect(page.totalCount).toBe(25) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +})