diff --git a/RELEASES.md b/RELEASES.md index ea16c8d2..72cb7d48 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -10,6 +10,44 @@ Full auto-generated changelog: `CHANGELOG.md` · Releases: https://github.com/so --- +## v7.32.1 — 2026-06-17 + +**Affected products:** consumers on `filesystem` / `mmap-filesystem` storage whose logs showed +`🔄 Small dataset (1 items) - rebuilding all indexes…` on cold start, or noise from a +`mmap-vector backend not wired` line on every init. Two fixes. Drop-in; no API or data changes. + +### Fix — `getNouns().totalCount` reports the true total, not the page size + +`storage.getNouns({ pagination: { limit } })` returned `totalCount` equal to the **page size**, not +the dataset total: the type-first shard scan early-terminates at `offset + limit` for memory +efficiency, and the page-collected length was returned as the total. So +`getNouns({ pagination: { limit: 1 } })` reported `totalCount: 1` for **any** non-empty brain. + +The index-rebuild gate uses exactly this call to size the corpus, so a cold start that needs a +rebuild logged `Small dataset (1 items) - rebuilding all indexes…` regardless of the real entity +count (a production deployment saw this for an ~8,800-entity brain — the rebuild then ran from +scratch instead of loading the persisted vector snapshot). + +`getNounsWithPagination` now reports the authoritative O(1) noun counter (maintained on every +add/delete and rehydrated from `counts.json` on init) as the unfiltered `totalCount`, and computes +`hasMore` from it. Filtered scans are unchanged (collected length, a lower bound). Layout-independent +(applies equally to branch/COW layouts). Regression: `tests/unit/storage/getNouns-totalCount.test.ts`. + +### Log — benign "mmap-vector backend not wired" downgraded to debug + +When a native vector provider replaces the JS HNSW index (it owns its own vector storage and exposes +no `setVectorBackend` hook), brainy logged `mmap-vector backend not wired … per-entity reads in use` +on **every** init. This is expected and benign in the native-index model — not a fault, and not by +itself an indication of per-entity reads — but it appeared on every warm and was repeatedly mistaken +for a cold-start cause. It is now a debug-level line (surface it with `BRAINY_LOG_LEVEL=debug`). + +> Note: this release fixes the misleading *count/log*. The remaining cold-start symptom on the +> native line (rebuilding instead of loading the persisted vector snapshot) is resolved when the +> native provider loads its snapshot at construction so the index reports a non-zero size before +> brainy's rebuild gate — already the model in the next major (8.0 + native 3.0). + +--- + ## v7.32.0 — 2026-06-16 **Affected products:** anyone who needs a **portable graph backup**, a partial export, or a diff --git a/src/brainy.ts b/src/brainy.ts index 33bcb91d..01ac6014 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -9473,12 +9473,17 @@ export class Brainy implements BrainyInterface { setVectorBackend?: (backend: MmapVectorBackend) => void } if (typeof indexWithBackend.setVectorBackend !== 'function') { - if (!this.config.silent) { - console.log( - '[brainy] mmap-vector backend not wired (vector index manages its own ' + - 'vector storage; no setVectorBackend hook) — per-entity reads in use' - ) - } + // Expected in the native-vector-index model: a native provider (e.g. + // @soulcraft/cortex) replaces the JS HNSW index and owns its own vector + // storage + persisted snapshot, so there is no setVectorBackend hook to + // wire here. This is benign, not a fault, and does NOT by itself imply + // per-entity reads — keep it at debug level so it never reads as a problem + // in normal operation. (The old console.log fired on every init and was + // repeatedly mistaken for the cold-start cause; see BRAINY-MMAP-VECTOR-HOOK.) + prodLog.debug( + '[brainy] mmap-vector backend not wired (native vector index manages ' + + 'its own vector storage; no setVectorBackend hook)' + ) return } diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index c600038c..ec511292 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -1515,11 +1515,28 @@ export abstract class BaseStorage extends BaseStorageAdapter { // Apply pagination const paginatedNouns = collectedNouns.slice(offset, offset + limit) - const hasMore = collectedNouns.length > targetCount + + // totalCount must be the TRUE dataset total, not the size of this page. + // The shard scan above early-terminates at `targetCount = offset + limit` + // for memory efficiency, so `collectedNouns.length` only ever reaches the + // page size — returning it as `totalCount` made every non-empty brain look + // like it held exactly `limit` items. In particular + // `getNouns({ pagination: { limit: 1 } })` reported `totalCount: 1`, which + // tripped the index-rebuild gate into logging "Small dataset (1 items)" and + // rebuilding from scratch regardless of the real corpus size. For the + // unfiltered case the authoritative total is the O(1) counter maintained on + // every add/delete and rehydrated from `counts.json` on init; `Math.max` + // guards against a stale counter ever under-reporting below what we + // actually collected. A filtered scan has no cheap exact total, so it keeps + // the collected length (a lower bound — unchanged behaviour). + const totalCount = filter + ? collectedNouns.length + : Math.max(this.totalNounCount, collectedNouns.length) + const hasMore = offset + paginatedNouns.length < totalCount return { items: paginatedNouns, - totalCount: collectedNouns.length, + totalCount, hasMore, nextCursor: hasMore && paginatedNouns.length > 0 ? paginatedNouns[paginatedNouns.length - 1].id diff --git a/tests/unit/storage/getNouns-totalCount.test.ts b/tests/unit/storage/getNouns-totalCount.test.ts new file mode 100644 index 00000000..bddc6dbb --- /dev/null +++ b/tests/unit/storage/getNouns-totalCount.test.ts @@ -0,0 +1,88 @@ +/** + * @module tests/unit/storage/getNouns-totalCount + * @description Regression for BRAINY-MMAP-VECTOR-HOOK (Section H). The + * index-rebuild gate (`rebuildIndexesIfNeeded`) reads + * `getNouns({ pagination: { limit: 1 } }).totalCount` to decide whether a brain + * is "small" enough to rebuild inline. The shard-scan pagination + * (`BaseStorage.getNounsWithPagination`) early-terminates at `offset + limit` + * for memory efficiency, then returned `collectedNouns.length` as `totalCount` — + * i.e. the PAGE size, never the dataset total. So every non-empty filesystem + * brain reported `totalCount: 1` and logged "Small dataset (1 items)" on cold + * start, regardless of the real corpus size (a consumer saw this for an + * ~8.8k-entity brain). `totalCount` must be the authoritative O(1) noun counter + * (maintained on every add/delete, rehydrated from `counts.json` on init). + */ +import { describe, it, expect } from 'vitest' +import { mkdtempSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { FileSystemStorage } from '../../../src/storage/adapters/fileSystemStorage.js' +import type { NounMetadata } from '../../../src/coreTypes.js' + +const DIM = 8 + +/** A deterministic non-zero vector so the noun record is well-formed. */ +function vec(seed: number): number[] { + return Array.from({ length: DIM }, (_, i) => ((seed + i) % 7) / 7 - 0.5) +} + +/** Storage shards by UUID, so ids must be 32 hex chars. */ +function uuid(i: number): string { + return i.toString(16).padStart(32, '0') +} + +/** + * Seed `n` unique nouns. Metadata is saved BEFORE the noun record: it is the + * recommended order (avoids stat drift) and it is also where the O(1) noun + * counter is incremented, so this mirrors the real add() path. + */ +async function seed(storage: FileSystemStorage, n: number): Promise { + for (let i = 0; i < n; i++) { + const id = uuid(i) + await storage.saveNounMetadata(id, { + noun: 'thing', + createdAt: Date.now(), + updatedAt: Date.now() + } as NounMetadata) + await storage.saveNoun({ id, vector: vec(i), connections: new Map(), level: 0 }) + } +} + +describe('FileSystemStorage.getNouns totalCount (rebuild-gate count)', () => { + it('reports the TRUE total, not the page size, for a 1-item page', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-totalcount-')) + try { + const storage = new FileSystemStorage(dir) + await storage.init() + await seed(storage, 25) + + // The exact call the index-rebuild gate makes. + const onePage = await storage.getNouns({ pagination: { limit: 1 } }) + expect(onePage.items.length).toBe(1) + // Before the fix this was 1 (collectedNouns.length === limit). + expect(onePage.totalCount).toBe(25) + expect(onePage.hasMore).toBe(true) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it('still reports the true total after a cold reopen (counts rehydrate)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'brainy-totalcount-reopen-')) + try { + const first = new FileSystemStorage(dir) + await first.init() + await seed(first, 25) + // Flush the count to counts.json so a fresh instance rehydrates it. + await (first as unknown as { persistCounts(): Promise }).persistCounts() + + // Cold start: a brand-new instance over the same directory. + const reopened = new FileSystemStorage(dir) + await reopened.init() + const page = await reopened.getNouns({ pagination: { limit: 1 } }) + expect(page.totalCount).toBe(25) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +})