diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 39de77a6..e66ad772 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -508,10 +508,41 @@ export class MetadataIndexManager implements MetadataIndexProvider { this.entityCountsByTypeFixed.fill(0) this.verbCountsByTypeFixed.fill(0) - // Load counts from sparse index (correct source) + // PRIMARY (8.0+): rehydrate per-type counts from the column store's 'noun' + // field — the authoritative on-disk source after a cold reopen. + // + // The chunked sparse-index WRITE path was removed in 7.20.0 (commit + // 11be039): new workspaces persist the 'noun' field ONLY to the column + // store, never to a `__sparse_index__noun` blob. So the legacy sparse + // path below finds nothing and leaves every count at 0 — which is exactly + // why counts.byType/byTypeEnum/topTypes/allNounTypeCounts all read empty + // after close()+reopen while find()/getNounCount() (different sources) + // stay correct. The column store's per-value cardinality matches the warm + // `updateTypeFieldAffinity` counts EXACTLY because both are driven from the + // same `addToIndex` field set, in lockstep, with no visibility gate on + // either — so this rehydration reproduces the warm values precisely. + if (this.columnStore && this.columnStore.getIndexedFields().includes('noun')) { + const nounValues = await this.columnStore.getFilterValues('noun') + for (const value of nounValues) { + const bitmap = await this.columnStore.filter('noun', value) + if (bitmap.size > 0) { + // Use the stored value directly as the key (the legacy sparse path + // did the same): it is already the normalized type string that + // getNounFromIndex/getEntityCountByType expect, so syncTypeCountsToFixed + // — called immediately after lazyLoadCounts in init() — copies it into + // entityCountsByTypeFixed without re-normalization drift. + this.totalEntitiesByType.set(value, bitmap.size) + } + } + prodLog.debug(`✅ Rehydrated type counts from column store: ${this.totalEntitiesByType.size} types`) + return + } + + // LEGACY FALLBACK (pre-7.20.0 workspaces still on the chunked sparse index). const nounSparseIndex = await this.loadSparseIndex('noun') if (!nounSparseIndex) { - // No sparse index yet - counts will be populated as entities are added + // No column-store 'noun' field and no sparse index yet — counts will be + // populated as entities are added. return } @@ -530,7 +561,7 @@ export class MetadataIndexManager implements MetadataIndexProvider { } catch (error) { // Silently fail - counts will be populated as entities are added // This maintains zero-configuration principle - prodLog.debug('Could not load type counts from sparse index:', error) + prodLog.debug('Could not load type counts:', error) } } @@ -2684,10 +2715,11 @@ export class MetadataIndexManager implements MetadataIndexProvider { } /** - * Get all entity types and their counts - O(1) operation - * Fixed - totalEntitiesByType is correctly populated by updateTypeFieldAffinity - * during add operations. lazyLoadCounts was reading wrong data but that doesn't - * affect freshly-added entities within the same session. + * Get all entity types and their counts - O(1) operation. + * `totalEntitiesByType` is populated by `updateTypeFieldAffinity` during add + * operations (warm path) and rehydrated from the column store's 'noun' field + * by `lazyLoadCounts` on init (cold reopen), so this is accurate both within a + * session and after close()+reopen. */ getAllEntityCounts(): Map { return new Map(this.totalEntitiesByType) diff --git a/tests/integration/brainy-phase1c-integration.test.ts b/tests/integration/brainy-phase1c-integration.test.ts index 3f3b6e86..0e18a68b 100644 --- a/tests/integration/brainy-phase1c-integration.test.ts +++ b/tests/integration/brainy-phase1c-integration.test.ts @@ -377,23 +377,69 @@ describe('Brainy - Phase 1c: Type-Aware Integration', () => { silent: true }) - await brainy2.init() // init() is expected to rehydrate type counts from the persisted noun index + await brainy2.init() // init() rehydrates type counts from the persisted column store try { // After reopening a persisted brain, counts.topTypes() must reflect the - // stored data. NOTE (8.0): this currently FAILS — the metadataIndex-backed - // counts.* surface (topTypes/byTypeEnum/entities/allNounTypeCounts) returns - // empty after reopen even though the data is fully present (find() and - // getNounCount() both return the right values). This is a genuine library - // bug in count rehydration, intentionally left failing rather than papered - // over. See the agent's realBugs report for the precise repro. + // stored data. Regression guard for the 8.0 cold-reopen count bug: + // lazyLoadCounts read the dead `__sparse_index__noun` blob (sparse WRITE + // path removed in 7.20.0) and left every per-type count at 0, so + // counts.topTypes/byTypeEnum/allNounTypeCounts returned empty after reopen + // even though find()/getNounCount() were correct. Fixed by rehydrating + // from the column store's 'noun' field. const topTypes = brainy2.counts.topTypes(3) expect(topTypes[0]).toBe('person') // Most common type expect(topTypes[1]).toBe('document') + + // Counts must rehydrate to the EXACT persisted values, not just be ordered. + expect(brainy2.counts.byTypeEnum('person')).toBe(100) + expect(brainy2.counts.byTypeEnum('document')).toBe(50) + expect(await brainy2.counts.byType('person')).toBe(100) + + const allNoun = brainy2.counts.allNounTypeCounts() + expect(allNoun.get('person' as any)).toBe(100) + expect(allNoun.get('document' as any)).toBe(50) } finally { await brainy2.close() } }) + + it('rehydrated per-type counts after cold reopen equal the warm counts exactly', async () => { + // Audit mandate: add N of a type → byTypeEnum(t) === N, both WARM and after + // a close()+reopen, with warm and cold reporting identical maps. + for (let i = 0; i < 7; i++) { + await brainy.add({ data: `Person ${i}`, type: NounType.Person, metadata: { name: `P${i}` } }) + } + for (let i = 0; i < 3; i++) { + await brainy.add({ data: `Task ${i}`, type: NounType.Task, metadata: { title: `T${i}` } }) + } + await brainy.flush() + + // Capture the warm (in-session) counts before closing. + const warmPerson = brainy.counts.byTypeEnum('person') + const warmTask = brainy.counts.byTypeEnum('task') + const warmAll = Object.fromEntries(brainy.counts.allNounTypeCounts() as Map) + expect(warmPerson).toBe(7) + expect(warmTask).toBe(3) + + const reopened = new Brainy({ + requireSubtype: false, + storage: { type: 'filesystem', rootDirectory: testDir }, + dimensions: 384, + silent: true + }) + await reopened.init() + try { + // Cold counts equal the exact persisted values... + expect(reopened.counts.byTypeEnum('person')).toBe(7) + expect(reopened.counts.byTypeEnum('task')).toBe(3) + // ...and equal the warm counts map element-for-element. + const coldAll = Object.fromEntries(reopened.counts.allNounTypeCounts() as Map) + expect(coldAll).toEqual(warmAll) + } finally { + await reopened.close() + } + }) }) describe('Performance Characteristics', () => {