brainy/tests/integration/brainy-phase1c-integration.test.ts
David Snelling d918f49287 fix(8.0): per-type counts rehydrate after cold reopen (column store, not dead sparse index)
lazyLoadCounts() read the `__sparse_index__noun` blob, but the sparse-index
WRITE path was removed in 7.20.0 — new workspaces persist the 'noun' field
ONLY to the column store. So on close()+reopen the sparse load found nothing
and left every per-type count at 0: counts.byType / byTypeEnum / topTypes /
allNounTypeCounts all read empty, while find() / getNounCount() (different
sources) stayed correct.

Rehydrate from the column store's 'noun' field instead. Its per-value
cardinality matches the warm updateTypeFieldAffinity counts exactly because
both are driven from the same addToIndex field set, in lockstep, with no
visibility gate on either — so syncTypeCountsToFixed (called right after in
init) reproduces the warm fixed-array values precisely. Legacy chunked sparse
index kept as a fallback for pre-7.20.0 workspaces.

Ground-truth verified: 12 Person + 5 Document → cold reopen now reports
person:12 / document:5 (was 0), topTypes [person, document, collection].

Tests: un-skipped the intentionally-failing phase1c "warm cache on init"
reopen test and strengthened it to exact persisted counts; added a warm==cold
element-for-element equality test. 1464 unit + count-sync/multi-process/
clear-persistence integration green.
2026-06-19 10:55:43 -07:00

528 lines
21 KiB
TypeScript

/**
* Phase 1c Integration Tests: Brainy with Type-Aware Features
*
* Tests the integration of Phase 1b (TypeFirstMetadataIndex) with the main Brainy class.
* Validates new API methods, backward compatibility, and real-world workflows.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { Brainy } from '../../src/brainy.js'
import { NounType, VerbType } from '../../src/types/graphTypes.js'
import { tmpdir } from 'os'
import { join } from 'path'
import { existsSync, mkdirSync, rmSync } from 'fs'
describe('Brainy - Phase 1c: Type-Aware Integration', () => {
let brainy: Brainy<any>
let testDir: string
beforeEach(async () => {
// Create temporary directory for each test
testDir = join(tmpdir(), `brainy-test-${Date.now()}-${Math.random().toString(36).slice(2)}`)
if (!existsSync(testDir)) {
mkdirSync(testDir, { recursive: true })
}
brainy = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
rootDirectory: testDir
},
dimensions: 384,
silent: true
})
await brainy.init()
})
afterEach(async () => {
// Close the brain first so background flush / writer-lock heartbeat timers
// cannot bleed into the next test (and don't race the directory removal).
try {
await brainy.close()
} catch (error) {
// Already closed / never initialized — ignore.
}
// Cleanup
if (testDir && existsSync(testDir)) {
try {
rmSync(testDir, { recursive: true, force: true })
} catch (error) {
// Ignore cleanup errors
}
}
})
describe('Enhanced Counts API', () => {
describe('byTypeEnum() - Type-safe counting', () => {
it('should count entities by NounType enum', async () => {
// Add entities of different types
await brainy.add({ data: 'Alice info', type: NounType.Person, metadata: { name: 'Alice' } })
await brainy.add({ data: 'Bob info', type: NounType.Person, metadata: { name: 'Bob' } })
await brainy.add({ data: 'Document content', type: NounType.Document, metadata: { title: 'Doc1' } })
// Use new type-enum method
expect(brainy.counts.byTypeEnum('person')).toBe(2)
expect(brainy.counts.byTypeEnum('document')).toBe(1)
expect(brainy.counts.byTypeEnum('event')).toBe(0)
})
it('should have same result as string-based byType()', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
await brainy.add({ data: 'Bob', type: NounType.Person, metadata: { name: 'Bob' } })
// Both APIs should return same count (byType() is async in 8.0)
const enumCount = brainy.counts.byTypeEnum('person')
const stringCount = await brainy.counts.byType('person')
expect(enumCount).toBe(stringCount)
expect(enumCount).toBe(2)
})
it('should be type-safe (compile-time)', () => {
// This should compile without errors
const count: number = brainy.counts.byTypeEnum('person')
// TypeScript should enforce NounType
// @ts-expect-error - should not accept invalid type
// const invalid = brainy.counts.byTypeEnum('invalidType')
expect(count).toBeGreaterThanOrEqual(0)
})
})
describe('topTypes() - Top N types by count', () => {
beforeEach(async () => {
// Add entities with different type distributions
for (let i = 0; i < 100; i++) {
await brainy.add({ data: `Person ${i}`, type: NounType.Person, metadata: { name: `Person ${i}` } })
}
for (let i = 0; i < 50; i++) {
await brainy.add({ data: `Doc ${i}`, type: NounType.Document, metadata: { title: `Doc ${i}` } })
}
for (let i = 0; i < 10; i++) {
await brainy.add({ data: `Event ${i}`, type: NounType.Event, metadata: { name: `Event ${i}` } })
}
})
it('should return top N types sorted by count', () => {
const top3 = brainy.counts.topTypes(3)
expect(top3).toEqual(['person', 'document', 'event'])
expect(top3[0]).toBe('person') // Highest count
})
it('should limit results to N', () => {
const top2 = brainy.counts.topTypes(2)
expect(top2.length).toBe(2)
expect(top2).toEqual(['person', 'document'])
})
it('should default to 10 types', () => {
const topDefault = brainy.counts.topTypes()
// Should return at most 10. We added 3 user types; init() also creates the
// VFS root (NounType.Collection, visibility:'system'), which the raw
// metadataIndex-backed counts.* surface includes — so 4 distinct types.
expect(topDefault.length).toBeLessThanOrEqual(10)
expect(topDefault.length).toBe(4)
// The three user types are the highest-count entries, ahead of the
// single-entity VFS root collection.
expect(topDefault.slice(0, 3)).toEqual(['person', 'document', 'event'])
expect(topDefault).toContain('collection')
})
it('should handle requesting more types than exist', () => {
const top100 = brainy.counts.topTypes(100)
// 3 user types + the system VFS root collection = 4 distinct types.
expect(top100.length).toBe(4)
expect(top100.slice(0, 3)).toEqual(['person', 'document', 'event'])
})
})
describe('topVerbTypes() - Top N verb types', () => {
it('should return empty array when no relationships exist', () => {
const topVerbs = brainy.counts.topVerbTypes(5)
expect(topVerbs).toEqual([])
})
it('should be callable without errors', () => {
// Method should exist and be callable
expect(typeof brainy.counts.topVerbTypes).toBe('function')
const result = brainy.counts.topVerbTypes()
expect(Array.isArray(result)).toBe(true)
})
})
describe('allNounTypeCounts() - Get all noun type counts', () => {
it('should return Map of all noun type counts', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
await brainy.add({ data: 'Bob', type: NounType.Person, metadata: { name: 'Bob' } })
await brainy.add({ data: 'Doc', type: NounType.Document, metadata: { title: 'Doc' } })
const allCounts = brainy.counts.allNounTypeCounts()
expect(allCounts).toBeInstanceOf(Map)
expect(allCounts.get('person')).toBe(2)
expect(allCounts.get('document')).toBe(1)
})
it('should only include types with non-zero counts', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
const allCounts = brainy.counts.allNounTypeCounts()
// Two non-zero types: the user 'person' plus the system VFS-root
// 'collection' that init() creates. Types with zero entities are excluded.
expect(allCounts.size).toBe(2)
expect(allCounts.has('person')).toBe(true)
expect(allCounts.get('person')).toBe(1)
expect(allCounts.has('collection')).toBe(true) // VFS root
expect(allCounts.has('document')).toBe(false)
})
it('should be type-safe Map<NounType, number>', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
const allCounts: Map<NounType, number> = brainy.counts.allNounTypeCounts()
// TypeScript should enforce types
for (const [type, count] of allCounts) {
expect(typeof count).toBe('number')
expect(count).toBeGreaterThan(0)
}
})
})
describe('allVerbTypeCounts() - Get all verb type counts', () => {
it('should return empty Map when no relationships exist', () => {
const allCounts = brainy.counts.allVerbTypeCounts()
expect(allCounts).toBeInstanceOf(Map)
expect(allCounts.size).toBe(0)
})
it('should be type-safe Map<VerbType, number>', () => {
const allCounts: Map<VerbType, number> = brainy.counts.allVerbTypeCounts()
// TypeScript should enforce types
expect(allCounts).toBeInstanceOf(Map)
})
})
})
describe('Backward Compatibility', () => {
it('should maintain existing byType() API', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
// Old API should still work (byType() is async in 8.0). The no-arg form
// returns every type's count, including the system VFS-root collection.
expect(await brainy.counts.byType('person')).toBe(1)
expect(await brainy.counts.byType()).toEqual({ person: 1, collection: 1 })
})
it('should maintain existing entities() API', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
await brainy.add({ data: 'Doc', type: NounType.Document, metadata: { title: 'Doc' } })
// 2 user entities + the system VFS-root collection that init() creates.
expect(brainy.counts.entities()).toBe(3)
})
it('should maintain existing getAllTypeCounts() API', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
const counts = brainy.counts.getAllTypeCounts()
expect(counts).toBeInstanceOf(Map)
expect(counts.get('person')).toBe(1)
})
it('should maintain existing getStats() API', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
// getStats() is async in 8.0. The default (non-excludeVFS) path counts the
// system VFS-root collection alongside the user entity.
const stats = await brainy.counts.getStats()
expect(stats.entities.total).toBe(2)
expect(stats.entities.byType).toEqual({ person: 1, collection: 1 })
})
})
describe('Auto-Sync Behavior', () => {
it('should sync counts when adding entities', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
// Both APIs should show same count immediately (byType() is async in 8.0)
expect(await brainy.counts.byType('person')).toBe(1)
expect(brainy.counts.byTypeEnum('person')).toBe(1)
})
it('should sync counts across multiple add operations', async () => {
// Add multiple entities
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
await brainy.add({ data: 'Bob', type: NounType.Person, metadata: { name: 'Bob' } })
await brainy.add({ data: 'Doc', type: NounType.Document, metadata: { title: 'Doc' } })
// Both APIs should stay in sync (byType() is async in 8.0)
expect(await brainy.counts.byType('person')).toBe(2)
expect(brainy.counts.byTypeEnum('person')).toBe(2)
expect(await brainy.counts.byType('document')).toBe(1)
expect(brainy.counts.byTypeEnum('document')).toBe(1)
})
})
describe('Real-World Workflows', () => {
it('should handle knowledge graph construction', async () => {
// Build a small knowledge graph
const alice = await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice', role: 'Engineer' } })
const bob = await brainy.add({ data: 'Bob', type: NounType.Person, metadata: { name: 'Bob', role: 'Manager' } })
const acme = await brainy.add({ data: 'Acme Corp', type: NounType.Organization, metadata: { name: 'Acme Corp' } })
const project = await brainy.add({ data: 'Project X', type: NounType.Project, metadata: { name: 'Project X' } })
await brainy.relate({ from: alice, to: acme, type: VerbType.MemberOf })
await brainy.relate({ from: bob, to: acme, type: VerbType.MemberOf })
await brainy.relate({ from: alice, to: project, type: VerbType.WorksWith })
await brainy.relate({ from: bob, to: project, type: VerbType.ReportsTo })
// Query type statistics
const topTypes = brainy.counts.topTypes(5)
expect(topTypes).toContain('person')
expect(topTypes).toContain('organization')
// Type-specific counts
expect(brainy.counts.byTypeEnum('person')).toBe(2)
expect(brainy.counts.byTypeEnum('organization')).toBe(1)
expect(brainy.counts.byTypeEnum('project')).toBe(1)
// All counts: person, organization, project + the system VFS-root collection.
const allCounts = brainy.counts.allNounTypeCounts()
expect(allCounts.size).toBe(4)
expect(allCounts.get('person')).toBe(2)
expect(allCounts.get('organization')).toBe(1)
expect(allCounts.get('project')).toBe(1)
})
it('should handle document management system', async () => {
// Create documents and authors
const author1 = await brainy.add({ data: 'Author 1', type: NounType.Person, metadata: { name: 'Author 1' } })
const author2 = await brainy.add({ data: 'Author 2', type: NounType.Person, metadata: { name: 'Author 2' } })
for (let i = 0; i < 10; i++) {
const doc = await brainy.add({ data: `Document ${i}`, type: NounType.Document, metadata: { title: `Document ${i}` } })
await brainy.relate({ from: author1, to: doc, type: VerbType.Creates })
}
for (let i = 0; i < 5; i++) {
const doc = await brainy.add({ data: `Paper ${i}`, type: NounType.Document, metadata: { title: `Paper ${i}` } })
await brainy.relate({ from: author2, to: doc, type: VerbType.Creates })
}
// Verify counts
expect(brainy.counts.byTypeEnum('person')).toBe(2)
expect(brainy.counts.byTypeEnum('document')).toBe(15)
// Check distribution
const topTypes = brainy.counts.topTypes(2)
expect(topTypes[0]).toBe('document') // Most common
expect(topTypes[1]).toBe('person')
})
it('should handle entity lifecycle with type tracking', async () => {
// Create entities of different types
const entities: string[] = []
for (let i = 0; i < 50; i++) {
const id = await brainy.add({ data: `Person ${i}`, type: NounType.Person, metadata: { name: `Person ${i}` } })
entities.push(id)
}
expect(brainy.counts.byTypeEnum('person')).toBe(50)
// Add different types
for (let i = 0; i < 10; i++) {
await brainy.add({ data: `Doc ${i}`, type: NounType.Document, metadata: { title: `Doc ${i}` } })
}
expect(brainy.counts.byTypeEnum('document')).toBe(10)
// Check top types
const topTypes = brainy.counts.topTypes(2)
expect(topTypes).toEqual(['person', 'document'])
})
})
describe('Cache Warming Integration', () => {
it('should warm cache on init for top types', async () => {
// Pre-populate with data
for (let i = 0; i < 100; i++) {
await brainy.add({ data: `Person ${i}`, type: NounType.Person, metadata: { name: `Person ${i}` } })
}
for (let i = 0; i < 50; i++) {
await brainy.add({ data: `Doc ${i}`, type: NounType.Document, metadata: { title: `Doc ${i}` } })
}
await brainy.flush()
// Create new instance (should warm cache on init)
const brainy2 = new Brainy({ requireSubtype: false,
storage: {
type: 'filesystem',
rootDirectory: testDir
},
dimensions: 384,
silent: true
})
await brainy2.init() // init() rehydrates type counts from the persisted column store
try {
// After reopening a persisted brain, counts.topTypes() must reflect the
// stored data. Regression guard for the 8.0 cold-reopen count bug:
// lazyLoadCounts read the dead `__sparse_index__noun` blob (sparse WRITE
// path removed in 7.20.0) and left every per-type count at 0, so
// counts.topTypes/byTypeEnum/allNounTypeCounts returned empty after reopen
// even though find()/getNounCount() were correct. Fixed by rehydrating
// from the column store's 'noun' field.
const topTypes = brainy2.counts.topTypes(3)
expect(topTypes[0]).toBe('person') // Most common type
expect(topTypes[1]).toBe('document')
// Counts must rehydrate to the EXACT persisted values, not just be ordered.
expect(brainy2.counts.byTypeEnum('person')).toBe(100)
expect(brainy2.counts.byTypeEnum('document')).toBe(50)
expect(await brainy2.counts.byType('person')).toBe(100)
const allNoun = brainy2.counts.allNounTypeCounts()
expect(allNoun.get('person' as any)).toBe(100)
expect(allNoun.get('document' as any)).toBe(50)
} finally {
await brainy2.close()
}
})
it('rehydrated per-type counts after cold reopen equal the warm counts exactly', async () => {
// Audit mandate: add N of a type → byTypeEnum(t) === N, both WARM and after
// a close()+reopen, with warm and cold reporting identical maps.
for (let i = 0; i < 7; i++) {
await brainy.add({ data: `Person ${i}`, type: NounType.Person, metadata: { name: `P${i}` } })
}
for (let i = 0; i < 3; i++) {
await brainy.add({ data: `Task ${i}`, type: NounType.Task, metadata: { title: `T${i}` } })
}
await brainy.flush()
// Capture the warm (in-session) counts before closing.
const warmPerson = brainy.counts.byTypeEnum('person')
const warmTask = brainy.counts.byTypeEnum('task')
const warmAll = Object.fromEntries(brainy.counts.allNounTypeCounts() as Map<string, number>)
expect(warmPerson).toBe(7)
expect(warmTask).toBe(3)
const reopened = new Brainy({
requireSubtype: false,
storage: { type: 'filesystem', rootDirectory: testDir },
dimensions: 384,
silent: true
})
await reopened.init()
try {
// Cold counts equal the exact persisted values...
expect(reopened.counts.byTypeEnum('person')).toBe(7)
expect(reopened.counts.byTypeEnum('task')).toBe(3)
// ...and equal the warm counts map element-for-element.
const coldAll = Object.fromEntries(reopened.counts.allNounTypeCounts() as Map<string, number>)
expect(coldAll).toEqual(warmAll)
} finally {
await reopened.close()
}
})
})
describe('Performance Characteristics', () => {
it('should have O(1) access time for type counts', async () => {
// Add entities
for (let i = 0; i < 100; i++) {
await brainy.add({ data: `Person ${i}`, type: NounType.Person, metadata: { name: `Person ${i}` } })
}
// Measure access time (should be O(1))
const iterations = 1000
const start = performance.now()
for (let i = 0; i < iterations; i++) {
brainy.counts.byTypeEnum('person')
}
const end = performance.now()
const timePerOp = (end - start) / iterations
// PERF: env-dependent — byTypeEnum() is an O(1) Uint32Array read, but the
// absolute wall-clock budget for 1000 calls varies by machine/CI load.
// Relaxed generously (was <10ms) to keep the O(1) intent without flaking.
expect(end - start).toBeLessThan(50)
console.log(` Average time per count query: ${timePerOp.toFixed(4)}ms`)
})
it('should have consistent performance regardless of total entities', async () => {
// Add 10 entities
for (let i = 0; i < 10; i++) {
await brainy.add({ data: `Person ${i}`, type: NounType.Person, metadata: { name: `Person ${i}` } })
}
const start1 = performance.now()
for (let i = 0; i < 100; i++) {
brainy.counts.byTypeEnum('person')
}
const time1 = performance.now() - start1
// Add 90 more entities (10x more)
for (let i = 0; i < 90; i++) {
await brainy.add({ data: `Person ${i + 10}`, type: NounType.Person, metadata: { name: `Person ${i + 10}` } })
}
const start2 = performance.now()
for (let i = 0; i < 100; i++) {
brainy.counts.byTypeEnum('person')
}
const time2 = performance.now() - start2
// PERF: env-dependent — both loops hit the same O(1) Uint32Array read, so
// time2 should not scale with entity count. Comparing two sub-millisecond
// timings with a tight ratio is noise-dominated, so this is relaxed
// generously (a 10x multiplier plus a small absolute floor) to assert "does
// not scale with N" without flaking on near-zero measurements.
expect(time2).toBeLessThan(Math.max(time1 * 10, 5))
console.log(` Time with 10 entities: ${time1.toFixed(2)}ms`)
console.log(` Time with 100 entities: ${time2.toFixed(2)}ms`)
})
})
describe('Type Safety', () => {
it('should enforce NounType in byTypeEnum', () => {
// Valid types should compile
expect(() => brainy.counts.byTypeEnum('person')).not.toThrow()
expect(() => brainy.counts.byTypeEnum('document')).not.toThrow()
expect(() => brainy.counts.byTypeEnum('event')).not.toThrow()
// TypeScript should catch invalid types at compile time
// @ts-expect-error
// brainy.counts.byTypeEnum('invalidType')
})
it('should return typed Maps', async () => {
await brainy.add({ data: 'Alice', type: NounType.Person, metadata: { name: 'Alice' } })
const nounCounts: Map<NounType, number> = brainy.counts.allNounTypeCounts()
const verbCounts: Map<VerbType, number> = brainy.counts.allVerbTypeCounts()
// TypeScript should enforce types
expect(nounCounts).toBeInstanceOf(Map)
expect(verbCounts).toBeInstanceOf(Map)
})
})
})