diff --git a/src/brainy.ts b/src/brainy.ts index e6828fe6..3a5c429e 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -784,7 +784,9 @@ export class Brainy implements BrainyInterface { return this.augmentationRegistry.execute('relate', params, async () => { // v4.0.0: Prepare verb metadata + // CRITICAL (v4.1.2): Include verb type in metadata for count tracking const verbMetadata = { + verb: params.type, // Store verb type for count synchronization weight: params.weight ?? 1.0, ...(params.metadata || {}), createdAt: Date.now() diff --git a/src/storage/adapters/azureBlobStorage.ts b/src/storage/adapters/azureBlobStorage.ts index 37c153ac..6c5a708b 100644 --- a/src/storage/adapters/azureBlobStorage.ts +++ b/src/storage/adapters/azureBlobStorage.ts @@ -1057,11 +1057,8 @@ export class AzureBlobStorage extends BaseStorage { // Update cache this.verbCacheManager.set(edge.id, edge) - // Increment verb count - const metadata = await this.getVerbMetadata(edge.id) - if (metadata && metadata.type) { - await this.incrementVerbCount(metadata.type as string) - } + // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2) + // This fixes the race condition where metadata didn't exist yet this.logger.trace(`Edge ${edge.id} saved successfully`) this.releaseBackpressure(true, requestId) diff --git a/src/storage/adapters/baseStorageAdapter.ts b/src/storage/adapters/baseStorageAdapter.ts index 261b70bf..6ae690d3 100644 --- a/src/storage/adapters/baseStorageAdapter.ts +++ b/src/storage/adapters/baseStorageAdapter.ts @@ -1015,47 +1015,63 @@ export abstract class BaseStorageAdapter implements StorageAdapter { } /** - * Increment verb count - O(1) operation with mutex protection + * Increment verb count - O(1) operation (v4.1.2: now synchronous) + * Protected by storage-specific mechanisms (mutex, distributed consensus, etc.) * @param type The verb type */ - protected async incrementVerbCount(type: string): Promise { + protected incrementVerbCount(type: string): void { + this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1) + this.totalVerbCount++ + // Update cache + this.countCache.set('verbs_count', { + count: this.totalVerbCount, + timestamp: Date.now() + }) + } + + /** + * Thread-safe increment for verb counts (v4.1.2) + * Uses mutex for single-node, distributed consensus for multi-node + * @param type The verb type + */ + protected async incrementVerbCountSafe(type: string): Promise { const mutex = getGlobalMutex() await mutex.runExclusive(`count-verb-${type}`, async () => { - this.verbCounts.set(type, (this.verbCounts.get(type) || 0) + 1) - this.totalVerbCount++ - // Update cache - this.countCache.set('verbs_count', { - count: this.totalVerbCount, - timestamp: Date.now() - }) - + this.incrementVerbCount(type) // Smart batching (v3.32.3+): Adapts to storage type await this.scheduleCountPersist() }) } /** - * Decrement verb count - O(1) operation with mutex protection + * Decrement verb count - O(1) operation (v4.1.2: now synchronous) * @param type The verb type */ - protected async decrementVerbCount(type: string): Promise { + protected decrementVerbCount(type: string): void { + const current = this.verbCounts.get(type) || 0 + if (current > 1) { + this.verbCounts.set(type, current - 1) + } else { + this.verbCounts.delete(type) + } + if (this.totalVerbCount > 0) { + this.totalVerbCount-- + } + // Update cache + this.countCache.set('verbs_count', { + count: this.totalVerbCount, + timestamp: Date.now() + }) + } + + /** + * Thread-safe decrement for verb counts (v4.1.2) + * @param type The verb type + */ + protected async decrementVerbCountSafe(type: string): Promise { const mutex = getGlobalMutex() await mutex.runExclusive(`count-verb-${type}`, async () => { - const current = this.verbCounts.get(type) || 0 - if (current > 1) { - this.verbCounts.set(type, current - 1) - } else { - this.verbCounts.delete(type) - } - if (this.totalVerbCount > 0) { - this.totalVerbCount-- - } - // Update cache - this.countCache.set('verbs_count', { - count: this.totalVerbCount, - timestamp: Date.now() - }) - + this.decrementVerbCount(type) // Smart batching (v3.32.3+): Adapts to storage type await this.scheduleCountPersist() }) diff --git a/src/storage/adapters/fileSystemStorage.ts b/src/storage/adapters/fileSystemStorage.ts index 2d02af6f..4c5a0e42 100644 --- a/src/storage/adapters/fileSystemStorage.ts +++ b/src/storage/adapters/fileSystemStorage.ts @@ -253,9 +253,6 @@ export class FileSystemStorage extends BaseStorage { protected async saveNode(node: HNSWNode): Promise { await this.ensureInitialized() - // Check if this is a new node to update counts - const isNew = !(await this.fileExists(this.getNodePath(node.id))) - // Convert connections Map to a serializable format // CRITICAL: Only save lightweight vector data (no metadata) // Metadata is saved separately via saveNounMetadata() (2-file system) @@ -276,18 +273,8 @@ export class FileSystemStorage extends BaseStorage { JSON.stringify(serializableNode, null, 2) ) - // Update counts for new nodes (v4.0.0: load metadata separately) - if (isNew) { - // v4.0.0: Get type from separate metadata storage - const metadata = await this.getNounMetadata(node.id) - const type = metadata?.noun || 'default' - this.incrementEntityCount(type) - - // Persist counts periodically (every 10 operations for efficiency) - if (this.totalNounCount % 10 === 0) { - await this.persistCounts() - } - } + // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2) + // This fixes the race condition where metadata didn't exist yet } /** @@ -460,9 +447,6 @@ export class FileSystemStorage extends BaseStorage { protected async saveEdge(edge: Edge): Promise { await this.ensureInitialized() - // Check if this is a new edge to update counts - const isNew = !(await this.fileExists(this.getVerbPath(edge.id))) - // Convert connections Map to a serializable format // ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file // These fields are essential for 90% of operations - no metadata lookup needed @@ -489,15 +473,8 @@ export class FileSystemStorage extends BaseStorage { JSON.stringify(serializableEdge, null, 2) ) - // Update verb count for new edges (production-scale optimizations) - if (isNew) { - this.totalVerbCount++ - - // Persist counts periodically (every 10 operations for efficiency) - if (this.totalVerbCount % 10 === 0) { - this.persistCounts() // Async persist, don't await - } - } + // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2) + // This fixes the race condition where metadata didn't exist yet } /** diff --git a/src/storage/adapters/gcsStorage.ts b/src/storage/adapters/gcsStorage.ts index c290d9ae..e8b59d15 100644 --- a/src/storage/adapters/gcsStorage.ts +++ b/src/storage/adapters/gcsStorage.ts @@ -488,11 +488,8 @@ export class GcsStorage extends BaseStorage { } // Note: Empty vectors are intentional during HNSW lazy mode - not logged - // Increment noun count - const metadata = await this.getNounMetadata(node.id) - if (metadata && metadata.type) { - await this.incrementEntityCountSafe(metadata.type as string) - } + // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2) + // This fixes the race condition where metadata didn't exist yet this.logger.trace(`Node ${node.id} saved successfully`) this.releaseBackpressure(true, requestId) @@ -858,11 +855,8 @@ export class GcsStorage extends BaseStorage { // Update cache this.verbCacheManager.set(edge.id, edge) - // Increment verb count - const metadata = await this.getVerbMetadata(edge.id) - if (metadata && metadata.type) { - await this.incrementVerbCount(metadata.type as string) - } + // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2) + // This fixes the race condition where metadata didn't exist yet this.logger.trace(`Edge ${edge.id} saved successfully`) this.releaseBackpressure(true, requestId) diff --git a/src/storage/adapters/memoryStorage.ts b/src/storage/adapters/memoryStorage.ts index eee18232..1d7ff179 100644 --- a/src/storage/adapters/memoryStorage.ts +++ b/src/storage/adapters/memoryStorage.ts @@ -79,7 +79,8 @@ export class MemoryStorage extends BaseStorage { // Save the noun directly in the nouns map this.nouns.set(noun.id, nounCopy) - // Note: Count tracking happens in saveNounMetadata since type info is in metadata now + // Count tracking happens in baseStorage.saveNounMetadata_internal (v4.1.2) + // This fixes the race condition where metadata didn't exist yet } /** diff --git a/src/storage/adapters/r2Storage.ts b/src/storage/adapters/r2Storage.ts index c2cd6fc7..8e6a94c3 100644 --- a/src/storage/adapters/r2Storage.ts +++ b/src/storage/adapters/r2Storage.ts @@ -771,10 +771,8 @@ export class R2Storage extends BaseStorage { this.verbCacheManager.set(edge.id, edge) - const metadata = await this.getVerbMetadata(edge.id) - if (metadata && metadata.type) { - await this.incrementVerbCount(metadata.type as string) - } + // Count tracking happens in baseStorage.saveVerbMetadata_internal (v4.1.2) + // This fixes the race condition where metadata didn't exist yet this.releaseBackpressure(true, requestId) } catch (error: any) { diff --git a/src/storage/baseStorage.ts b/src/storage/baseStorage.ts index 2dc876b6..16adef74 100644 --- a/src/storage/baseStorage.ts +++ b/src/storage/baseStorage.ts @@ -925,12 +925,35 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Internal method for saving noun metadata (v4.0.0: now typed) * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded) + * + * CRITICAL (v4.1.2): Count synchronization happens here + * This ensures counts are updated AFTER metadata exists, fixing the race condition + * where storage adapters tried to read metadata before it was saved. + * * @protected */ protected async saveNounMetadata_internal(id: string, metadata: NounMetadata): Promise { await this.ensureInitialized() + + // Determine if this is a new entity by checking if metadata already exists const keyInfo = this.analyzeKey(id, 'noun-metadata') - return this.writeObjectToPath(keyInfo.fullPath, metadata) + const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath) + const isNew = !existingMetadata + + // Save the metadata + await this.writeObjectToPath(keyInfo.fullPath, metadata) + + // CRITICAL FIX (v4.1.2): Increment count for new entities + // This runs AFTER metadata is saved, guaranteeing type information is available + // Uses synchronous increment since storage operations are already serialized + // Fixes Bug #1: Count synchronization failure during add() and import() + if (isNew && metadata.noun) { + this.incrementEntityCount(metadata.noun) + // Persist counts asynchronously (fire and forget) + this.scheduleCountPersist().catch(() => { + // Ignore persist errors - will retry on next operation + }) + } } /** @@ -965,12 +988,38 @@ export abstract class BaseStorage extends BaseStorageAdapter { /** * Internal method for saving verb metadata (v4.0.0: now typed) * Uses routing logic to handle both UUIDs (sharded) and system keys (unsharded) + * + * CRITICAL (v4.1.2): Count synchronization happens here + * This ensures verb counts are updated AFTER metadata exists, fixing the race condition + * where storage adapters tried to read metadata before it was saved. + * + * Note: Verb type is now stored in both HNSWVerb (vector file) and VerbMetadata for count tracking + * * @protected */ protected async saveVerbMetadata_internal(id: string, metadata: VerbMetadata): Promise { await this.ensureInitialized() + + // Determine if this is a new verb by checking if metadata already exists const keyInfo = this.analyzeKey(id, 'verb-metadata') - return this.writeObjectToPath(keyInfo.fullPath, metadata) + const existingMetadata = await this.readObjectFromPath(keyInfo.fullPath) + const isNew = !existingMetadata + + // Save the metadata + await this.writeObjectToPath(keyInfo.fullPath, metadata) + + // CRITICAL FIX (v4.1.2): Increment verb count for new relationships + // This runs AFTER metadata is saved + // Verb type is now stored in metadata (as of v4.1.2) to avoid loading HNSWVerb + // Uses synchronous increment since storage operations are already serialized + // Fixes Bug #2: Count synchronization failure during relate() and import() + if (isNew && (metadata as any).verb) { + this.incrementVerbCount((metadata as any).verb) + // Persist counts asynchronously (fire and forget) + this.scheduleCountPersist().catch(() => { + // Ignore persist errors - will retry on next operation + }) + } } /** diff --git a/src/utils/rebuildCounts.ts b/src/utils/rebuildCounts.ts new file mode 100644 index 00000000..277ca198 --- /dev/null +++ b/src/utils/rebuildCounts.ts @@ -0,0 +1,154 @@ +/** + * Rebuild Counts Utility + * + * Scans storage and rebuilds counts.json from actual data + * Use this to fix databases affected by the v4.1.1 count synchronization bug + * + * NO MOCKS - Production-ready implementation + */ + +import type { BaseStorage } from '../storage/baseStorage.js' + +export interface RebuildCountsResult { + /** Total number of entities (nouns) found */ + nounCount: number + + /** Total number of relationships (verbs) found */ + verbCount: number + + /** Entity counts by type */ + entityCounts: Map + + /** Verb counts by type */ + verbCounts: Map + + /** Processing time in milliseconds */ + duration: number +} + +/** + * Rebuild counts.json from actual storage data + * + * This scans all entities and relationships in storage and reconstructs + * the counts index from scratch. Use this to fix count desynchronization. + * + * @param storage - The storage adapter to rebuild counts for + * @returns Promise that resolves to rebuild statistics + * + * @example + * ```typescript + * const brain = new Brainy({ storage: { type: 'filesystem', path: './brainy-data' } }) + * await brain.init() + * + * const result = await rebuildCounts(brain.storage) + * console.log(`Rebuilt counts: ${result.nounCount} nouns, ${result.verbCount} verbs`) + * ``` + */ +export async function rebuildCounts(storage: BaseStorage): Promise { + const startTime = Date.now() + + console.log('🔧 Rebuilding counts from storage...') + + const entityCounts = new Map() + const verbCounts = new Map() + let totalNouns = 0 + let totalVerbs = 0 + + // Scan all nouns using pagination + console.log('📊 Scanning entities...') + + // Check if pagination method exists + const storageWithPagination = storage as any + if (typeof storageWithPagination.getNounsWithPagination !== 'function') { + throw new Error('Storage adapter does not support getNounsWithPagination') + } + + let hasMore = true + let cursor: string | undefined + + while (hasMore) { + const result: any = await storageWithPagination.getNounsWithPagination({ limit: 100, cursor }) + + for (const noun of result.items) { + const metadata = await storage.getNounMetadata(noun.id) + if (metadata?.noun) { + const entityType = metadata.noun + entityCounts.set(entityType, (entityCounts.get(entityType) || 0) + 1) + totalNouns++ + } + } + + hasMore = result.hasMore + cursor = result.nextCursor + } + + console.log(` Found ${totalNouns} entities across ${entityCounts.size} types`) + + // Scan all verbs using pagination + console.log('🔗 Scanning relationships...') + + if (typeof storageWithPagination.getVerbsWithPagination !== 'function') { + throw new Error('Storage adapter does not support getVerbsWithPagination') + } + + hasMore = true + cursor = undefined + + while (hasMore) { + const result: any = await storageWithPagination.getVerbsWithPagination({ limit: 100, cursor }) + + for (const verb of result.items) { + if (verb.verb) { + const verbType = verb.verb + verbCounts.set(verbType, (verbCounts.get(verbType) || 0) + 1) + totalVerbs++ + } + } + + hasMore = result.hasMore + cursor = result.nextCursor + } + + console.log(` Found ${totalVerbs} relationships across ${verbCounts.size} types`) + + // Update storage adapter's in-memory counts FIRST + storageWithPagination.totalNounCount = totalNouns + storageWithPagination.totalVerbCount = totalVerbs + storageWithPagination.entityCounts = entityCounts + storageWithPagination.verbCounts = verbCounts + + // Mark counts as pending persist (required for flushCounts to actually persist) + storageWithPagination.pendingCountPersist = true + storageWithPagination.pendingCountOperations = 1 + + // Persist counts using storage adapter's own persist method + // This ensures counts.json is written correctly (compressed or uncompressed) + await storageWithPagination.flushCounts() + + const duration = Date.now() - startTime + + console.log(`✅ Counts rebuilt successfully in ${duration}ms`) + console.log(` Entities: ${totalNouns}`) + console.log(` Relationships: ${totalVerbs}`) + console.log('') + console.log('Entity breakdown:') + entityCounts.forEach((count, entityType) => { + console.log(` ${entityType}: ${count}`) + }) + + if (verbCounts.size > 0) { + console.log('') + console.log('Relationship breakdown:') + verbCounts.forEach((count, verbType) => { + console.log(` ${verbType}: ${count}`) + }) + } + + return { + nounCount: totalNouns, + verbCount: totalVerbs, + entityCounts, + verbCounts, + duration + } +} diff --git a/tests/integration/count-synchronization.test.ts b/tests/integration/count-synchronization.test.ts new file mode 100644 index 00000000..e10e5199 --- /dev/null +++ b/tests/integration/count-synchronization.test.ts @@ -0,0 +1,334 @@ +/** + * Count Synchronization Integration Tests (v4.1.2) + * + * Tests for Bug #1 and Bug #2: Count synchronization failures during add() and relate() + * This validates that counts.json is properly updated when entities and relationships are created + * + * NO MOCKS - Real integration tests with actual storage + */ + +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Brainy } from '../../src/brainy.js' +import { NounType, VerbType } from '../../src/types/graphTypes.js' +import { rebuildCounts } from '../../src/utils/rebuildCounts.js' +import * as fs from 'fs/promises' +import * as path from 'path' + +describe('Count Synchronization (Bug Fix v4.1.2)', () => { + let brain: Brainy + const testPath = './test-brainy-counts-sync' + + beforeEach(async () => { + // Clean up test directory + try { + await fs.rm(testPath, { recursive: true, force: true }) + } catch { + // Ignore if doesn't exist + } + + // Create fresh Brainy instance + brain = new Brainy({ + storage: { type: 'filesystem', path: testPath } + }) + await brain.init() + }) + + afterEach(async () => { + // Clean up test directory + try { + await fs.rm(testPath, { recursive: true, force: true }) + } catch { + // Ignore cleanup errors + } + }) + + describe('Noun Count Synchronization', () => { + it('should update counts.json when adding a single entity', async () => { + // Add entity + const id = await brain.add({ + data: 'Test Entity', + type: NounType.Document + }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify counts + expect(counts.totalNounCount).toBe(1) + expect(counts.entityCounts[NounType.Document]).toBe(1) + }) + + it('should update counts.json when adding multiple entities of same type', async () => { + // Add 3 entities of same type + await brain.add({ data: 'Entity 1', type: NounType.Document }) + await brain.add({ data: 'Entity 2', type: NounType.Document }) + await brain.add({ data: 'Entity 3', type: NounType.Document }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify counts + expect(counts.totalNounCount).toBe(3) + expect(counts.entityCounts[NounType.Document]).toBe(3) + }) + + it('should update counts.json when adding multiple entities of different types', async () => { + // Add entities of different types + await brain.add({ data: 'Document 1', type: NounType.Document }) + await brain.add({ data: 'Person 1', type: NounType.Person }) + await brain.add({ data: 'Document 2', type: NounType.Document }) + await brain.add({ data: 'Organization 1', type: NounType.Organization }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify counts + expect(counts.totalNounCount).toBe(4) + expect(counts.entityCounts[NounType.Document]).toBe(2) + expect(counts.entityCounts[NounType.Person]).toBe(1) + expect(counts.entityCounts[NounType.Organization]).toBe(1) + }) + + it('should match find() results with counts.json', async () => { + // Add entities + await brain.add({ data: 'Entity 1', type: NounType.Document }) + await brain.add({ data: 'Entity 2', type: NounType.Person }) + await brain.add({ data: 'Entity 3', type: NounType.Document }) + + // Flush + await brain.flush() + + // Get all entities using find with high limit + const entities = await brain.find({ limit: 1000 }) + + // Read counts + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // CRITICAL: Counts must match actual data + expect(counts.totalNounCount).toBe(entities.length) + expect(counts.totalNounCount).toBe(3) + }) + }) + + describe('Verb Count Synchronization', () => { + it('should update counts.json when creating a single relationship', async () => { + // Create entities + const id1 = await brain.add({ data: 'Person 1', type: NounType.Person }) + const id2 = await brain.add({ data: 'Person 2', type: NounType.Person }) + + // Create relationship + await brain.relate({ + from: id1, + to: id2, + type: VerbType.FriendOf + }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify verb counts + expect(counts.totalVerbCount).toBe(1) + expect(counts.verbCounts[VerbType.FriendOf]).toBe(1) + }) + + it('should update counts.json when creating multiple relationships of same type', async () => { + // Create entities + const id1 = await brain.add({ data: 'Person 1', type: NounType.Person }) + const id2 = await brain.add({ data: 'Person 2', type: NounType.Person }) + const id3 = await brain.add({ data: 'Person 3', type: NounType.Person }) + + // Create relationships + await brain.relate({ from: id1, to: id2, type: VerbType.FriendOf }) + await brain.relate({ from: id2, to: id3, type: VerbType.FriendOf }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify verb counts + expect(counts.totalVerbCount).toBe(2) + expect(counts.verbCounts[VerbType.FriendOf]).toBe(2) + }) + + it('should update counts.json when creating multiple relationships of different types', async () => { + // Create entities + const personId = await brain.add({ data: 'John', type: NounType.Person }) + const docId = await brain.add({ data: 'Resume', type: NounType.Document }) + const orgId = await brain.add({ data: 'Company', type: NounType.Organization }) + + // Create relationships of different types + await brain.relate({ from: personId, to: orgId, type: VerbType.WorksWith }) + await brain.relate({ from: personId, to: docId, type: VerbType.CreatedBy }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify verb counts + expect(counts.totalVerbCount).toBe(2) + expect(counts.verbCounts[VerbType.WorksWith]).toBe(1) + expect(counts.verbCounts[VerbType.CreatedBy]).toBe(1) + }) + }) + + describe('Batch Operations', () => { + it('should update counts.json during addMany()', async () => { + // Add multiple entities via batch + await brain.addMany({ + items: [ + { data: 'Entity 1', type: NounType.Document }, + { data: 'Entity 2', type: NounType.Person }, + { data: 'Entity 3', type: NounType.Document } + ] + }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify counts + expect(counts.totalNounCount).toBe(3) + expect(counts.entityCounts[NounType.Document]).toBe(2) + expect(counts.entityCounts[NounType.Person]).toBe(1) + }) + + it('should update counts.json during relateMany()', async () => { + // Create entities + const id1 = await brain.add({ data: 'Person 1', type: NounType.Person }) + const id2 = await brain.add({ data: 'Person 2', type: NounType.Person }) + const id3 = await brain.add({ data: 'Person 3', type: NounType.Person }) + + // Create multiple relationships via batch + await brain.relateMany({ + items: [ + { from: id1, to: id2, type: VerbType.FriendOf }, + { from: id2, to: id3, type: VerbType.FriendOf } + ] + }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify verb counts + expect(counts.totalVerbCount).toBe(2) + expect(counts.verbCounts[VerbType.FriendOf]).toBe(2) + }) + }) + + describe('rebuildCounts Utility', () => { + it('should rebuild counts from corrupted state', async () => { + // Add entities + await brain.add({ data: 'Entity 1', type: NounType.Document }) + await brain.add({ data: 'Entity 2', type: NounType.Person }) + await brain.add({ data: 'Entity 3', type: NounType.Document }) + + // Flush to disk + await brain.flush() + + // Manually corrupt counts.json (simulate the bug) + const countsPath = path.join(testPath, '_system', 'counts.json') + const corruptedCounts = { + entityCounts: {}, + verbCounts: {}, + totalNounCount: 0, + totalVerbCount: 0, + lastUpdated: new Date().toISOString() + } + await fs.writeFile(countsPath, JSON.stringify(corruptedCounts, null, 2)) + + // Force storage to reload corrupted counts by updating in-memory counts + // This simulates the scenario where counts are out of sync + ;(brain.storage as any).totalNounCount = 0 + ;(brain.storage as any).totalVerbCount = 0 + ;(brain.storage as any).entityCounts = new Map() + ;(brain.storage as any).verbCounts = new Map() + + // Rebuild counts + const result = await rebuildCounts(brain.storage) + + // Verify rebuild results + expect(result.nounCount).toBe(3) + expect(result.entityCounts.get(NounType.Document)).toBe(2) + expect(result.entityCounts.get(NounType.Person)).toBe(1) + + // Verify counts.json was updated + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + expect(counts.totalNounCount).toBe(3) + expect(counts.entityCounts[NounType.Document]).toBe(2) + expect(counts.entityCounts[NounType.Person]).toBe(1) + }) + }) + + describe('Import Operations', () => { + it('should update counts.json during import', async () => { + // Create simple test data as JSON + const testData = { + entities: [ + { id: '1', name: 'Entity 1', type: 'document' }, + { id: '2', name: 'Entity 2', type: 'person' }, + { id: '3', name: 'Entity 3', type: 'document' } + ] + } + + // Import data + const result = await brain.import(testData, { + format: 'json', + createEntities: true, + createRelationships: false + }) + + // Flush to disk + await brain.flush() + + // Read counts.json + const countsPath = path.join(testPath, '_system', 'counts.json') + const countsRaw = await fs.readFile(countsPath, 'utf8') + const counts = JSON.parse(countsRaw) + + // Verify counts are updated (import may create additional VFS entities) + // So we check that totalNounCount >= graphNodesCreated + expect(counts.totalNounCount).toBeGreaterThanOrEqual(result.stats.graphNodesCreated) + expect(counts.totalNounCount).toBeGreaterThan(0) + }) + }) +})