diff --git a/src/augmentations/metricsAugmentation.ts b/src/augmentations/metricsAugmentation.ts index e2c718ec..7c1569df 100644 --- a/src/augmentations/metricsAugmentation.ts +++ b/src/augmentations/metricsAugmentation.ts @@ -1,11 +1,19 @@ /** * Metrics Augmentation - Optional Performance & Usage Metrics - * - * Replaces the hardcoded StatisticsCollector in Brainy with an optional augmentation. - * Tracks performance metrics, usage patterns, and system statistics. - * + * + * IMPORTANT: This is SEPARATE from core counting (brain.counts.*) which is always enabled. + * + * Core counting provides O(1) entity/relationship counts for scalability. + * This augmentation provides performance analytics and observability. + * + * Features: + * - Search performance tracking (latency, throughput) + * - Cache hit/miss rates + * - Operation timing analysis + * - Usage pattern insights + * * Zero-config: Automatically enabled for observability - * Can be disabled or customized via augmentation registry + * Can be disabled via augmentation registry without affecting core performance */ import { BaseAugmentation, AugmentationContext } from './brainyAugmentation.js' @@ -267,12 +275,18 @@ export class MetricsAugmentation extends BaseAugmentation { } /** - * Get current metrics + * Get current metrics (performance analytics only) + * + * NOTE: For production counting (entities, relationships), use: + * - brain.counts.entities() - O(1) total count + * - brain.counts.byType() - O(1) type-specific counts + * - brain.counts.relationships() - O(1) relationship counts */ getStatistics() { if (!this.statisticsCollector) { return { enabled: false, + note: 'For core counting, use brain.counts.* APIs which are always available', totalSearches: 0, totalUpdates: 0, contentTypes: {}, @@ -287,6 +301,7 @@ export class MetricsAugmentation extends BaseAugmentation { return { enabled: true, + note: 'Performance analytics only. Core counting available via brain.counts.*', ...this.statisticsCollector.getStatistics() } } diff --git a/src/brainy.ts b/src/brainy.ts index 88938bb5..3f86657f 100644 --- a/src/brainy.ts +++ b/src/brainy.ts @@ -22,6 +22,7 @@ import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js' import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js' import { MetadataIndexManager } from './utils/metadataIndex.js' import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js' +import { createPipeline } from './streaming/pipeline.js' import { configureLogger, LogLevel } from './utils/logger.js' import { Entity, @@ -597,21 +598,56 @@ export class Brainy { // CRITICAL FIX: Handle both cases properly if (results.length > 0) { - // Filter existing results (from vector search) + // OPTIMIZED: Filter existing results (from vector search) efficiently const filteredIdSet = new Set(filteredIds) results = results.filter((r) => filteredIdSet.has(r.id)) + + // Apply early pagination for vector + metadata queries + const limit = params.limit || 10 + const offset = params.offset || 0 + + // If we have enough filtered results, sort and paginate early + if (results.length >= offset + limit) { + results.sort((a, b) => b.score - a.score) + results = results.slice(offset, offset + limit) + + // Load entities only for the paginated results + for (const result of results) { + if (!result.entity) { + const entity = await this.get(result.id) + if (entity) { + result.entity = entity + } + } + } + + // Early return if no other processing needed + if (!params.connected && !params.fusion) { + return results + } + } } else { - // Create results from metadata matches (metadata-only query) - for (const id of filteredIds) { + // OPTIMIZED: Apply pagination to filtered IDs BEFORE loading entities + const limit = params.limit || 10 + const offset = params.offset || 0 + const pageIds = filteredIds.slice(offset, offset + limit) + + // Load only entities for current page - O(page_size) instead of O(total_results) + for (const id of pageIds) { const entity = await this.get(id) if (entity) { results.push({ id, score: 1.0, // All metadata matches are equally relevant - entity + entity: entity as Entity }) } } + + // Early return for metadata-only queries with pagination applied + if (!params.query && !params.connected) { + return results + } } } @@ -625,11 +661,12 @@ export class Brainy { results = this.applyFusionScoring(results, params.fusion) } - // Sort by score and apply pagination + // OPTIMIZED: Sort first, then apply efficient pagination results.sort((a, b) => b.score - a.score) const limit = params.limit || 10 const offset = params.offset || 0 + // Efficient pagination - only slice what we need return results.slice(offset, offset + limit) }) @@ -1098,27 +1135,19 @@ export class Brainy { }> { await this.ensureInitialized() - // Get all entities count - use getNouns with high limit - const entitiesResult = await this.storage.getNouns({ - pagination: { limit: 10000 } - }) - const entities = entitiesResult.totalCount || entitiesResult.items.length + // O(1) entity counting using existing MetadataIndexManager + const entities = this.metadataIndex.getTotalEntityCount() + + // O(1) count by type using existing index tracking + const typeCountsMap = this.metadataIndex.getAllEntityCounts() + const types: Record = Object.fromEntries(typeCountsMap) + + // O(1) relationships count using GraphAdjacencyIndex + const relationships = this.graphIndex.getTotalRelationshipCount() - // Get relationships count - use getVerbs with high limit - const verbsResult = await this.storage.getVerbs({ - pagination: { limit: 10000 } - }) - const relationships = verbsResult.totalCount || verbsResult.items.length - - // Count by type - const types: Record = {} - for (const entity of entitiesResult.items) { - const type = (entity.metadata?.noun as string) || 'unknown' - types[type] = (types[type] || 0) + 1 - } - - // Get unique services - const services = [...new Set(entitiesResult.items.map((e: any) => e.metadata?.service).filter(Boolean))] as string[] + // Get unique services - O(log n) using index + const serviceValues = await this.metadataIndex.getFilterValues('service') + const services = serviceValues.filter(Boolean) // Calculate density (relationships per entity) const density = entities > 0 ? relationships / entities : 0 @@ -1132,6 +1161,264 @@ export class Brainy { } } + /** + * Efficient Pagination API - Production-scale pagination using index-first approach + * Automatically optimizes based on query type and applies pagination at the index level + */ + get pagination() { + return { + // Get paginated results with automatic optimization + find: async (params: FindParams & { page?: number, pageSize?: number }) => { + const page = params.page || 1 + const pageSize = params.pageSize || 10 + const offset = (page - 1) * pageSize + + return this.find({ + ...params, + limit: pageSize, + offset + }) + }, + + // Get total count for pagination UI (O(1) when possible) + count: async (params: Omit, 'limit' | 'offset'>) => { + // For simple type queries, use O(1) index counting + if (params.type && !params.query && !params.where && !params.connected) { + const types = Array.isArray(params.type) ? params.type : [params.type] + return types.reduce((sum, type) => sum + this.metadataIndex.getEntityCountByType(type), 0) + } + + // For complex queries, use metadata index for efficient counting + if (params.where || params.service) { + let filter: any = {} + if (params.where) Object.assign(filter, params.where) + if (params.service) filter.service = params.service + if (params.type) { + const types = Array.isArray(params.type) ? params.type : [params.type] + if (types.length === 1) { + filter.noun = types[0] + } else { + const baseFilter = { ...filter } + filter = { + anyOf: types.map(type => ({ noun: type, ...baseFilter })) + } + } + } + + const filteredIds = await this.metadataIndex.getIdsForFilter(filter) + return filteredIds.length + } + + // Fallback: total entity count + return this.metadataIndex.getTotalEntityCount() + }, + + // Get pagination metadata + meta: async (params: FindParams & { page?: number, pageSize?: number }) => { + const page = params.page || 1 + const pageSize = params.pageSize || 10 + const totalCount = await this.pagination.count(params) + const totalPages = Math.ceil(totalCount / pageSize) + + return { + page, + pageSize, + totalCount, + totalPages, + hasNext: page < totalPages, + hasPrev: page > 1 + } + } + } + } + + /** + * Streaming API - Process millions of entities with constant memory using existing Pipeline + * Integrates with index-based optimizations for maximum efficiency + */ + get streaming(): { + entities: (filter?: Partial>) => AsyncGenerator> + search: (params: FindParams, batchSize?: number) => AsyncGenerator<{ id: string; score: number; entity: Entity }> + relationships: (filter?: { type?: string; sourceId?: string; targetId?: string }) => AsyncGenerator + pipeline: (source: AsyncIterable) => any + process: (processor: (entity: Entity) => Promise>, filter?: Partial>, options?: { batchSize: number; parallel: number }) => Promise + } { + return { + // Stream all entities with optional filtering + entities: async function* (this: Brainy, filter?: Partial>) { + if (filter?.type || filter?.where || filter?.service) { + // Use MetadataIndexManager for efficient filtered streaming + let filterObj: any = {} + if (filter.where) Object.assign(filterObj, filter.where) + if (filter.service) filterObj.service = filter.service + if (filter.type) { + const types = Array.isArray(filter.type) ? filter.type : [filter.type] + if (types.length === 1) { + filterObj.noun = types[0] + } else { + const baseFilterObj = { ...filterObj } + filterObj = { + anyOf: types.map(type => ({ noun: type, ...baseFilterObj })) + } + } + } + + const filteredIds = await this.metadataIndex.getIdsForFilter(filterObj) + + // Stream filtered entities in batches for memory efficiency + const batchSize = 100 + for (let i = 0; i < filteredIds.length; i += batchSize) { + const batchIds = filteredIds.slice(i, i + batchSize) + for (const id of batchIds) { + const entity = await this.get(id) + if (entity) yield entity as Entity + } + } + } else { + // Stream all entities using storage adapter pagination + let offset = 0 + const batchSize = 100 + let hasMore = true + + while (hasMore) { + const result = await this.storage.getNouns({ + pagination: { offset, limit: batchSize } + }) + + for (const noun of result.items) { + // Convert HNSWNoun to Entity + yield noun as unknown as Entity + } + + hasMore = result.hasMore + offset += batchSize + } + } + }.bind(this), + + // Stream search results efficiently + search: async function* (this: Brainy, params: FindParams, batchSize = 50) { + const originalLimit = params.limit + let offset = 0 + let hasMore = true + + while (hasMore) { + const batchResults = await this.find({ + ...params, + limit: batchSize, + offset + }) + + for (const result of batchResults) { + yield result + } + + hasMore = batchResults.length === batchSize + offset += batchSize + + // Respect original limit if specified + if (originalLimit && offset >= originalLimit) { + break + } + } + }.bind(this), + + // Stream relationships efficiently + relationships: async function* (this: Brainy, filter?: { type?: string, sourceId?: string, targetId?: string }) { + let offset = 0 + const batchSize = 100 + let hasMore = true + + while (hasMore) { + const result = await this.storage.getVerbs({ + pagination: { offset, limit: batchSize }, + filter + }) + + for (const verb of result.items) { + yield verb + } + + hasMore = result.hasMore + offset += batchSize + } + }.bind(this), + + // Create processing pipeline from stream + pipeline: (source: AsyncIterable) => { + return createPipeline(this).source(source) + }, + + // Batch process entities with Pipeline system + process: async function (this: Brainy, + processor: (entity: Entity) => Promise>, + filter?: Partial>, + options = { batchSize: 50, parallel: 4 } + ) { + return createPipeline(this) + .source(this.streaming.entities(filter)) + .batch(options.batchSize) + .parallelSink(async (batch: Entity[]) => { + await Promise.all(batch.map(processor)) + }, options.parallel) + .run() + }.bind(this) + } + } + + /** + * O(1) Count API - Production-scale counting using existing indexes + * Works across all storage adapters (FileSystem, OPFS, S3, Memory) + */ + get counts() { + return { + // O(1) total entity count + entities: () => this.metadataIndex.getTotalEntityCount(), + + // O(1) total relationship count + relationships: () => this.graphIndex.getTotalRelationshipCount(), + + // O(1) count by type + byType: (type?: string) => { + if (type) { + return this.metadataIndex.getEntityCountByType(type) + } + return Object.fromEntries(this.metadataIndex.getAllEntityCounts()) + }, + + // O(1) count by relationship type + byRelationshipType: (type?: string) => { + if (type) { + return this.graphIndex.getRelationshipCountByType(type) + } + return Object.fromEntries(this.graphIndex.getAllRelationshipCounts()) + }, + + // O(1) count by field-value criteria + byCriteria: async (field: string, value: any) => { + return this.metadataIndex.getCountForCriteria(field, value) + }, + + // Get all type counts as Map for performance-critical operations + getAllTypeCounts: () => this.metadataIndex.getAllEntityCounts(), + + // Get complete statistics + getStats: () => { + const entityStats = { + total: this.metadataIndex.getTotalEntityCount(), + byType: Object.fromEntries(this.metadataIndex.getAllEntityCounts()) + } + const relationshipStats = this.graphIndex.getRelationshipStats() + + return { + entities: entityStats, + relationships: relationshipStats, + density: entityStats.total > 0 ? relationshipStats.totalRelationships / entityStats.total : 0 + } + } + } + } + /** * Augmentations API - Clean and simple */ diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index 7b362b43..a43f1b0f 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -54,6 +54,9 @@ export class GraphAdjacencyIndex { private rebuildStartTime = 0 private totalRelationshipsIndexed = 0 + // Production-scale relationship counting by type + private relationshipCountsByType = new Map() + constructor(storage: StorageAdapter, config: GraphIndexConfig = {}) { this.storage = storage this.config = { @@ -107,12 +110,63 @@ export class GraphAdjacencyIndex { } /** - * Get relationship count + * Get total relationship count - O(1) operation */ size(): number { return this.verbIndex.size } + /** + * Get relationship count by type - O(1) operation using existing tracking + */ + getRelationshipCountByType(type: string): number { + return this.relationshipCountsByType.get(type) || 0 + } + + /** + * Get total relationship count - O(1) operation + */ + getTotalRelationshipCount(): number { + return this.verbIndex.size + } + + /** + * Get all relationship types and their counts - O(1) operation + */ + getAllRelationshipCounts(): Map { + return new Map(this.relationshipCountsByType) + } + + /** + * Get relationship statistics with enhanced counting information + */ + getRelationshipStats(): { + totalRelationships: number + relationshipsByType: Record + uniqueSourceNodes: number + uniqueTargetNodes: number + totalNodes: number + } { + const totalRelationships = this.verbIndex.size + const relationshipsByType = Object.fromEntries(this.relationshipCountsByType) + const uniqueSourceNodes = this.sourceIndex.size + const uniqueTargetNodes = this.targetIndex.size + + // Calculate total unique nodes (source ∪ target) + const allNodes = new Set() + this.sourceIndex.keys().forEach(id => allNodes.add(id)) + this.targetIndex.keys().forEach(id => allNodes.add(id)) + const totalNodes = allNodes.size + + return { + totalRelationships, + relationshipsByType, + uniqueSourceNodes, + uniqueTargetNodes, + totalNodes + } + } + /** * Add relationship to index - O(1) amortized */ @@ -142,6 +196,13 @@ export class GraphAdjacencyIndex { await this.cacheIndexEntry(verb.sourceId, 'source') await this.cacheIndexEntry(verb.targetId, 'target') + // Update type-specific counts atomically + const verbType = verb.type || 'unknown' + this.relationshipCountsByType.set( + verbType, + (this.relationshipCountsByType.get(verbType) || 0) + 1 + ) + const elapsed = performance.now() - startTime this.totalRelationshipsIndexed++ @@ -163,6 +224,15 @@ export class GraphAdjacencyIndex { // Remove from verb cache this.verbIndex.delete(verbId) + // Update type-specific counts atomically + const verbType = verb.type || 'unknown' + const currentCount = this.relationshipCountsByType.get(verbType) || 0 + if (currentCount > 1) { + this.relationshipCountsByType.set(verbType, currentCount - 1) + } else { + this.relationshipCountsByType.delete(verbType) + } + // Remove from source index const sourceNeighbors = this.sourceIndex.get(verb.sourceId) if (sourceNeighbors) { diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 969b8558..ffd0dd6d 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1461,19 +1461,63 @@ export class MetadataIndexManager { } /** - * Get index statistics + * Get count of entities by type - O(1) operation using existing tracking + * This exposes the production-ready counting that's already maintained + */ + getEntityCountByType(type: string): number { + return this.totalEntitiesByType.get(type) || 0 + } + + /** + * Get total count of all entities - O(1) operation + */ + getTotalEntityCount(): number { + let total = 0 + for (const count of this.totalEntitiesByType.values()) { + total += count + } + return total + } + + /** + * Get all entity types and their counts - O(1) operation + */ + getAllEntityCounts(): Map { + return new Map(this.totalEntitiesByType) + } + + /** + * Get count of entities matching field-value criteria - O(1) lookup from existing indexes + */ + async getCountForCriteria(field: string, value: any): Promise { + const key = this.getIndexKey(field, value) + let entry = this.indexCache.get(key) + + if (!entry) { + const loadedEntry = await this.loadIndexEntry(key) + if (loadedEntry) { + entry = loadedEntry + this.indexCache.set(key, entry) + } + } + + return entry ? entry.ids.size : 0 + } + + /** + * Get index statistics with enhanced counting information */ async getStats(): Promise { const fields = new Set() let totalEntries = 0 let totalIds = 0 - + for (const entry of this.indexCache.values()) { fields.add(entry.field) totalEntries++ totalIds += entry.ids.size } - + return { totalEntries, totalIds,