diff --git a/docs/README.md b/docs/README.md index 2ec32813..bca25416 100644 --- a/docs/README.md +++ b/docs/README.md @@ -104,6 +104,11 @@ docs/ │ ├── noun-verb-taxonomy.md # Data model │ ├── triple-intelligence.md # Query system │ └── storage.md # Storage layer +├── vfs/ # Virtual Filesystem +│ ├── README.md # VFS overview +│ ├── SEMANTIC_VFS.md # Semantic projections +│ ├── VFS_API_GUIDE.md # Complete API reference +│ └── QUICK_START.md # 5-minute setup └── api/ # API documentation ├── README.md # API overview ├── brainy-data.md # Main class diff --git a/src/neural/improvedNeuralAPI.ts b/src/neural/improvedNeuralAPI.ts index 205c0dd5..b7bd951c 100644 --- a/src/neural/improvedNeuralAPI.ts +++ b/src/neural/improvedNeuralAPI.ts @@ -2414,8 +2414,70 @@ export class ImprovedNeuralAPI { } private async _getItemsByField(field: string): Promise { - // Implementation would query items by metadata field - return [] + try { + // Query all items from brain (limit to reasonable number for clustering) + const result = await this.brain.find({ + query: '', + limit: 10000 // Max items for clustering + }) + + if (!result || !Array.isArray(result)) { + return [] + } + + // Filter items that have the specified field (check both root level and metadata) + const itemsWithField = result.filter((item: any) => { + if (!item || !item.entity) return false + + const entity = item.entity + + // Check root level fields first (e.g., 'noun' for type) + if (field === 'type' || field === 'nounType') { + return entity.noun != null + } + + // Check if field exists at root level + if (entity[field] != null) { + return true + } + + // Check if field exists in metadata/data + if (entity.metadata?.[field] != null) { + return true + } + + if (entity.data?.[field] != null) { + return true + } + + return false + }) + + // Map to format expected by clustering methods + return itemsWithField.map((item: any) => { + const entity = item.entity + return { + id: entity.id, + vector: entity.embedding || entity.vector || [], + metadata: { + ...(entity.metadata || {}), + ...(entity.data || {}), + // Include root-level fields in metadata for easy access + noun: entity.noun, + type: entity.noun, + createdAt: entity.createdAt, + updatedAt: entity.updatedAt, + label: entity.label + }, + nounType: entity.noun, + label: entity.label || entity.data || '', + data: entity.data + } + }) + } catch (error) { + console.error('Error in _getItemsByField:', error) + return [] + } } // ===== TRIPLE INTELLIGENCE INTEGRATION ===== @@ -2875,11 +2937,39 @@ export class ImprovedNeuralAPI { private _groupByDomain(items: any[], field: string): Map { const groups = new Map() for (const item of items) { - const domain = item.metadata?.[field] || 'unknown' - if (!groups.has(domain)) { - groups.set(domain, []) + // Check multiple locations for the field value + let domain: any = 'unknown' + + // Special handling for type/nounType field + if (field === 'type' || field === 'nounType') { + domain = item.nounType || item.metadata?.noun || item.metadata?.type || 'unknown' + } else { + // Check root level first + domain = item[field] + + // Then check metadata + if (domain == null) { + domain = item.metadata?.[field] + } + + // Then check data + if (domain == null) { + domain = item.data?.[field] + } + + // Fallback to unknown + if (domain == null) { + domain = 'unknown' + } } - groups.get(domain).push(item) + + // Convert domain to string for Map key + const domainKey = String(domain) + + if (!groups.has(domainKey)) { + groups.set(domainKey, []) + } + groups.get(domainKey).push(item) } return groups } @@ -2902,18 +2992,203 @@ export class ImprovedNeuralAPI { } private async _findCrossDomainMembers(cluster: SemanticCluster, threshold: number): Promise { - // Find members that might belong to multiple domains - return [] + try { + // Find cluster members that have high similarity to items in other domains + const crossDomainMembers: string[] = [] + + for (const memberId of cluster.members) { + try { + // Get neighbors for this member + const neighbors = await this.neighbors(memberId, { + limit: 10, + minSimilarity: threshold + }) + + if (Array.isArray(neighbors) && neighbors.length > 0) { + // Check if any neighbors are NOT in this cluster + const hasExternalNeighbors = neighbors.some(neighbor => + !cluster.members.includes(typeof neighbor === 'object' ? neighbor.id : neighbor) + ) + + if (hasExternalNeighbors) { + crossDomainMembers.push(memberId) + } + } + } catch (error) { + // Skip members that can't be processed + continue + } + } + + return crossDomainMembers + } catch (error) { + console.error('Error in _findCrossDomainMembers:', error) + return [] + } } private async _findCrossDomainClusters(clusters: DomainCluster[], threshold: number): Promise { - // Find clusters that span multiple domains - return [] + try { + const crossDomainClusters: DomainCluster[] = [] + + // Group clusters by domain + const domainMap = new Map() + for (const cluster of clusters) { + const domain = cluster.domain || 'unknown' + if (!domainMap.has(domain)) { + domainMap.set(domain, []) + } + domainMap.get(domain)!.push(cluster) + } + + // Find clusters with high inter-domain similarity + const domains = Array.from(domainMap.keys()) + for (let i = 0; i < domains.length; i++) { + for (let j = i + 1; j < domains.length; j++) { + const domain1 = domains[i] + const domain2 = domains[j] + const clusters1 = domainMap.get(domain1)! + const clusters2 = domainMap.get(domain2)! + + // Compare clusters between domains + for (const c1 of clusters1) { + for (const c2 of clusters2) { + try { + // Calculate similarity between cluster centroids + if (!c1.centroid || !c2.centroid || c1.centroid.length === 0 || c2.centroid.length === 0) { + continue + } + + const similarity = 1 - cosineDistance( + Array.from(c1.centroid) as number[], + Array.from(c2.centroid) as number[] + ) + + if (similarity >= threshold) { + // Create a cross-domain cluster + const mergedMembers = [...new Set([...c1.members, ...c2.members])] + const mergedCentroid = this._averageVectors([ + Array.from(c1.centroid) as number[], + Array.from(c2.centroid) as number[] + ]) + + crossDomainClusters.push({ + ...c1, + id: `cross-${domain1}-${domain2}-${crossDomainClusters.length}`, + label: `Cross-domain: ${c1.label} + ${c2.label}`, + members: mergedMembers, + centroid: mergedCentroid, + domain: `${domain1}+${domain2}`, + domainConfidence: similarity, + crossDomainMembers: mergedMembers + }) + } + } catch (error) { + // Skip cluster pairs that can't be compared + continue + } + } + } + } + } + + return crossDomainClusters + } catch (error) { + console.error('Error in _findCrossDomainClusters:', error) + return [] + } + } + + private _averageVectors(vectors: number[][]): number[] { + if (vectors.length === 0) return [] + if (vectors.length === 1) return [...vectors[0]] + + const dim = vectors[0].length + const result = new Array(dim).fill(0) + + for (const vector of vectors) { + for (let i = 0; i < dim; i++) { + result[i] += vector[i] + } + } + + for (let i = 0; i < dim; i++) { + result[i] /= vectors.length + } + + return result } private async _getItemsByTimeWindow(timeField: string, window: TimeWindow): Promise { - // Implementation would query items within time window - return [] + try { + // Query all items from brain + const result = await this.brain.find({ + query: '', + limit: 10000 // Max items for clustering + }) + + if (!result || !Array.isArray(result)) { + return [] + } + + // Filter items within the time window + const itemsInWindow = result.filter((item: any) => { + if (!item || !item.entity) return false + + const entity = item.entity + + // Get timestamp value from various possible locations + let timestamp: any = null + + // Check root level first + if (timeField === 'createdAt' || timeField === 'updatedAt') { + timestamp = entity[timeField] + } + + // Check metadata/data + if (timestamp == null) { + timestamp = entity.metadata?.[timeField] || entity.data?.[timeField] + } + + if (timestamp == null) { + return false + } + + // Convert to Date if needed + const itemDate = timestamp instanceof Date ? timestamp : new Date(timestamp) + + if (isNaN(itemDate.getTime())) { + return false // Invalid date + } + + // Check if item falls within window + return itemDate >= window.start && itemDate <= window.end + }) + + // Map to format expected by clustering methods + return itemsInWindow.map((item: any) => { + const entity = item.entity + return { + id: entity.id, + vector: entity.embedding || entity.vector || [], + metadata: { + ...(entity.metadata || {}), + ...(entity.data || {}), + noun: entity.noun, + type: entity.noun, + createdAt: entity.createdAt, + updatedAt: entity.updatedAt, + label: entity.label + }, + nounType: entity.noun, + label: entity.label || entity.data || '', + data: entity.data + } + }) + } catch (error) { + console.error('Error in _getItemsByTimeWindow:', error) + return [] + } } private async _calculateTemporalMetrics(cluster: SemanticCluster, items: any[], timeField: string): Promise { diff --git a/tests/unit/neural/domain-time-clustering.test.ts b/tests/unit/neural/domain-time-clustering.test.ts new file mode 100644 index 00000000..0b74958c --- /dev/null +++ b/tests/unit/neural/domain-time-clustering.test.ts @@ -0,0 +1,263 @@ +/** + * Domain and Time Clustering Tests + * + * Tests for clusterByDomain() and clusterByTime() methods + * that were previously stub implementations. + */ + +import { describe, it, expect, beforeEach } from 'vitest' +import { Brainy } from '../../../src/brainy' +import { NounType } from '../../../src/types/graphTypes' +import { createAddParams } from '../../helpers/test-factory' + +describe('Domain and Time Clustering', () => { + let brain: Brainy + + beforeEach(async () => { + brain = new Brainy({ enableCache: false }) + await brain.init() + }) + + describe('clusterByDomain() - Field-based clustering', () => { + it('should cluster entities by type field', async () => { + // Add entities of different types + await brain.add(createAddParams({ + data: 'John Smith is a person', + type: NounType.Person + })) + await brain.add(createAddParams({ + data: 'Jane Doe is also a person', + type: NounType.Person + })) + await brain.add(createAddParams({ + data: 'Technical document about AI', + type: NounType.Document + })) + await brain.add(createAddParams({ + data: 'Research paper on machine learning', + type: NounType.Document + })) + await brain.add(createAddParams({ + data: 'Microsoft Corporation', + type: NounType.Organization + })) + + // Cluster by type field + const clusters = await brain.neural().clusterByDomain('type', { + minClusterSize: 1, + maxClusters: 10 + }) + + // Should have clusters for each type + expect(Array.isArray(clusters)).toBe(true) + expect(clusters.length).toBeGreaterThan(0) + + // Verify domain values exist + const domains = new Set(clusters.map(c => c.domain)) + expect(domains.has(NounType.Person) || domains.has('person')).toBe(true) + expect(domains.has(NounType.Document) || domains.has('document')).toBe(true) + }) + + it('should cluster entities by metadata field', async () => { + // Add entities with category metadata + await brain.add(createAddParams({ + data: 'JavaScript programming guide', + type: NounType.Document, + metadata: { category: 'programming' } + })) + await brain.add(createAddParams({ + data: 'Python tutorial', + type: NounType.Document, + metadata: { category: 'programming' } + })) + await brain.add(createAddParams({ + data: 'Chocolate cake recipe', + type: NounType.Document, + metadata: { category: 'cooking' } + })) + await brain.add(createAddParams({ + data: 'Pasta preparation', + type: NounType.Document, + metadata: { category: 'cooking' } + })) + + // Cluster by category field + const clusters = await brain.neural().clusterByDomain('category', { + minClusterSize: 1, + maxClusters: 5 + }) + + expect(Array.isArray(clusters)).toBe(true) + expect(clusters.length).toBeGreaterThan(0) + + // Verify categories are in domains + const domains = new Set(clusters.map(c => c.domain)) + expect(domains.has('programming')).toBe(true) + expect(domains.has('cooking')).toBe(true) + }) + + it('should handle entities without the specified field', async () => { + // Add entities with and without category + await brain.add(createAddParams({ + data: 'Has category', + metadata: { category: 'tech' } + })) + await brain.add(createAddParams({ + data: 'No category' + })) + + const clusters = await brain.neural().clusterByDomain('category', { + minClusterSize: 1 + }) + + expect(Array.isArray(clusters)).toBe(true) + + // Should have 'tech' and 'unknown' domains + const domains = new Set(clusters.map(c => c.domain)) + expect(domains.has('tech')).toBe(true) + expect(domains.has('unknown')).toBe(true) + }) + }) + + describe('clusterByTime() - Temporal clustering', () => { + it('should cluster entities by time windows', async () => { + const now = new Date() + const oneDayAgo = new Date(now.getTime() - 24 * 60 * 60 * 1000) + const oneWeekAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000) + const oneMonthAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000) + + // Add entities with different timestamps + await brain.add(createAddParams({ + data: 'Recent item 1', + metadata: { publishedAt: now.toISOString() } + })) + await brain.add(createAddParams({ + data: 'Recent item 2', + metadata: { publishedAt: oneDayAgo.toISOString() } + })) + await brain.add(createAddParams({ + data: 'Old item 1', + metadata: { publishedAt: oneWeekAgo.toISOString() } + })) + await brain.add(createAddParams({ + data: 'Very old item', + metadata: { publishedAt: oneMonthAgo.toISOString() } + })) + + // Define time windows + const timeWindows = [ + { + start: new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000), // Last 2 days + end: now, + label: 'Recent' + }, + { + start: new Date(now.getTime() - 14 * 24 * 60 * 60 * 1000), // 2-14 days ago + end: new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000), + label: 'This Week' + }, + { + start: new Date(now.getTime() - 60 * 24 * 60 * 60 * 1000), // 14-60 days ago + end: new Date(now.getTime() - 14 * 24 * 60 * 60 * 1000), + label: 'Older' + } + ] + + // Cluster by time + const clusters = await brain.neural().clusterByTime('publishedAt', timeWindows, { + timeField: 'publishedAt', + windows: timeWindows + }) + + expect(Array.isArray(clusters)).toBe(true) + expect(clusters.length).toBeGreaterThan(0) + + // Verify time windows are represented + const windowLabels = new Set(clusters.map(c => c.timeWindow?.label)) + expect(windowLabels.size).toBeGreaterThan(0) + }) + + it('should cluster entities by createdAt timestamps', async () => { + // These will use the auto-generated createdAt timestamps + const id1 = await brain.add(createAddParams({ + data: 'First item' + })) + + // Wait a bit to ensure different timestamps + await new Promise(resolve => setTimeout(resolve, 10)) + + const id2 = await brain.add(createAddParams({ + data: 'Second item' + })) + + const now = new Date() + const timeWindows = [ + { + start: new Date(now.getTime() - 60 * 60 * 1000), // Last hour + end: new Date(now.getTime() + 60 * 60 * 1000), // Next hour (to include all) + label: 'Now' + } + ] + + const clusters = await brain.neural().clusterByTime('createdAt', timeWindows, { + timeField: 'createdAt', + windows: timeWindows + }) + + expect(Array.isArray(clusters)).toBe(true) + + // Both items should be in the 'Now' time window + const nowCluster = clusters.find(c => c.timeWindow?.label === 'Now') + expect(nowCluster).toBeDefined() + if (nowCluster) { + expect(nowCluster.members.length).toBeGreaterThanOrEqual(2) + } + }) + + it('should handle empty time windows gracefully', async () => { + const futureStart = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) // 1 year from now + const futureEnd = new Date(Date.now() + 2 * 365 * 24 * 60 * 60 * 1000) // 2 years from now + + const timeWindows = [ + { + start: futureStart, + end: futureEnd, + label: 'Future' + } + ] + + const clusters = await brain.neural().clusterByTime('createdAt', timeWindows, { + timeField: 'createdAt', + windows: timeWindows + }) + + // Should return empty array or array with empty clusters + expect(Array.isArray(clusters)).toBe(true) + }) + }) + + describe('Cross-domain functionality', () => { + it('should find cross-domain clusters when enabled', async () => { + // Add entities from different domains with similar content + await brain.add(createAddParams({ + data: 'Machine learning and artificial intelligence', + type: NounType.Document, + metadata: { category: 'tech' } + })) + await brain.add(createAddParams({ + data: 'AI and neural networks', + type: NounType.Concept, + metadata: { category: 'science' } + })) + + const clusters = await brain.neural().clusterByDomain('category', { + minClusterSize: 1, + preserveDomainBoundaries: false, // Enable cross-domain clustering + crossDomainThreshold: 0.5 + }) + + expect(Array.isArray(clusters)).toBe(true) + expect(clusters.length).toBeGreaterThan(0) + }) + }) +})