fix: implement stub methods in Neural API clustering
Previously, clusterByDomain() and clusterByTime() methods contained stub implementations that always returned empty arrays. This caused empty results when attempting domain-based or temporal clustering. Changes: - Implement _getItemsByField() to query brain storage - Implement _getItemsByTimeWindow() to filter by time windows - Fix _groupByDomain() to check root, metadata, and data fields - Implement _findCrossDomainMembers() for cross-domain analysis - Implement _findCrossDomainClusters() to merge similar clusters - Add comprehensive tests for domain and time clustering - Update documentation structure to include VFS guides The methods now properly query the brain's storage, filter results, and return functional clustering data.
This commit is contained in:
parent
df13c196be
commit
1d2da823ed
3 changed files with 555 additions and 12 deletions
|
|
@ -104,6 +104,11 @@ docs/
|
|||
│ ├── noun-verb-taxonomy.md # Data model
|
||||
│ ├── triple-intelligence.md # Query system
|
||||
│ └── storage.md # Storage layer
|
||||
├── vfs/ # Virtual Filesystem
|
||||
│ ├── README.md # VFS overview
|
||||
│ ├── SEMANTIC_VFS.md # Semantic projections
|
||||
│ ├── VFS_API_GUIDE.md # Complete API reference
|
||||
│ └── QUICK_START.md # 5-minute setup
|
||||
└── api/ # API documentation
|
||||
├── README.md # API overview
|
||||
├── brainy-data.md # Main class
|
||||
|
|
|
|||
|
|
@ -2414,8 +2414,70 @@ export class ImprovedNeuralAPI {
|
|||
}
|
||||
|
||||
private async _getItemsByField(field: string): Promise<any[]> {
|
||||
// Implementation would query items by metadata field
|
||||
return []
|
||||
try {
|
||||
// Query all items from brain (limit to reasonable number for clustering)
|
||||
const result = await this.brain.find({
|
||||
query: '',
|
||||
limit: 10000 // Max items for clustering
|
||||
})
|
||||
|
||||
if (!result || !Array.isArray(result)) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Filter items that have the specified field (check both root level and metadata)
|
||||
const itemsWithField = result.filter((item: any) => {
|
||||
if (!item || !item.entity) return false
|
||||
|
||||
const entity = item.entity
|
||||
|
||||
// Check root level fields first (e.g., 'noun' for type)
|
||||
if (field === 'type' || field === 'nounType') {
|
||||
return entity.noun != null
|
||||
}
|
||||
|
||||
// Check if field exists at root level
|
||||
if (entity[field] != null) {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if field exists in metadata/data
|
||||
if (entity.metadata?.[field] != null) {
|
||||
return true
|
||||
}
|
||||
|
||||
if (entity.data?.[field] != null) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
})
|
||||
|
||||
// Map to format expected by clustering methods
|
||||
return itemsWithField.map((item: any) => {
|
||||
const entity = item.entity
|
||||
return {
|
||||
id: entity.id,
|
||||
vector: entity.embedding || entity.vector || [],
|
||||
metadata: {
|
||||
...(entity.metadata || {}),
|
||||
...(entity.data || {}),
|
||||
// Include root-level fields in metadata for easy access
|
||||
noun: entity.noun,
|
||||
type: entity.noun,
|
||||
createdAt: entity.createdAt,
|
||||
updatedAt: entity.updatedAt,
|
||||
label: entity.label
|
||||
},
|
||||
nounType: entity.noun,
|
||||
label: entity.label || entity.data || '',
|
||||
data: entity.data
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error in _getItemsByField:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
// ===== TRIPLE INTELLIGENCE INTEGRATION =====
|
||||
|
|
@ -2875,11 +2937,39 @@ export class ImprovedNeuralAPI {
|
|||
private _groupByDomain(items: any[], field: string): Map<string, any[]> {
|
||||
const groups = new Map()
|
||||
for (const item of items) {
|
||||
const domain = item.metadata?.[field] || 'unknown'
|
||||
if (!groups.has(domain)) {
|
||||
groups.set(domain, [])
|
||||
// Check multiple locations for the field value
|
||||
let domain: any = 'unknown'
|
||||
|
||||
// Special handling for type/nounType field
|
||||
if (field === 'type' || field === 'nounType') {
|
||||
domain = item.nounType || item.metadata?.noun || item.metadata?.type || 'unknown'
|
||||
} else {
|
||||
// Check root level first
|
||||
domain = item[field]
|
||||
|
||||
// Then check metadata
|
||||
if (domain == null) {
|
||||
domain = item.metadata?.[field]
|
||||
}
|
||||
|
||||
// Then check data
|
||||
if (domain == null) {
|
||||
domain = item.data?.[field]
|
||||
}
|
||||
|
||||
// Fallback to unknown
|
||||
if (domain == null) {
|
||||
domain = 'unknown'
|
||||
}
|
||||
}
|
||||
groups.get(domain).push(item)
|
||||
|
||||
// Convert domain to string for Map key
|
||||
const domainKey = String(domain)
|
||||
|
||||
if (!groups.has(domainKey)) {
|
||||
groups.set(domainKey, [])
|
||||
}
|
||||
groups.get(domainKey).push(item)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
|
@ -2902,18 +2992,203 @@ export class ImprovedNeuralAPI {
|
|||
}
|
||||
|
||||
private async _findCrossDomainMembers(cluster: SemanticCluster, threshold: number): Promise<string[]> {
|
||||
// Find members that might belong to multiple domains
|
||||
return []
|
||||
try {
|
||||
// Find cluster members that have high similarity to items in other domains
|
||||
const crossDomainMembers: string[] = []
|
||||
|
||||
for (const memberId of cluster.members) {
|
||||
try {
|
||||
// Get neighbors for this member
|
||||
const neighbors = await this.neighbors(memberId, {
|
||||
limit: 10,
|
||||
minSimilarity: threshold
|
||||
})
|
||||
|
||||
if (Array.isArray(neighbors) && neighbors.length > 0) {
|
||||
// Check if any neighbors are NOT in this cluster
|
||||
const hasExternalNeighbors = neighbors.some(neighbor =>
|
||||
!cluster.members.includes(typeof neighbor === 'object' ? neighbor.id : neighbor)
|
||||
)
|
||||
|
||||
if (hasExternalNeighbors) {
|
||||
crossDomainMembers.push(memberId)
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Skip members that can't be processed
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return crossDomainMembers
|
||||
} catch (error) {
|
||||
console.error('Error in _findCrossDomainMembers:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
private async _findCrossDomainClusters(clusters: DomainCluster[], threshold: number): Promise<DomainCluster[]> {
|
||||
// Find clusters that span multiple domains
|
||||
return []
|
||||
try {
|
||||
const crossDomainClusters: DomainCluster[] = []
|
||||
|
||||
// Group clusters by domain
|
||||
const domainMap = new Map<string, DomainCluster[]>()
|
||||
for (const cluster of clusters) {
|
||||
const domain = cluster.domain || 'unknown'
|
||||
if (!domainMap.has(domain)) {
|
||||
domainMap.set(domain, [])
|
||||
}
|
||||
domainMap.get(domain)!.push(cluster)
|
||||
}
|
||||
|
||||
// Find clusters with high inter-domain similarity
|
||||
const domains = Array.from(domainMap.keys())
|
||||
for (let i = 0; i < domains.length; i++) {
|
||||
for (let j = i + 1; j < domains.length; j++) {
|
||||
const domain1 = domains[i]
|
||||
const domain2 = domains[j]
|
||||
const clusters1 = domainMap.get(domain1)!
|
||||
const clusters2 = domainMap.get(domain2)!
|
||||
|
||||
// Compare clusters between domains
|
||||
for (const c1 of clusters1) {
|
||||
for (const c2 of clusters2) {
|
||||
try {
|
||||
// Calculate similarity between cluster centroids
|
||||
if (!c1.centroid || !c2.centroid || c1.centroid.length === 0 || c2.centroid.length === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
const similarity = 1 - cosineDistance(
|
||||
Array.from(c1.centroid) as number[],
|
||||
Array.from(c2.centroid) as number[]
|
||||
)
|
||||
|
||||
if (similarity >= threshold) {
|
||||
// Create a cross-domain cluster
|
||||
const mergedMembers = [...new Set([...c1.members, ...c2.members])]
|
||||
const mergedCentroid = this._averageVectors([
|
||||
Array.from(c1.centroid) as number[],
|
||||
Array.from(c2.centroid) as number[]
|
||||
])
|
||||
|
||||
crossDomainClusters.push({
|
||||
...c1,
|
||||
id: `cross-${domain1}-${domain2}-${crossDomainClusters.length}`,
|
||||
label: `Cross-domain: ${c1.label} + ${c2.label}`,
|
||||
members: mergedMembers,
|
||||
centroid: mergedCentroid,
|
||||
domain: `${domain1}+${domain2}`,
|
||||
domainConfidence: similarity,
|
||||
crossDomainMembers: mergedMembers
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
// Skip cluster pairs that can't be compared
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return crossDomainClusters
|
||||
} catch (error) {
|
||||
console.error('Error in _findCrossDomainClusters:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
private _averageVectors(vectors: number[][]): number[] {
|
||||
if (vectors.length === 0) return []
|
||||
if (vectors.length === 1) return [...vectors[0]]
|
||||
|
||||
const dim = vectors[0].length
|
||||
const result = new Array(dim).fill(0)
|
||||
|
||||
for (const vector of vectors) {
|
||||
for (let i = 0; i < dim; i++) {
|
||||
result[i] += vector[i]
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < dim; i++) {
|
||||
result[i] /= vectors.length
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
private async _getItemsByTimeWindow(timeField: string, window: TimeWindow): Promise<any[]> {
|
||||
// Implementation would query items within time window
|
||||
return []
|
||||
try {
|
||||
// Query all items from brain
|
||||
const result = await this.brain.find({
|
||||
query: '',
|
||||
limit: 10000 // Max items for clustering
|
||||
})
|
||||
|
||||
if (!result || !Array.isArray(result)) {
|
||||
return []
|
||||
}
|
||||
|
||||
// Filter items within the time window
|
||||
const itemsInWindow = result.filter((item: any) => {
|
||||
if (!item || !item.entity) return false
|
||||
|
||||
const entity = item.entity
|
||||
|
||||
// Get timestamp value from various possible locations
|
||||
let timestamp: any = null
|
||||
|
||||
// Check root level first
|
||||
if (timeField === 'createdAt' || timeField === 'updatedAt') {
|
||||
timestamp = entity[timeField]
|
||||
}
|
||||
|
||||
// Check metadata/data
|
||||
if (timestamp == null) {
|
||||
timestamp = entity.metadata?.[timeField] || entity.data?.[timeField]
|
||||
}
|
||||
|
||||
if (timestamp == null) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Convert to Date if needed
|
||||
const itemDate = timestamp instanceof Date ? timestamp : new Date(timestamp)
|
||||
|
||||
if (isNaN(itemDate.getTime())) {
|
||||
return false // Invalid date
|
||||
}
|
||||
|
||||
// Check if item falls within window
|
||||
return itemDate >= window.start && itemDate <= window.end
|
||||
})
|
||||
|
||||
// Map to format expected by clustering methods
|
||||
return itemsInWindow.map((item: any) => {
|
||||
const entity = item.entity
|
||||
return {
|
||||
id: entity.id,
|
||||
vector: entity.embedding || entity.vector || [],
|
||||
metadata: {
|
||||
...(entity.metadata || {}),
|
||||
...(entity.data || {}),
|
||||
noun: entity.noun,
|
||||
type: entity.noun,
|
||||
createdAt: entity.createdAt,
|
||||
updatedAt: entity.updatedAt,
|
||||
label: entity.label
|
||||
},
|
||||
nounType: entity.noun,
|
||||
label: entity.label || entity.data || '',
|
||||
data: entity.data
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error in _getItemsByTimeWindow:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
private async _calculateTemporalMetrics(cluster: SemanticCluster, items: any[], timeField: string): Promise<any> {
|
||||
|
|
|
|||
263
tests/unit/neural/domain-time-clustering.test.ts
Normal file
263
tests/unit/neural/domain-time-clustering.test.ts
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
/**
|
||||
* Domain and Time Clustering Tests
|
||||
*
|
||||
* Tests for clusterByDomain() and clusterByTime() methods
|
||||
* that were previously stub implementations.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest'
|
||||
import { Brainy } from '../../../src/brainy'
|
||||
import { NounType } from '../../../src/types/graphTypes'
|
||||
import { createAddParams } from '../../helpers/test-factory'
|
||||
|
||||
describe('Domain and Time Clustering', () => {
|
||||
let brain: Brainy
|
||||
|
||||
beforeEach(async () => {
|
||||
brain = new Brainy({ enableCache: false })
|
||||
await brain.init()
|
||||
})
|
||||
|
||||
describe('clusterByDomain() - Field-based clustering', () => {
|
||||
it('should cluster entities by type field', async () => {
|
||||
// Add entities of different types
|
||||
await brain.add(createAddParams({
|
||||
data: 'John Smith is a person',
|
||||
type: NounType.Person
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Jane Doe is also a person',
|
||||
type: NounType.Person
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Technical document about AI',
|
||||
type: NounType.Document
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Research paper on machine learning',
|
||||
type: NounType.Document
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Microsoft Corporation',
|
||||
type: NounType.Organization
|
||||
}))
|
||||
|
||||
// Cluster by type field
|
||||
const clusters = await brain.neural().clusterByDomain('type', {
|
||||
minClusterSize: 1,
|
||||
maxClusters: 10
|
||||
})
|
||||
|
||||
// Should have clusters for each type
|
||||
expect(Array.isArray(clusters)).toBe(true)
|
||||
expect(clusters.length).toBeGreaterThan(0)
|
||||
|
||||
// Verify domain values exist
|
||||
const domains = new Set(clusters.map(c => c.domain))
|
||||
expect(domains.has(NounType.Person) || domains.has('person')).toBe(true)
|
||||
expect(domains.has(NounType.Document) || domains.has('document')).toBe(true)
|
||||
})
|
||||
|
||||
it('should cluster entities by metadata field', async () => {
|
||||
// Add entities with category metadata
|
||||
await brain.add(createAddParams({
|
||||
data: 'JavaScript programming guide',
|
||||
type: NounType.Document,
|
||||
metadata: { category: 'programming' }
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Python tutorial',
|
||||
type: NounType.Document,
|
||||
metadata: { category: 'programming' }
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Chocolate cake recipe',
|
||||
type: NounType.Document,
|
||||
metadata: { category: 'cooking' }
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Pasta preparation',
|
||||
type: NounType.Document,
|
||||
metadata: { category: 'cooking' }
|
||||
}))
|
||||
|
||||
// Cluster by category field
|
||||
const clusters = await brain.neural().clusterByDomain('category', {
|
||||
minClusterSize: 1,
|
||||
maxClusters: 5
|
||||
})
|
||||
|
||||
expect(Array.isArray(clusters)).toBe(true)
|
||||
expect(clusters.length).toBeGreaterThan(0)
|
||||
|
||||
// Verify categories are in domains
|
||||
const domains = new Set(clusters.map(c => c.domain))
|
||||
expect(domains.has('programming')).toBe(true)
|
||||
expect(domains.has('cooking')).toBe(true)
|
||||
})
|
||||
|
||||
it('should handle entities without the specified field', async () => {
|
||||
// Add entities with and without category
|
||||
await brain.add(createAddParams({
|
||||
data: 'Has category',
|
||||
metadata: { category: 'tech' }
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'No category'
|
||||
}))
|
||||
|
||||
const clusters = await brain.neural().clusterByDomain('category', {
|
||||
minClusterSize: 1
|
||||
})
|
||||
|
||||
expect(Array.isArray(clusters)).toBe(true)
|
||||
|
||||
// Should have 'tech' and 'unknown' domains
|
||||
const domains = new Set(clusters.map(c => c.domain))
|
||||
expect(domains.has('tech')).toBe(true)
|
||||
expect(domains.has('unknown')).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('clusterByTime() - Temporal clustering', () => {
|
||||
it('should cluster entities by time windows', async () => {
|
||||
const now = new Date()
|
||||
const oneDayAgo = new Date(now.getTime() - 24 * 60 * 60 * 1000)
|
||||
const oneWeekAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000)
|
||||
const oneMonthAgo = new Date(now.getTime() - 30 * 24 * 60 * 60 * 1000)
|
||||
|
||||
// Add entities with different timestamps
|
||||
await brain.add(createAddParams({
|
||||
data: 'Recent item 1',
|
||||
metadata: { publishedAt: now.toISOString() }
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Recent item 2',
|
||||
metadata: { publishedAt: oneDayAgo.toISOString() }
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Old item 1',
|
||||
metadata: { publishedAt: oneWeekAgo.toISOString() }
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'Very old item',
|
||||
metadata: { publishedAt: oneMonthAgo.toISOString() }
|
||||
}))
|
||||
|
||||
// Define time windows
|
||||
const timeWindows = [
|
||||
{
|
||||
start: new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000), // Last 2 days
|
||||
end: now,
|
||||
label: 'Recent'
|
||||
},
|
||||
{
|
||||
start: new Date(now.getTime() - 14 * 24 * 60 * 60 * 1000), // 2-14 days ago
|
||||
end: new Date(now.getTime() - 2 * 24 * 60 * 60 * 1000),
|
||||
label: 'This Week'
|
||||
},
|
||||
{
|
||||
start: new Date(now.getTime() - 60 * 24 * 60 * 60 * 1000), // 14-60 days ago
|
||||
end: new Date(now.getTime() - 14 * 24 * 60 * 60 * 1000),
|
||||
label: 'Older'
|
||||
}
|
||||
]
|
||||
|
||||
// Cluster by time
|
||||
const clusters = await brain.neural().clusterByTime('publishedAt', timeWindows, {
|
||||
timeField: 'publishedAt',
|
||||
windows: timeWindows
|
||||
})
|
||||
|
||||
expect(Array.isArray(clusters)).toBe(true)
|
||||
expect(clusters.length).toBeGreaterThan(0)
|
||||
|
||||
// Verify time windows are represented
|
||||
const windowLabels = new Set(clusters.map(c => c.timeWindow?.label))
|
||||
expect(windowLabels.size).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('should cluster entities by createdAt timestamps', async () => {
|
||||
// These will use the auto-generated createdAt timestamps
|
||||
const id1 = await brain.add(createAddParams({
|
||||
data: 'First item'
|
||||
}))
|
||||
|
||||
// Wait a bit to ensure different timestamps
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
|
||||
const id2 = await brain.add(createAddParams({
|
||||
data: 'Second item'
|
||||
}))
|
||||
|
||||
const now = new Date()
|
||||
const timeWindows = [
|
||||
{
|
||||
start: new Date(now.getTime() - 60 * 60 * 1000), // Last hour
|
||||
end: new Date(now.getTime() + 60 * 60 * 1000), // Next hour (to include all)
|
||||
label: 'Now'
|
||||
}
|
||||
]
|
||||
|
||||
const clusters = await brain.neural().clusterByTime('createdAt', timeWindows, {
|
||||
timeField: 'createdAt',
|
||||
windows: timeWindows
|
||||
})
|
||||
|
||||
expect(Array.isArray(clusters)).toBe(true)
|
||||
|
||||
// Both items should be in the 'Now' time window
|
||||
const nowCluster = clusters.find(c => c.timeWindow?.label === 'Now')
|
||||
expect(nowCluster).toBeDefined()
|
||||
if (nowCluster) {
|
||||
expect(nowCluster.members.length).toBeGreaterThanOrEqual(2)
|
||||
}
|
||||
})
|
||||
|
||||
it('should handle empty time windows gracefully', async () => {
|
||||
const futureStart = new Date(Date.now() + 365 * 24 * 60 * 60 * 1000) // 1 year from now
|
||||
const futureEnd = new Date(Date.now() + 2 * 365 * 24 * 60 * 60 * 1000) // 2 years from now
|
||||
|
||||
const timeWindows = [
|
||||
{
|
||||
start: futureStart,
|
||||
end: futureEnd,
|
||||
label: 'Future'
|
||||
}
|
||||
]
|
||||
|
||||
const clusters = await brain.neural().clusterByTime('createdAt', timeWindows, {
|
||||
timeField: 'createdAt',
|
||||
windows: timeWindows
|
||||
})
|
||||
|
||||
// Should return empty array or array with empty clusters
|
||||
expect(Array.isArray(clusters)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Cross-domain functionality', () => {
|
||||
it('should find cross-domain clusters when enabled', async () => {
|
||||
// Add entities from different domains with similar content
|
||||
await brain.add(createAddParams({
|
||||
data: 'Machine learning and artificial intelligence',
|
||||
type: NounType.Document,
|
||||
metadata: { category: 'tech' }
|
||||
}))
|
||||
await brain.add(createAddParams({
|
||||
data: 'AI and neural networks',
|
||||
type: NounType.Concept,
|
||||
metadata: { category: 'science' }
|
||||
}))
|
||||
|
||||
const clusters = await brain.neural().clusterByDomain('category', {
|
||||
minClusterSize: 1,
|
||||
preserveDomainBoundaries: false, // Enable cross-domain clustering
|
||||
crossDomainThreshold: 0.5
|
||||
})
|
||||
|
||||
expect(Array.isArray(clusters)).toBe(true)
|
||||
expect(clusters.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue