fix: metadata explosion bug - 69K files reduced to ~1K

Critical fix for metadata indexing that was creating 60+ chunk files per entity.

Root cause: Vector embeddings (384-dimensional arrays) were being indexed in
metadata, causing each dimension to create a separate chunk file with numeric
field names ("0", "1", "2", etc.).

Changes:
- Modified extractIndexableFields() to exclude vector/embedding fields
- Added NEVER_INDEX set: ['vector', 'embedding', 'embeddings', 'connections']
- Added safety check to skip arrays > 10 elements
- Preserves small array indexing (tags, categories, roles)

Impact:
- Reduces metadata files from 69,429 → ~1,200 (58x reduction)
- Fixes server initialization hangs
- Fixes metadata batch loading stalling at batch 23
- Fixes VFS getDescendants() hanging with large datasets
- Fixes Graph View UI not loading

Test Results:
- 7/7 integration tests passing
- Verified: 6 chunk files for 10 entities (was 7,210 before fix)
- 611/622 unit tests passing

Files Modified:
- src/utils/metadataIndex.ts - Core fix
- src/coreTypes.ts - HNSWVerb type enforcement with VerbType enum
- src/storage/adapters/* - Include core relational fields in HNSWVerb
- src/storage/adapters/baseStorageAdapter.ts - Type enforcement (HNSWNoun, GraphVerb)
- tests/integration/metadata-vector-exclusion.test.ts - Comprehensive test coverage

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
David Snelling 2025-10-16 16:10:31 -07:00
parent 7921a5b744
commit e600865d96
13 changed files with 571 additions and 112 deletions

View file

@ -3,7 +3,7 @@
* Provides common functionality for all storage adapters, including statistics tracking
*/
import { StatisticsData, StorageAdapter } from '../../coreTypes.js'
import { StatisticsData, StorageAdapter, HNSWNoun, GraphVerb } from '../../coreTypes.js'
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
@ -14,23 +14,23 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
// Abstract methods that must be implemented by subclasses
abstract init(): Promise<void>
abstract saveNoun(noun: any): Promise<void>
abstract saveNoun(noun: HNSWNoun): Promise<void>
abstract getNoun(id: string): Promise<any | null>
abstract getNoun(id: string): Promise<HNSWNoun | null>
abstract getNounsByNounType(nounType: string): Promise<any[]>
abstract getNounsByNounType(nounType: string): Promise<HNSWNoun[]>
abstract deleteNoun(id: string): Promise<void>
abstract saveVerb(verb: any): Promise<void>
abstract saveVerb(verb: GraphVerb): Promise<void>
abstract getVerb(id: string): Promise<any | null>
abstract getVerb(id: string): Promise<GraphVerb | null>
abstract getVerbsBySource(sourceId: string): Promise<any[]>
abstract getVerbsBySource(sourceId: string): Promise<GraphVerb[]>
abstract getVerbsByTarget(targetId: string): Promise<any[]>
abstract getVerbsByTarget(targetId: string): Promise<GraphVerb[]>
abstract getVerbsByType(type: string): Promise<any[]>
abstract getVerbsByType(type: string): Promise<GraphVerb[]>
abstract deleteVerb(id: string): Promise<void>
@ -98,7 +98,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
metadata?: Record<string, any>
}
}): Promise<{
items: any[]
items: HNSWNoun[]
totalCount?: number
hasMore: boolean
nextCursor?: string
@ -123,7 +123,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
metadata?: Record<string, any>
}
}): Promise<{
items: any[]
items: GraphVerb[]
totalCount?: number
hasMore: boolean
nextCursor?: string
@ -144,7 +144,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
metadata?: Record<string, any>
}
}): Promise<{
items: any[]
items: HNSWNoun[]
totalCount?: number
hasMore: boolean
nextCursor?: string
@ -167,7 +167,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
metadata?: Record<string, any>
}
}): Promise<{
items: any[]
items: GraphVerb[]
totalCount?: number
hasMore: boolean
nextCursor?: string