fix: metadata explosion bug - 69K files reduced to ~1K
Critical fix for metadata indexing that was creating 60+ chunk files per entity.
Root cause: Vector embeddings (384-dimensional arrays) were being indexed in
metadata, causing each dimension to create a separate chunk file with numeric
field names ("0", "1", "2", etc.).
Changes:
- Modified extractIndexableFields() to exclude vector/embedding fields
- Added NEVER_INDEX set: ['vector', 'embedding', 'embeddings', 'connections']
- Added safety check to skip arrays > 10 elements
- Preserves small array indexing (tags, categories, roles)
Impact:
- Reduces metadata files from 69,429 → ~1,200 (58x reduction)
- Fixes server initialization hangs
- Fixes metadata batch loading stalling at batch 23
- Fixes VFS getDescendants() hanging with large datasets
- Fixes Graph View UI not loading
Test Results:
- 7/7 integration tests passing
- Verified: 6 chunk files for 10 entities (was 7,210 before fix)
- 611/622 unit tests passing
Files Modified:
- src/utils/metadataIndex.ts - Core fix
- src/coreTypes.ts - HNSWVerb type enforcement with VerbType enum
- src/storage/adapters/* - Include core relational fields in HNSWVerb
- src/storage/adapters/baseStorageAdapter.ts - Type enforcement (HNSWNoun, GraphVerb)
- tests/integration/metadata-vector-exclusion.test.ts - Comprehensive test coverage
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
7921a5b744
commit
e600865d96
13 changed files with 571 additions and 112 deletions
|
|
@ -3,7 +3,7 @@
|
|||
* Provides common functionality for all storage adapters, including statistics tracking
|
||||
*/
|
||||
|
||||
import { StatisticsData, StorageAdapter } from '../../coreTypes.js'
|
||||
import { StatisticsData, StorageAdapter, HNSWNoun, GraphVerb } from '../../coreTypes.js'
|
||||
import { extractFieldNamesFromJson, mapToStandardField } from '../../utils/fieldNameTracking.js'
|
||||
import { getGlobalMutex, cleanupMutexes } from '../../utils/mutex.js'
|
||||
|
||||
|
|
@ -14,23 +14,23 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
// Abstract methods that must be implemented by subclasses
|
||||
abstract init(): Promise<void>
|
||||
|
||||
abstract saveNoun(noun: any): Promise<void>
|
||||
abstract saveNoun(noun: HNSWNoun): Promise<void>
|
||||
|
||||
abstract getNoun(id: string): Promise<any | null>
|
||||
abstract getNoun(id: string): Promise<HNSWNoun | null>
|
||||
|
||||
abstract getNounsByNounType(nounType: string): Promise<any[]>
|
||||
abstract getNounsByNounType(nounType: string): Promise<HNSWNoun[]>
|
||||
|
||||
abstract deleteNoun(id: string): Promise<void>
|
||||
|
||||
abstract saveVerb(verb: any): Promise<void>
|
||||
abstract saveVerb(verb: GraphVerb): Promise<void>
|
||||
|
||||
abstract getVerb(id: string): Promise<any | null>
|
||||
abstract getVerb(id: string): Promise<GraphVerb | null>
|
||||
|
||||
abstract getVerbsBySource(sourceId: string): Promise<any[]>
|
||||
abstract getVerbsBySource(sourceId: string): Promise<GraphVerb[]>
|
||||
|
||||
abstract getVerbsByTarget(targetId: string): Promise<any[]>
|
||||
abstract getVerbsByTarget(targetId: string): Promise<GraphVerb[]>
|
||||
|
||||
abstract getVerbsByType(type: string): Promise<any[]>
|
||||
abstract getVerbsByType(type: string): Promise<GraphVerb[]>
|
||||
|
||||
abstract deleteVerb(id: string): Promise<void>
|
||||
|
||||
|
|
@ -98,7 +98,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
}): Promise<{
|
||||
items: any[]
|
||||
items: HNSWNoun[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
@ -123,7 +123,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
}): Promise<{
|
||||
items: any[]
|
||||
items: GraphVerb[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
@ -144,7 +144,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
}): Promise<{
|
||||
items: any[]
|
||||
items: HNSWNoun[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
@ -167,7 +167,7 @@ export abstract class BaseStorageAdapter implements StorageAdapter {
|
|||
metadata?: Record<string, any>
|
||||
}
|
||||
}): Promise<{
|
||||
items: any[]
|
||||
items: GraphVerb[]
|
||||
totalCount?: number
|
||||
hasMore: boolean
|
||||
nextCursor?: string
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue