fix: metadata explosion bug - 69K files reduced to ~1K
Critical fix for metadata indexing that was creating 60+ chunk files per entity.
Root cause: Vector embeddings (384-dimensional arrays) were being indexed in
metadata, causing each dimension to create a separate chunk file with numeric
field names ("0", "1", "2", etc.).
Changes:
- Modified extractIndexableFields() to exclude vector/embedding fields
- Added NEVER_INDEX set: ['vector', 'embedding', 'embeddings', 'connections']
- Added safety check to skip arrays > 10 elements
- Preserves small array indexing (tags, categories, roles)
Impact:
- Reduces metadata files from 69,429 → ~1,200 (58x reduction)
- Fixes server initialization hangs
- Fixes metadata batch loading stalling at batch 23
- Fixes VFS getDescendants() hanging with large datasets
- Fixes Graph View UI not loading
Test Results:
- 7/7 integration tests passing
- Verified: 6 chunk files for 10 entities (was 7,210 before fix)
- 611/622 unit tests passing
Files Modified:
- src/utils/metadataIndex.ts - Core fix
- src/coreTypes.ts - HNSWVerb type enforcement with VerbType enum
- src/storage/adapters/* - Include core relational fields in HNSWVerb
- src/storage/adapters/baseStorageAdapter.ts - Type enforcement (HNSWNoun, GraphVerb)
- tests/integration/metadata-vector-exclusion.test.ts - Comprehensive test coverage
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
7921a5b744
commit
e600865d96
13 changed files with 571 additions and 112 deletions
|
|
@ -728,6 +728,8 @@ export class R2Storage extends BaseStorage {
|
|||
const requestId = await this.applyBackpressure()
|
||||
|
||||
try {
|
||||
// ARCHITECTURAL FIX (v3.50.1): Include core relational fields in verb vector file
|
||||
// These fields are essential for 90% of operations - no metadata lookup needed
|
||||
const serializableEdge = {
|
||||
id: edge.id,
|
||||
vector: edge.vector,
|
||||
|
|
@ -736,7 +738,15 @@ export class R2Storage extends BaseStorage {
|
|||
level,
|
||||
Array.from(verbIds)
|
||||
])
|
||||
)
|
||||
),
|
||||
|
||||
// CORE RELATIONAL DATA (v3.50.1+)
|
||||
verb: edge.verb,
|
||||
sourceId: edge.sourceId,
|
||||
targetId: edge.targetId,
|
||||
|
||||
// User metadata (if any) - saved separately for scalability
|
||||
// metadata field is saved separately via saveVerbMetadata()
|
||||
}
|
||||
|
||||
const key = this.getVerbKey(edge.id)
|
||||
|
|
@ -814,10 +824,19 @@ export class R2Storage extends BaseStorage {
|
|||
connections.set(Number(level), new Set(verbIds as string[]))
|
||||
}
|
||||
|
||||
// ARCHITECTURAL FIX (v3.50.1): Return HNSWVerb with core relational fields
|
||||
const edge: Edge = {
|
||||
id: data.id,
|
||||
vector: data.vector,
|
||||
connections
|
||||
connections,
|
||||
|
||||
// CORE RELATIONAL DATA (read from vector file)
|
||||
verb: data.verb,
|
||||
sourceId: data.sourceId,
|
||||
targetId: data.targetId,
|
||||
|
||||
// User metadata (retrieved separately via getVerbMetadata())
|
||||
metadata: data.metadata
|
||||
}
|
||||
|
||||
this.verbCacheManager.set(id, edge)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue