fix: v3.50.2 emergency hotfix - exclude numeric field names from metadata indexing
Critical fix for incomplete v3.50.1 release.
Problem: v3.50.1 prevented vector fields by name ('vector', 'embedding')
but missed vectors stored as objects with numeric keys: {0: 0.1, 1: 0.2, ...}
Studio team diagnostics showed:
- 212,531 chunk files with NUMERIC field names
- Examples: "field": "54716", "field": "100000", "field": "100001"
- 424,837 total files (expected ~1,200)
Root Cause: Vectors converted to objects with numeric keys were still
being indexed because field name check only caught semantic names.
Fix Applied (src/utils/metadataIndex.ts:1106):
- Added regex check: if (/^\d+$/.test(key)) continue
- Skips ANY purely numeric field name (array indices as object keys)
- Catches: "0", "1", "2", "100", "54716", "100000", etc.
Test Coverage:
- Added new test: "should NOT index objects with numeric keys (v3.50.2 fix)"
- Verifies NO chunk files have numeric field names
- All 8 integration tests passing
Impact:
- Prevents 212K+ chunk files from being created
- Reduces file count from 424K to ~1,200 (354x reduction)
- Fixes server hangs during initialization
- Completes the metadata explosion fix started in v3.50.1
This commit is contained in:
parent
8fb811d5f9
commit
d02359ec60
2 changed files with 50 additions and 2 deletions
|
|
@ -1083,8 +1083,9 @@ export class MetadataIndexManager {
|
|||
* Extract indexable field-value pairs from metadata
|
||||
*
|
||||
* BUG FIX (v3.50.1): Exclude vector embeddings and large arrays from indexing
|
||||
* BUG FIX (v3.50.2): Also exclude purely numeric field names (array indices)
|
||||
* - Vector fields (384+ dimensions) were creating 825K chunk files for 1,144 entities
|
||||
* - Arrays should not have their indices indexed as separate fields
|
||||
* - Arrays converted to objects with numeric keys were still being indexed
|
||||
*/
|
||||
private extractIndexableFields(metadata: any): Array<{ field: string, value: any }> {
|
||||
const fields: Array<{ field: string, value: any }> = []
|
||||
|
|
@ -1099,6 +1100,11 @@ export class MetadataIndexManager {
|
|||
// Skip fields in never-index list (CRITICAL: prevents vector indexing bug)
|
||||
if (NEVER_INDEX.has(key)) continue
|
||||
|
||||
// Skip purely numeric field names (array indices converted to object keys)
|
||||
// Legitimate field names should never be purely numeric
|
||||
// This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...}
|
||||
if (/^\d+$/.test(key)) continue
|
||||
|
||||
// Skip fields based on user configuration
|
||||
if (!this.shouldIndexField(fullKey)) continue
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue