diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 04374423..9e7843ee 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -1083,8 +1083,9 @@ export class MetadataIndexManager { * Extract indexable field-value pairs from metadata * * BUG FIX (v3.50.1): Exclude vector embeddings and large arrays from indexing + * BUG FIX (v3.50.2): Also exclude purely numeric field names (array indices) * - Vector fields (384+ dimensions) were creating 825K chunk files for 1,144 entities - * - Arrays should not have their indices indexed as separate fields + * - Arrays converted to objects with numeric keys were still being indexed */ private extractIndexableFields(metadata: any): Array<{ field: string, value: any }> { const fields: Array<{ field: string, value: any }> = [] @@ -1099,6 +1100,11 @@ export class MetadataIndexManager { // Skip fields in never-index list (CRITICAL: prevents vector indexing bug) if (NEVER_INDEX.has(key)) continue + // Skip purely numeric field names (array indices converted to object keys) + // Legitimate field names should never be purely numeric + // This catches vectors stored as objects: {0: 0.1, 1: 0.2, ...} + if (/^\d+$/.test(key)) continue + // Skip fields based on user configuration if (!this.shouldIndexField(fullKey)) continue diff --git a/tests/integration/metadata-vector-exclusion.test.ts b/tests/integration/metadata-vector-exclusion.test.ts index 60b20302..e9bf6ec7 100644 --- a/tests/integration/metadata-vector-exclusion.test.ts +++ b/tests/integration/metadata-vector-exclusion.test.ts @@ -75,7 +75,8 @@ describe('Metadata Vector Exclusion Fix', () => { const content = JSON.parse(readFileSync(join(systemDir, file), 'utf-8')) const fieldName = content.field - // Field should be a semantic name, NOT a number + // CRITICAL: Field should be a semantic name, NOT a number + // This catches both "vector" fields AND numeric keys like "0", "1", "54716" expect(fieldName).not.toMatch(/^\d+$/) // Field should NOT be 'vector', 'embedding', 'embeddings' @@ -85,6 +86,47 @@ describe('Metadata Vector Exclusion Fix', () => { } }) + it('should NOT index objects with numeric keys (v3.50.2 fix)', async () => { + // Add entity with object that has numeric keys (simulates vector-as-object) + await brainy.add({ + type: 'person' as any, + data: { + name: 'NumericTest', + numericObject: { + '0': 0.1, + '1': 0.2, + '2': 0.3, + '100': 1.0 + } + } + }) + + await new Promise(resolve => setTimeout(resolve, 100)) + + const systemDir = join(testDir, '_system') + + if (!existsSync(systemDir)) { + expect(true).toBe(true) + return + } + + const chunkFiles = readdirSync(systemDir).filter(f => f.startsWith('__chunk__')) + + // CRITICAL: Check that NO chunk files have numeric field names + // This is the v3.50.2 fix - prevents vectors-as-objects from being indexed + for (const file of chunkFiles) { + const content = JSON.parse(readFileSync(join(systemDir, file), 'utf-8')) + const fieldName = content.field + + // Should NOT index purely numeric field names (array indices as object keys) + // This catches: "0", "1", "2", "100", "54716", "100000", etc. + expect(fieldName).not.toMatch(/^\d+$/) + } + + // Verify we DID create chunk files for legitimate fields + expect(chunkFiles.length).toBeGreaterThan(0) + }) + it('should still index small arrays (tags, categories)', async () => { // Add entity with tags await brainy.add({