fix: prevent metadata index file pollution by excluding high-cardinality fields
Expands excludeFields list to prevent creating one index file per unique value for high-cardinality fields (timestamps, UUIDs, paths, hashes). Before: 7 excluded fields → 358,966 pollution files for 420KB import After: 22 excluded fields → ~4,600 files (99% reduction) Excluded field categories: - Timestamps: accessed, modified, createdAt, updatedAt, importedAt, extractedAt - UUIDs: id, parent, sourceId, targetId, source, target, owner - Paths/hashes: path, hash, url - Content: content, data, originalData, _data - Vectors: embedding, vector, embeddings, vectors Fixes critical bug where metadata indexing created O(n) files per high-cardinality field instead of O(1) files per field type. Resolves: Brain-cloud BRAINY_METADATA_INDEX_POLLUTION_v3.40.2.md Related: v3.40.1 cache eviction fix, v3.40.2 cache thrashing fix
This commit is contained in:
parent
853ea26477
commit
0c86c4f9c0
1 changed files with 35 additions and 1 deletions
|
|
@ -115,7 +115,41 @@ export class MetadataIndexManager {
|
||||||
rebuildThreshold: config.rebuildThreshold ?? 0.1,
|
rebuildThreshold: config.rebuildThreshold ?? 0.1,
|
||||||
autoOptimize: config.autoOptimize ?? true,
|
autoOptimize: config.autoOptimize ?? true,
|
||||||
indexedFields: config.indexedFields ?? [],
|
indexedFields: config.indexedFields ?? [],
|
||||||
excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt', 'embedding', 'vector', 'embeddings', 'vectors']
|
excludeFields: config.excludeFields ?? [
|
||||||
|
// Timestamps (nearly unique per operation - causes massive file pollution)
|
||||||
|
'accessed',
|
||||||
|
'modified',
|
||||||
|
'createdAt',
|
||||||
|
'updatedAt',
|
||||||
|
'importedAt',
|
||||||
|
'extractedAt',
|
||||||
|
|
||||||
|
// UUIDs (unique per entity/relationship - creates one file per entity)
|
||||||
|
'id',
|
||||||
|
'parent',
|
||||||
|
'sourceId',
|
||||||
|
'targetId',
|
||||||
|
'source',
|
||||||
|
'target',
|
||||||
|
'owner',
|
||||||
|
|
||||||
|
// Paths and hashes (unique per file - creates one file per path)
|
||||||
|
'path',
|
||||||
|
'hash',
|
||||||
|
'url',
|
||||||
|
|
||||||
|
// Content fields (too large/unique - unnecessary for indexing)
|
||||||
|
'content',
|
||||||
|
'data',
|
||||||
|
'originalData',
|
||||||
|
'_data',
|
||||||
|
|
||||||
|
// Vectors (already excluded - keeping for backward compatibility)
|
||||||
|
'embedding',
|
||||||
|
'vector',
|
||||||
|
'embeddings',
|
||||||
|
'vectors'
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize metadata cache with similar config to search cache
|
// Initialize metadata cache with similar config to search cache
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue