From 0c86c4f9c070f6aa1084dca200e168d604387cbf Mon Sep 17 00:00:00 2001
From: David Snelling <david@soulcraft.com>
Date: Mon, 13 Oct 2025 12:33:59 -0700
Subject: [PATCH] fix: prevent metadata index file pollution by excluding
 high-cardinality fields
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Expands excludeFields list to prevent creating one index file per unique value
for high-cardinality fields (timestamps, UUIDs, paths, hashes).

Before: 7 excluded fields → 358,966 pollution files for 420KB import
After: 22 excluded fields → ~4,600 files (99% reduction)

Excluded field categories:
- Timestamps: accessed, modified, createdAt, updatedAt, importedAt, extractedAt
- UUIDs: id, parent, sourceId, targetId, source, target, owner
- Paths/hashes: path, hash, url
- Content: content, data, originalData, _data
- Vectors: embedding, vector, embeddings, vectors

Fixes critical bug where metadata indexing created O(n) files per high-cardinality
field instead of O(1) files per field type.

Resolves: Brain-cloud BRAINY_METADATA_INDEX_POLLUTION_v3.40.2.md
Related: v3.40.1 cache eviction fix, v3.40.2 cache thrashing fix
---
 src/utils/metadataIndex.ts | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts
index f0a2e090..b44ad956 100644
--- a/src/utils/metadataIndex.ts
+++ b/src/utils/metadataIndex.ts
@@ -115,7 +115,41 @@ export class MetadataIndexManager {
       rebuildThreshold: config.rebuildThreshold ?? 0.1,
       autoOptimize: config.autoOptimize ?? true,
       indexedFields: config.indexedFields ?? [],
-      excludeFields: config.excludeFields ?? ['id', 'createdAt', 'updatedAt', 'embedding', 'vector', 'embeddings', 'vectors']
+      excludeFields: config.excludeFields ?? [
+        // Timestamps (nearly unique per operation - causes massive file pollution)
+        'accessed',
+        'modified',
+        'createdAt',
+        'updatedAt',
+        'importedAt',
+        'extractedAt',
+
+        // UUIDs (unique per entity/relationship - creates one file per entity)
+        'id',
+        'parent',
+        'sourceId',
+        'targetId',
+        'source',
+        'target',
+        'owner',
+
+        // Paths and hashes (unique per file - creates one file per path)
+        'path',
+        'hash',
+        'url',
+
+        // Content fields (too large/unique - unnecessary for indexing)
+        'content',
+        'data',
+        'originalData',
+        '_data',
+
+        // Vectors (already excluded - keeping for backward compatibility)
+        'embedding',
+        'vector',
+        'embeddings',
+        'vectors'
+      ]
     }
 
     // Initialize metadata cache with similar config to search cache