fix: exclude __words__ keyword index from corruption detection and getStats()

The __words__ keyword index stores 50-5000 entries per entity (one per
word), which inflated avg entries/entity well above the corruption
threshold of 100. This caused:

1. validateConsistency() to falsely detect corruption on every startup,
   triggering unnecessary clearAllIndexData() + rebuild() cycles
2. getStats() to log false "Metadata index may be corrupted" warnings
   and report inflated totalEntries/totalIds stats

Both methods now skip __words__ when counting, so stats and health
checks reflect metadata fields only (noun, type, createdAt, etc.).
Keyword search is unaffected since the __words__ field index itself
is not modified.
This commit is contained in:
David Snelling 2026-01-27 15:38:21 -08:00
parent 32dbdcec61
commit 364360d447
128 changed files with 5637 additions and 5682 deletions

View file

@ -111,7 +111,7 @@ export class FormatDetector {
return 'docx'
}
// Images (v5.2.0: ImageHandler support)
// Images (ImageHandler support)
if (mimeType.startsWith('image/')) {
return 'image'
}
@ -134,7 +134,7 @@ export class FormatDetector {
}
}
// YAML detection (v4.2.0)
// YAML detection
if (this.looksLikeYAML(trimmed)) {
return {
format: 'yaml',
@ -206,7 +206,7 @@ export class FormatDetector {
}
}
// Image formats (v5.2.0)
// Image formats
// JPEG: FF D8 FF
if (buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF) {
return {
@ -384,7 +384,7 @@ export class FormatDetector {
/**
* Check if content looks like YAML
* v4.2.0: Added YAML detection
* Added YAML detection
*/
private looksLikeYAML(content: string): boolean {
const lines = content.split('\n').filter(l => l.trim()).slice(0, 20)