fix: exclude __words__ keyword index from corruption detection and getStats()

The __words__ keyword index stores 50-5000 entries per entity (one per
word), which inflated avg entries/entity well above the corruption
threshold of 100. This caused:

1. validateConsistency() to falsely detect corruption on every startup,
   triggering unnecessary clearAllIndexData() + rebuild() cycles
2. getStats() to log false "Metadata index may be corrupted" warnings
   and report inflated totalEntries/totalIds stats

Both methods now skip __words__ when counting, so stats and health
checks reflect metadata fields only (noun, type, createdAt, etc.).
Keyword search is unaffected since the __words__ field index itself
is not modified.
This commit is contained in:
David Snelling 2026-01-27 15:38:21 -08:00
parent 32dbdcec61
commit 364360d447
128 changed files with 5637 additions and 5682 deletions

View file

@ -172,7 +172,7 @@ export const coreCommands = {
metadata
}
// v4.3.x: Add confidence and weight if provided
// Add confidence and weight if provided
if (options.confidence) {
addParams.confidence = parseFloat(options.confidence)
}
@ -347,7 +347,7 @@ export const coreCommands = {
searchParams.includeRelations = true
}
// v4.7.0: VFS is now part of the knowledge graph (included by default)
// VFS is now part of the knowledge graph (included by default)
// Users can exclude VFS with --where vfsType exists:false if needed
// Triple Intelligence Fusion - custom weighting

View file

@ -312,7 +312,7 @@ ${chalk.cyan('Fork Statistics:')}
},
/**
* Migrate from v4.x to v5.0.0 (one-time)
* Migrate storage format (one-time)
*/
async migrate(options: MigrateOptions) {
let spinner: any = null
@ -388,7 +388,7 @@ ${chalk.cyan('Fork Statistics:')}
await oldBrain.init()
// Create new brain (v5.0.0)
// Create new brain
const newBrain = new Brainy({
storage: {
type: 'filesystem',

View file

@ -117,7 +117,7 @@ export const neuralCommand = {
await handleNeighborsCommand(neural, argv)
break
case 'path':
console.log(chalk.yellow('\n⚠ Semantic path finding coming in v3.21.0'))
console.log(chalk.yellow('\n⚠ Semantic path finding coming soon'))
console.log(chalk.dim('This feature requires implementing graph traversal algorithms'))
console.log(chalk.dim('Use "neighbors" and "hierarchy" commands to explore connections'))
break
@ -148,7 +148,7 @@ async function promptForAction(): Promise<string> {
{ name: '🎯 Find semantic clusters', value: 'clusters' },
{ name: '🌳 Show item hierarchy', value: 'hierarchy' },
{ name: '🕸️ Find semantic neighbors', value: 'neighbors' },
{ name: '🛣️ Find semantic path between items (v3.21.0)', value: 'path', disabled: true },
{ name: '🛣️ Find semantic path between items (coming soon)', value: 'path', disabled: true },
{ name: '🚨 Detect outliers', value: 'outliers' },
{ name: '📊 Generate visualization data', value: 'visualize' }
]

View file

@ -1,5 +1,5 @@
/**
* 💾 Storage Management Commands - v4.0.0
* 💾 Storage Management Commands
*
* Modern interactive CLI for storage lifecycle, cost optimization, and management
*/

View file

@ -133,7 +133,7 @@ export const utilityCommands = {
// removeOrphans and rebuildIndex would require new Brainy APIs
if (options.removeOrphans || options.rebuildIndex) {
spinner.warn('Advanced cleanup options not yet implemented')
console.log(chalk.yellow('\n⚠ Advanced cleanup features coming in v3.21.0:'))
console.log(chalk.yellow('\n⚠ Advanced cleanup features coming soon:'))
console.log(chalk.dim(' • --remove-orphans: Remove disconnected items'))
console.log(chalk.dim(' • --rebuild-index: Rebuild vector index'))
console.log(chalk.dim('\nUse "brainy clean" without options to clear the database'))

View file

@ -21,7 +21,7 @@ let brainyInstance: Brainy | null = null
const getBrainy = async (): Promise<Brainy> => {
if (!brainyInstance) {
brainyInstance = new Brainy()
await brainyInstance.init() // v5.0.1: Initialize brain (VFS auto-initialized here!)
await brainyInstance.init() // Initialize brain (VFS auto-initialized here!)
}
return brainyInstance
}
@ -52,8 +52,8 @@ export const vfsCommands = {
const spinner = ora('Reading file...').start()
try {
const brain = await getBrainy() // v5.0.1: Await async getBrainy
// v5.0.1: VFS auto-initialized, no need for vfs.init()
const brain = await getBrainy() // Await async getBrainy
// VFS auto-initialized, no need for vfs.init()
const buffer = await brain.vfs.readFile(path, {
encoding: options.encoding as any
})