soulcraft/brainy
2
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix: exclude __words__ keyword index from corruption detection and getStats()

Browse source 
The __words__ keyword index stores 50-5000 entries per entity (one per
word), which inflated avg entries/entity well above the corruption
threshold of 100. This caused:

1. validateConsistency() to falsely detect corruption on every startup,
   triggering unnecessary clearAllIndexData() + rebuild() cycles
2. getStats() to log false "Metadata index may be corrupted" warnings
   and report inflated totalEntries/totalIds stats

Both methods now skip __words__ when counting, so stats and health
checks reflect metadata fields only (noun, type, createdAt, etc.).
Keyword search is unaffected since the __words__ field index itself
is not modified.
This commit is contained in:
David Snelling 2026-01-27 15:38:21 -08:00
parent 32dbdcec61
commit 364360d447
128 changed files with 5637 additions and 5682 deletions
Download patch file Download diff file Expand all files Collapse all files

12
src/api/ConfigAPI.ts
View file

@ -60,7 +60,7 @@ export class ConfigAPI {
// Store in cache
this.configCache.set(key, entry)
// v4.0.0: Persist to storage as NounMetadata
// Persist to storage as NounMetadata
const configId = this.CONFIG_NOUN_PREFIX + key
const nounMetadata = {
noun: 'config' as any,
@ -94,7 +94,7 @@ export class ConfigAPI {
return defaultValue
}
// v4.0.0: Extract ConfigEntry from NounMetadata
// Extract ConfigEntry from NounMetadata
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
: ((metadata.createdAt as unknown as number) || Date.now())
@ -140,7 +140,7 @@ export class ConfigAPI {
// Remove from cache
this.configCache.delete(key)
// v4.0.0: Remove from storage
// Remove from storage
const configId = this.CONFIG_NOUN_PREFIX + key
await this.storage.deleteNounMetadata(configId)
}
@ -149,7 +149,7 @@ export class ConfigAPI {
* List all configuration keys
*/
async list(): Promise<string[]> {
// v4.0.0: Get all nouns and filter for config entries
// Get all nouns and filter for config entries
const result = await this.storage.getNouns({ pagination: { limit: 10000 } })
const configKeys: string[] = []
@ -213,7 +213,7 @@ export class ConfigAPI {
for (const [key, entry] of Object.entries(config)) {
this.configCache.set(key, entry)
const configId = this.CONFIG_NOUN_PREFIX + key
// v4.0.0: Convert ConfigEntry to NounMetadata
// Convert ConfigEntry to NounMetadata
const nounMetadata = {
noun: 'config' as any,
...entry,
@ -240,7 +240,7 @@ export class ConfigAPI {
return null
}
// v4.0.0: Extract ConfigEntry from NounMetadata
// Extract ConfigEntry from NounMetadata
const createdAtVal = typeof metadata.createdAt === 'object' && metadata.createdAt !== null && 'seconds' in metadata.createdAt
? metadata.createdAt.seconds * 1000 + Math.floor((metadata.createdAt.nanoseconds || 0) / 1000000)
: ((metadata.createdAt as unknown as number) || Date.now())

2
src/api/DataAPI.ts
View file

@ -210,7 +210,7 @@ export class DataAPI {
this.validateImportItem(mapped)
}
// v4.0.0: Save entity - separate vector and metadata
// Save entity - separate vector and metadata
const id = mapped.id || this.generateId()
const noun: HNSWNoun = {
id,