soulcraft/brainy
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases 335 Packages Wiki Activity Actions

fix(hnsw): entry point recovery prevents import failures and log spam

Browse source 
Fixes critical production bug where HNSW index operations would fail
and spam thousands of console.error messages during large imports.

Root cause: rebuild() nullifies entryPointId via clear(), and if
getHNSWSystem() returns null (missing/corrupted system data), the
entry point was never recovered from loaded nouns.

Changes:
- Add entry point recovery in rebuild() after loading nouns
- Add entry point recovery in search() for corrupted state
- Add entry point recovery in search() when entry point noun deleted
- Remove console.error spam in search() and addItem()
- Standardize 404 error detection in GCS/S3/Azure storage adapters

Tested: All entry point scenarios now recover gracefully without
log spam. Empty index returns empty results silently.
This commit is contained in:
David Snelling 2025-11-25 14:34:19 -08:00
parent 0c90eaa8cf
commit 52eae67cb8
4 changed files with 126 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

108
src/hnsw/hnswIndex.ts
View file

@ -268,9 +268,8 @@ export class HNSWIndex {
// Find entry point
if (!this.entryPointId) {
console.error('Entry point ID is null')
// If there's no entry point, this is the first noun, so we should have returned earlier
// This is a safety check
// No entry point but nouns exist - corrupted state, recover by using this item
// This shouldn't normally happen as first item sets entry point above
this.entryPointId = id
this.maxLevel = nounLevel
this.nouns.set(id, noun)
@ -279,7 +278,7 @@ export class HNSWIndex {
const entryPoint = this.nouns.get(this.entryPointId)
if (!entryPoint) {
console.error(`Entry point with ID ${this.entryPointId} not found`)
// Entry point was deleted but ID not updated - recover by using new item
// If the entry point doesn't exist, treat this as the first noun
this.entryPointId = id
this.maxLevel = nounLevel
@ -515,15 +514,51 @@ export class HNSWIndex {
}
// Start from the entry point
// If entry point is null but nouns exist, attempt recovery (v6.2.2)
if (!this.entryPointId && this.nouns.size > 0) {
// Corrupted state: nouns exist but entry point is null - recover
let maxLevel = 0
let recoveredId: string | null = null
for (const [id, noun] of this.nouns.entries()) {
if (noun.level >= maxLevel) {
maxLevel = noun.level
recoveredId = id
}
}
if (recoveredId) {
this.entryPointId = recoveredId
this.maxLevel = maxLevel
}
}
if (!this.entryPointId) {
console.error('Entry point ID is null')
// Truly empty index - return empty results silently
return []
}
const entryPoint = this.nouns.get(this.entryPointId)
let entryPoint = this.nouns.get(this.entryPointId)
if (!entryPoint) {
console.error(`Entry point with ID ${this.entryPointId} not found`)
return []
// Entry point ID exists but noun was deleted - attempt recovery
if (this.nouns.size > 0) {
let maxLevel = 0
let recoveredId: string | null = null
for (const [id, noun] of this.nouns.entries()) {
if (noun.level >= maxLevel) {
maxLevel = noun.level
recoveredId = id
}
}
if (recoveredId) {
this.entryPointId = recoveredId
this.maxLevel = maxLevel
entryPoint = this.nouns.get(recoveredId)
}
}
// If still no entry point, return empty
if (!entryPoint) {
return []
}
}
let currObj = entryPoint
@ -1165,6 +1200,63 @@ export class HNSWIndex {
}
}
// Step 5: CRITICAL - Recover entry point if missing (v6.2.2)
// This ensures consistency even if getHNSWSystem() returned null
if (this.nouns.size > 0 && this.entryPointId === null) {
prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering from loaded data')
let maxLevel = 0
let recoveredEntryPointId: string | null = null
for (const [id, noun] of this.nouns.entries()) {
if (noun.level >= maxLevel) {
maxLevel = noun.level
recoveredEntryPointId = id
}
}
this.entryPointId = recoveredEntryPointId
this.maxLevel = maxLevel
prodLog.info(`HNSW entry point recovered: ${recoveredEntryPointId} at level ${maxLevel}`)
// Persist recovered state to prevent future recovery
if (this.storage && recoveredEntryPointId) {
await this.storage.saveHNSWSystem({
entryPointId: this.entryPointId,
maxLevel: this.maxLevel
}).catch((error) => {
prodLog.error('Failed to persist recovered HNSW system data:', error)
})
}
}
// Step 6: Validate entry point exists if set (handles stale/deleted entry point)
if (this.entryPointId && !this.nouns.has(this.entryPointId)) {
prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering`)
let maxLevel = 0
let recoveredId: string | null = null
for (const [id, noun] of this.nouns.entries()) {
if (noun.level >= maxLevel) {
maxLevel = noun.level
recoveredId = id
}
}
this.entryPointId = recoveredId
this.maxLevel = maxLevel
// Persist corrected state
if (this.storage && recoveredId) {
await this.storage.saveHNSWSystem({
entryPointId: this.entryPointId,
maxLevel: this.maxLevel
}).catch((error) => {
prodLog.error('Failed to persist corrected HNSW system data:', error)
})
}
}
const cacheInfo = shouldPreload
? ` (vectors preloaded)`
: ` (adaptive caching - vectors loaded on-demand)`

11
src/storage/adapters/azureBlobStorage.ts
View file

@ -1629,7 +1629,16 @@ export class AzureBlobStorage extends BaseStorage {
return JSON.parse(downloaded.toString())
} catch (error: any) {
if (error.statusCode === 404 || error.code === 'BlobNotFound') {
// Azure may return not found errors in different formats
const isNotFound =
error.statusCode === 404 ||
error.code === 'BlobNotFound' ||
error.code === 404 ||
error.details?.code === 'BlobNotFound' ||
error.message?.includes('BlobNotFound') ||
error.message?.includes('not found') ||
error.message?.includes('404')
if (isNotFound) {
return null
}

10
src/storage/adapters/gcsStorage.ts
View file

@ -1567,7 +1567,15 @@ export class GcsStorage extends BaseStorage {
return JSON.parse(contents.toString())
} catch (error: any) {
if (error.code === 404) {
// GCS may return 404 in different formats depending on SDK version
const is404 =
error.code === 404 ||
error.statusCode === 404 ||
error.status === 404 ||
error.message?.includes('No such object') ||
error.message?.includes('not found') ||
error.message?.includes('404')
if (is404) {
return null
}

10
src/storage/adapters/s3CompatibleStorage.ts
View file

@ -3628,11 +3628,15 @@ export class S3CompatibleStorage extends BaseStorage {
const bodyContents = await response.Body.transformToString()
return JSON.parse(bodyContents)
} catch (error: any) {
if (
// S3 may return not found errors in different formats
const isNotFound =
error.name === 'NoSuchKey' ||
error.code === 'NoSuchKey' ||
error.$metadata?.httpStatusCode === 404 ||
error.message?.includes('NoSuchKey') ||
error.message?.includes('not found')
) {
error.message?.includes('not found') ||
error.message?.includes('404')
if (isNotFound) {
return null
}