fix(hnsw): entry point recovery prevents import failures and log spam

Fixes critical production bug where HNSW index operations would fail
and spam thousands of console.error messages during large imports.

Root cause: rebuild() nullifies entryPointId via clear(), and if
getHNSWSystem() returns null (missing/corrupted system data), the
entry point was never recovered from loaded nouns.

Changes:
- Add entry point recovery in rebuild() after loading nouns
- Add entry point recovery in search() for corrupted state
- Add entry point recovery in search() when entry point noun deleted
- Remove console.error spam in search() and addItem()
- Standardize 404 error detection in GCS/S3/Azure storage adapters

Tested: All entry point scenarios now recover gracefully without
log spam. Empty index returns empty results silently.
This commit is contained in:
David Snelling 2025-11-25 14:34:19 -08:00
parent 0c90eaa8cf
commit 52eae67cb8
4 changed files with 126 additions and 13 deletions

View file

@ -268,9 +268,8 @@ export class HNSWIndex {
// Find entry point // Find entry point
if (!this.entryPointId) { if (!this.entryPointId) {
console.error('Entry point ID is null') // No entry point but nouns exist - corrupted state, recover by using this item
// If there's no entry point, this is the first noun, so we should have returned earlier // This shouldn't normally happen as first item sets entry point above
// This is a safety check
this.entryPointId = id this.entryPointId = id
this.maxLevel = nounLevel this.maxLevel = nounLevel
this.nouns.set(id, noun) this.nouns.set(id, noun)
@ -279,7 +278,7 @@ export class HNSWIndex {
const entryPoint = this.nouns.get(this.entryPointId) const entryPoint = this.nouns.get(this.entryPointId)
if (!entryPoint) { if (!entryPoint) {
console.error(`Entry point with ID ${this.entryPointId} not found`) // Entry point was deleted but ID not updated - recover by using new item
// If the entry point doesn't exist, treat this as the first noun // If the entry point doesn't exist, treat this as the first noun
this.entryPointId = id this.entryPointId = id
this.maxLevel = nounLevel this.maxLevel = nounLevel
@ -515,15 +514,51 @@ export class HNSWIndex {
} }
// Start from the entry point // Start from the entry point
// If entry point is null but nouns exist, attempt recovery (v6.2.2)
if (!this.entryPointId && this.nouns.size > 0) {
// Corrupted state: nouns exist but entry point is null - recover
let maxLevel = 0
let recoveredId: string | null = null
for (const [id, noun] of this.nouns.entries()) {
if (noun.level >= maxLevel) {
maxLevel = noun.level
recoveredId = id
}
}
if (recoveredId) {
this.entryPointId = recoveredId
this.maxLevel = maxLevel
}
}
if (!this.entryPointId) { if (!this.entryPointId) {
console.error('Entry point ID is null') // Truly empty index - return empty results silently
return [] return []
} }
const entryPoint = this.nouns.get(this.entryPointId) let entryPoint = this.nouns.get(this.entryPointId)
if (!entryPoint) { if (!entryPoint) {
console.error(`Entry point with ID ${this.entryPointId} not found`) // Entry point ID exists but noun was deleted - attempt recovery
return [] if (this.nouns.size > 0) {
let maxLevel = 0
let recoveredId: string | null = null
for (const [id, noun] of this.nouns.entries()) {
if (noun.level >= maxLevel) {
maxLevel = noun.level
recoveredId = id
}
}
if (recoveredId) {
this.entryPointId = recoveredId
this.maxLevel = maxLevel
entryPoint = this.nouns.get(recoveredId)
}
}
// If still no entry point, return empty
if (!entryPoint) {
return []
}
} }
let currObj = entryPoint let currObj = entryPoint
@ -1165,6 +1200,63 @@ export class HNSWIndex {
} }
} }
// Step 5: CRITICAL - Recover entry point if missing (v6.2.2)
// This ensures consistency even if getHNSWSystem() returned null
if (this.nouns.size > 0 && this.entryPointId === null) {
prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering from loaded data')
let maxLevel = 0
let recoveredEntryPointId: string | null = null
for (const [id, noun] of this.nouns.entries()) {
if (noun.level >= maxLevel) {
maxLevel = noun.level
recoveredEntryPointId = id
}
}
this.entryPointId = recoveredEntryPointId
this.maxLevel = maxLevel
prodLog.info(`HNSW entry point recovered: ${recoveredEntryPointId} at level ${maxLevel}`)
// Persist recovered state to prevent future recovery
if (this.storage && recoveredEntryPointId) {
await this.storage.saveHNSWSystem({
entryPointId: this.entryPointId,
maxLevel: this.maxLevel
}).catch((error) => {
prodLog.error('Failed to persist recovered HNSW system data:', error)
})
}
}
// Step 6: Validate entry point exists if set (handles stale/deleted entry point)
if (this.entryPointId && !this.nouns.has(this.entryPointId)) {
prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering`)
let maxLevel = 0
let recoveredId: string | null = null
for (const [id, noun] of this.nouns.entries()) {
if (noun.level >= maxLevel) {
maxLevel = noun.level
recoveredId = id
}
}
this.entryPointId = recoveredId
this.maxLevel = maxLevel
// Persist corrected state
if (this.storage && recoveredId) {
await this.storage.saveHNSWSystem({
entryPointId: this.entryPointId,
maxLevel: this.maxLevel
}).catch((error) => {
prodLog.error('Failed to persist corrected HNSW system data:', error)
})
}
}
const cacheInfo = shouldPreload const cacheInfo = shouldPreload
? ` (vectors preloaded)` ? ` (vectors preloaded)`
: ` (adaptive caching - vectors loaded on-demand)` : ` (adaptive caching - vectors loaded on-demand)`

View file

@ -1629,7 +1629,16 @@ export class AzureBlobStorage extends BaseStorage {
return JSON.parse(downloaded.toString()) return JSON.parse(downloaded.toString())
} catch (error: any) { } catch (error: any) {
if (error.statusCode === 404 || error.code === 'BlobNotFound') { // Azure may return not found errors in different formats
const isNotFound =
error.statusCode === 404 ||
error.code === 'BlobNotFound' ||
error.code === 404 ||
error.details?.code === 'BlobNotFound' ||
error.message?.includes('BlobNotFound') ||
error.message?.includes('not found') ||
error.message?.includes('404')
if (isNotFound) {
return null return null
} }

View file

@ -1567,7 +1567,15 @@ export class GcsStorage extends BaseStorage {
return JSON.parse(contents.toString()) return JSON.parse(contents.toString())
} catch (error: any) { } catch (error: any) {
if (error.code === 404) { // GCS may return 404 in different formats depending on SDK version
const is404 =
error.code === 404 ||
error.statusCode === 404 ||
error.status === 404 ||
error.message?.includes('No such object') ||
error.message?.includes('not found') ||
error.message?.includes('404')
if (is404) {
return null return null
} }

View file

@ -3628,11 +3628,15 @@ export class S3CompatibleStorage extends BaseStorage {
const bodyContents = await response.Body.transformToString() const bodyContents = await response.Body.transformToString()
return JSON.parse(bodyContents) return JSON.parse(bodyContents)
} catch (error: any) { } catch (error: any) {
if ( // S3 may return not found errors in different formats
const isNotFound =
error.name === 'NoSuchKey' || error.name === 'NoSuchKey' ||
error.code === 'NoSuchKey' ||
error.$metadata?.httpStatusCode === 404 ||
error.message?.includes('NoSuchKey') || error.message?.includes('NoSuchKey') ||
error.message?.includes('not found') error.message?.includes('not found') ||
) { error.message?.includes('404')
if (isNotFound) {
return null return null
} }