fix(hnsw): entry point recovery prevents import failures and log spam
Fixes critical production bug where HNSW index operations would fail and spam thousands of console.error messages during large imports. Root cause: rebuild() nullifies entryPointId via clear(), and if getHNSWSystem() returns null (missing/corrupted system data), the entry point was never recovered from loaded nouns. Changes: - Add entry point recovery in rebuild() after loading nouns - Add entry point recovery in search() for corrupted state - Add entry point recovery in search() when entry point noun deleted - Remove console.error spam in search() and addItem() - Standardize 404 error detection in GCS/S3/Azure storage adapters Tested: All entry point scenarios now recover gracefully without log spam. Empty index returns empty results silently.
This commit is contained in:
parent
0c90eaa8cf
commit
52eae67cb8
4 changed files with 126 additions and 13 deletions
|
|
@ -268,9 +268,8 @@ export class HNSWIndex {
|
|||
|
||||
// Find entry point
|
||||
if (!this.entryPointId) {
|
||||
console.error('Entry point ID is null')
|
||||
// If there's no entry point, this is the first noun, so we should have returned earlier
|
||||
// This is a safety check
|
||||
// No entry point but nouns exist - corrupted state, recover by using this item
|
||||
// This shouldn't normally happen as first item sets entry point above
|
||||
this.entryPointId = id
|
||||
this.maxLevel = nounLevel
|
||||
this.nouns.set(id, noun)
|
||||
|
|
@ -279,7 +278,7 @@ export class HNSWIndex {
|
|||
|
||||
const entryPoint = this.nouns.get(this.entryPointId)
|
||||
if (!entryPoint) {
|
||||
console.error(`Entry point with ID ${this.entryPointId} not found`)
|
||||
// Entry point was deleted but ID not updated - recover by using new item
|
||||
// If the entry point doesn't exist, treat this as the first noun
|
||||
this.entryPointId = id
|
||||
this.maxLevel = nounLevel
|
||||
|
|
@ -515,16 +514,52 @@ export class HNSWIndex {
|
|||
}
|
||||
|
||||
// Start from the entry point
|
||||
// If entry point is null but nouns exist, attempt recovery (v6.2.2)
|
||||
if (!this.entryPointId && this.nouns.size > 0) {
|
||||
// Corrupted state: nouns exist but entry point is null - recover
|
||||
let maxLevel = 0
|
||||
let recoveredId: string | null = null
|
||||
for (const [id, noun] of this.nouns.entries()) {
|
||||
if (noun.level >= maxLevel) {
|
||||
maxLevel = noun.level
|
||||
recoveredId = id
|
||||
}
|
||||
}
|
||||
if (recoveredId) {
|
||||
this.entryPointId = recoveredId
|
||||
this.maxLevel = maxLevel
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.entryPointId) {
|
||||
console.error('Entry point ID is null')
|
||||
// Truly empty index - return empty results silently
|
||||
return []
|
||||
}
|
||||
|
||||
const entryPoint = this.nouns.get(this.entryPointId)
|
||||
let entryPoint = this.nouns.get(this.entryPointId)
|
||||
if (!entryPoint) {
|
||||
// Entry point ID exists but noun was deleted - attempt recovery
|
||||
if (this.nouns.size > 0) {
|
||||
let maxLevel = 0
|
||||
let recoveredId: string | null = null
|
||||
for (const [id, noun] of this.nouns.entries()) {
|
||||
if (noun.level >= maxLevel) {
|
||||
maxLevel = noun.level
|
||||
recoveredId = id
|
||||
}
|
||||
}
|
||||
if (recoveredId) {
|
||||
this.entryPointId = recoveredId
|
||||
this.maxLevel = maxLevel
|
||||
entryPoint = this.nouns.get(recoveredId)
|
||||
}
|
||||
}
|
||||
|
||||
// If still no entry point, return empty
|
||||
if (!entryPoint) {
|
||||
console.error(`Entry point with ID ${this.entryPointId} not found`)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
let currObj = entryPoint
|
||||
|
||||
|
|
@ -1165,6 +1200,63 @@ export class HNSWIndex {
|
|||
}
|
||||
}
|
||||
|
||||
// Step 5: CRITICAL - Recover entry point if missing (v6.2.2)
|
||||
// This ensures consistency even if getHNSWSystem() returned null
|
||||
if (this.nouns.size > 0 && this.entryPointId === null) {
|
||||
prodLog.warn('HNSW rebuild: Entry point was null after loading nouns - recovering from loaded data')
|
||||
|
||||
let maxLevel = 0
|
||||
let recoveredEntryPointId: string | null = null
|
||||
|
||||
for (const [id, noun] of this.nouns.entries()) {
|
||||
if (noun.level >= maxLevel) {
|
||||
maxLevel = noun.level
|
||||
recoveredEntryPointId = id
|
||||
}
|
||||
}
|
||||
|
||||
this.entryPointId = recoveredEntryPointId
|
||||
this.maxLevel = maxLevel
|
||||
|
||||
prodLog.info(`HNSW entry point recovered: ${recoveredEntryPointId} at level ${maxLevel}`)
|
||||
|
||||
// Persist recovered state to prevent future recovery
|
||||
if (this.storage && recoveredEntryPointId) {
|
||||
await this.storage.saveHNSWSystem({
|
||||
entryPointId: this.entryPointId,
|
||||
maxLevel: this.maxLevel
|
||||
}).catch((error) => {
|
||||
prodLog.error('Failed to persist recovered HNSW system data:', error)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Step 6: Validate entry point exists if set (handles stale/deleted entry point)
|
||||
if (this.entryPointId && !this.nouns.has(this.entryPointId)) {
|
||||
prodLog.warn(`HNSW: Entry point ${this.entryPointId} not found in loaded nouns - recovering`)
|
||||
|
||||
let maxLevel = 0
|
||||
let recoveredId: string | null = null
|
||||
for (const [id, noun] of this.nouns.entries()) {
|
||||
if (noun.level >= maxLevel) {
|
||||
maxLevel = noun.level
|
||||
recoveredId = id
|
||||
}
|
||||
}
|
||||
this.entryPointId = recoveredId
|
||||
this.maxLevel = maxLevel
|
||||
|
||||
// Persist corrected state
|
||||
if (this.storage && recoveredId) {
|
||||
await this.storage.saveHNSWSystem({
|
||||
entryPointId: this.entryPointId,
|
||||
maxLevel: this.maxLevel
|
||||
}).catch((error) => {
|
||||
prodLog.error('Failed to persist corrected HNSW system data:', error)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const cacheInfo = shouldPreload
|
||||
? ` (vectors preloaded)`
|
||||
: ` (adaptive caching - vectors loaded on-demand)`
|
||||
|
|
|
|||
|
|
@ -1629,7 +1629,16 @@ export class AzureBlobStorage extends BaseStorage {
|
|||
|
||||
return JSON.parse(downloaded.toString())
|
||||
} catch (error: any) {
|
||||
if (error.statusCode === 404 || error.code === 'BlobNotFound') {
|
||||
// Azure may return not found errors in different formats
|
||||
const isNotFound =
|
||||
error.statusCode === 404 ||
|
||||
error.code === 'BlobNotFound' ||
|
||||
error.code === 404 ||
|
||||
error.details?.code === 'BlobNotFound' ||
|
||||
error.message?.includes('BlobNotFound') ||
|
||||
error.message?.includes('not found') ||
|
||||
error.message?.includes('404')
|
||||
if (isNotFound) {
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1567,7 +1567,15 @@ export class GcsStorage extends BaseStorage {
|
|||
|
||||
return JSON.parse(contents.toString())
|
||||
} catch (error: any) {
|
||||
if (error.code === 404) {
|
||||
// GCS may return 404 in different formats depending on SDK version
|
||||
const is404 =
|
||||
error.code === 404 ||
|
||||
error.statusCode === 404 ||
|
||||
error.status === 404 ||
|
||||
error.message?.includes('No such object') ||
|
||||
error.message?.includes('not found') ||
|
||||
error.message?.includes('404')
|
||||
if (is404) {
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3628,11 +3628,15 @@ export class S3CompatibleStorage extends BaseStorage {
|
|||
const bodyContents = await response.Body.transformToString()
|
||||
return JSON.parse(bodyContents)
|
||||
} catch (error: any) {
|
||||
if (
|
||||
// S3 may return not found errors in different formats
|
||||
const isNotFound =
|
||||
error.name === 'NoSuchKey' ||
|
||||
error.code === 'NoSuchKey' ||
|
||||
error.$metadata?.httpStatusCode === 404 ||
|
||||
error.message?.includes('NoSuchKey') ||
|
||||
error.message?.includes('not found')
|
||||
) {
|
||||
error.message?.includes('not found') ||
|
||||
error.message?.includes('404')
|
||||
if (isNotFound) {
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue