Merge branch 'next/zero-norm-unvector-door'
# Conflicts: # src/hnsw/hnswIndex.ts
This commit is contained in:
commit
9b84ef5b02
6 changed files with 798 additions and 54 deletions
|
|
@ -10,7 +10,7 @@ import {
|
|||
Vector,
|
||||
VectorDocument
|
||||
} from '../coreTypes.js'
|
||||
import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
|
||||
import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js'
|
||||
import type { BaseStorage } from '../storage/baseStorage.js'
|
||||
import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
|
||||
import { prodLog } from '../utils/logger.js'
|
||||
|
|
@ -1834,10 +1834,24 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
// Process all nouns at once
|
||||
for (const nounData of result.items) {
|
||||
try {
|
||||
if (!nounData.vector || nounData.vector.length === 0) {
|
||||
if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) {
|
||||
skippedUnvectored++
|
||||
continue
|
||||
}
|
||||
// THE ZERO-NORM LAW — bulk-rebuild leg: a persisted zero-norm
|
||||
// vector (a pre-10.4.2 row the canonical write has not yet
|
||||
// normalized) must never enter the index either, mirroring the
|
||||
// belt AddToVectorIndexOperation enforces on the live write path.
|
||||
// Only the canonical vector is authoritative here — persisted
|
||||
// HNSW graph metadata (level/connections) can outlive an unvector.
|
||||
if (isZeroNormVector(nounData.vector)) {
|
||||
prodLog.warn(
|
||||
`[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` +
|
||||
`zero-norm (a zero-norm vector is not a vector and never crosses an ` +
|
||||
`engine boundary)`
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
// Restore the pinned dimension from the first real vector this
|
||||
// rebuild loads. `addItem`/`updateItem` only pin `this.dimension`
|
||||
|
|
|
|||
Reference in a new issue