Merge branch 'next/zero-norm-unvector-door'
All checks were successful
CI / Node 22 (push) Successful in 12m24s
CI / Node 24 (push) Successful in 12m6s
CI / Bun (latest) (push) Successful in 12m32s
CI / Integration + conformance (Node 22) (push) Successful in 19m41s

# Conflicts:
#	src/hnsw/hnswIndex.ts
This commit is contained in:
David Snelling 2026-08-27 13:54:06 -07:00
commit 9b84ef5b02
6 changed files with 798 additions and 54 deletions

View file

@ -10,7 +10,7 @@ import {
Vector,
VectorDocument
} from '../coreTypes.js'
import { euclideanDistance, calculateDistancesBatch } from '../utils/index.js'
import { euclideanDistance, calculateDistancesBatch, isZeroNormVector } from '../utils/index.js'
import type { BaseStorage } from '../storage/baseStorage.js'
import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js'
import { prodLog } from '../utils/logger.js'
@ -1834,10 +1834,24 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
// Process all nouns at once
for (const nounData of result.items) {
try {
if (!nounData.vector || nounData.vector.length === 0) {
if (!Array.isArray(nounData.vector) || nounData.vector.length === 0) {
skippedUnvectored++
continue
}
// THE ZERO-NORM LAW — bulk-rebuild leg: a persisted zero-norm
// vector (a pre-10.4.2 row the canonical write has not yet
// normalized) must never enter the index either, mirroring the
// belt AddToVectorIndexOperation enforces on the live write path.
// Only the canonical vector is authoritative here — persisted
// HNSW graph metadata (level/connections) can outlive an unvector.
if (isZeroNormVector(nounData.vector)) {
prodLog.warn(
`[HNSW] rebuild(): skipping entity ${nounData.id} — persisted vector is ` +
`zero-norm (a zero-norm vector is not a vector and never crosses an ` +
`engine boundary)`
)
continue
}
// Restore the pinned dimension from the first real vector this
// rebuild loads. `addItem`/`updateItem` only pin `this.dimension`