A zero-norm vector is lawful inside brainy (cosine distance scores it at maximum, never a false top hit) but a false attractor for a downstream engine serving squared-euclidean distance, which cannot tell a real all-zero vector apart from a legitimate origin point. - The VFS root now persists with vector [] (the existing "unvectored" shape) instead of a real all-zero 384-dim placeholder, and is never routed into the deferred-embed pipeline. - A one-time migration in the root-init path detects a pre-fix store's all-zero placeholder root (by norm, not length) and rewrites it to [] through a new sanctioned Brainy method that keeps the canonical vectored-noun ledger honest and removes the row from the vector index. - The vector-index write seam (AddToVectorIndexOperation, ReplaceInVectorIndexOperation, and the generation materializer's direct insert) now refuses any real all-zero vector before it reaches a provider, loudly naming the entity, while the canonical write still lands. - add()'s dimension-pinning and HNSW-insert gates, and the add-params validator, now treat any empty vector as carrying no dimension information, closing a latent trap where an explicit `vector: []` would have pinned dimensions to 0.
148 lines
5 KiB
TypeScript
148 lines
5 KiB
TypeScript
/**
|
|
* Distance functions for vector similarity calculations.
|
|
*
|
|
* Pure-JavaScript implementations using allocation-free indexed loops: a single
|
|
* pass over the two vectors with scalar accumulators and no per-element closures
|
|
* or intermediate objects. This is the open-core distance path (the native
|
|
* provider owns the SIMD/quantized billion-scale path); for the small/medium
|
|
* vectors it serves (e.g. 384-dim sentence embeddings) a tight loop keeps the
|
|
* whole computation in registers with zero GC pressure.
|
|
*
|
|
* MEASURED (tests/benchmarks/distance-microbench.mjs, dim=384, N=20000, median
|
|
* of 41): rewriting cosine from an object-accumulating `reduce` to this loop is
|
|
* ~6x on `number[]`; euclidean ~1.4x. (`number[]` is also measurably faster than
|
|
* `Float32Array` here — V8 widens f32→f64 on every element read — so the
|
|
* resident representation stays `number[]`.)
|
|
*/
|
|
|
|
import { DistanceFunction, Vector } from '../coreTypes.js'
|
|
|
|
/**
|
|
* Calculates the Euclidean (L2) distance between two vectors.
|
|
* Lower values indicate higher similarity.
|
|
*/
|
|
export const euclideanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
|
|
if (a.length !== b.length) {
|
|
throw new Error('Vectors must have the same dimensions')
|
|
}
|
|
|
|
let sum = 0
|
|
const len = a.length
|
|
for (let i = 0; i < len; i++) {
|
|
const diff = a[i] - b[i]
|
|
sum += diff * diff
|
|
}
|
|
return Math.sqrt(sum)
|
|
}
|
|
|
|
/**
|
|
* Calculates the cosine distance between two vectors.
|
|
* Lower values indicate higher similarity. Range: 0 (identical) to 2 (opposite).
|
|
*/
|
|
export const cosineDistance: DistanceFunction = (a: Vector, b: Vector): number => {
|
|
if (a.length !== b.length) {
|
|
throw new Error('Vectors must have the same dimensions')
|
|
}
|
|
|
|
let dotProduct = 0
|
|
let normA = 0
|
|
let normB = 0
|
|
const len = a.length
|
|
for (let i = 0; i < len; i++) {
|
|
const av = a[i]
|
|
const bv = b[i]
|
|
dotProduct += av * bv
|
|
normA += av * av
|
|
normB += bv * bv
|
|
}
|
|
|
|
if (normA === 0 || normB === 0) {
|
|
return 2 // Maximum distance for zero vectors
|
|
}
|
|
|
|
const similarity = dotProduct / (Math.sqrt(normA) * Math.sqrt(normB))
|
|
// Convert cosine similarity (-1 to 1) to distance (0 to 2)
|
|
return 1 - similarity
|
|
}
|
|
|
|
/**
|
|
* True when `vector` is a REAL (non-empty) all-zero vector — the "false
|
|
* attractor" shape this engine's own cosine distance treats safely (a
|
|
* zero-norm operand always scores the MAXIMUM distance, see
|
|
* {@link cosineDistance}) but a downstream engine serving squared-euclidean
|
|
* distance cannot distinguish from a legitimate origin point. THE LAW: a
|
|
* zero-norm vector is not a vector — it never crosses an engine boundary
|
|
* (never handed to a vector-index provider as a searchable item).
|
|
*
|
|
* A length-0 vector is the UNRELATED "unvectored, not yet embedded" shape
|
|
* (the deferred-embed stub, a permanently-vectorless system row) and is
|
|
* deliberately NOT zero-norm here — callers checking for "nothing to index"
|
|
* should test `vector.length === 0` separately; this only flags the
|
|
* dangerous non-empty all-zero case.
|
|
*/
|
|
export function isZeroNormVector(vector: readonly number[]): boolean {
|
|
if (vector.length === 0) return false
|
|
for (let i = 0; i < vector.length; i++) {
|
|
if (vector[i] !== 0) return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
/**
|
|
* Calculates the Manhattan (L1) distance between two vectors.
|
|
* Lower values indicate higher similarity.
|
|
*/
|
|
export const manhattanDistance: DistanceFunction = (a: Vector, b: Vector): number => {
|
|
if (a.length !== b.length) {
|
|
throw new Error('Vectors must have the same dimensions')
|
|
}
|
|
|
|
let sum = 0
|
|
const len = a.length
|
|
for (let i = 0; i < len; i++) {
|
|
sum += Math.abs(a[i] - b[i])
|
|
}
|
|
return sum
|
|
}
|
|
|
|
/**
|
|
* Calculates the dot-product similarity between two vectors, negated to a
|
|
* distance metric (lower is better).
|
|
*/
|
|
export const dotProductDistance: DistanceFunction = (a: Vector, b: Vector): number => {
|
|
if (a.length !== b.length) {
|
|
throw new Error('Vectors must have the same dimensions')
|
|
}
|
|
|
|
let dotProduct = 0
|
|
const len = a.length
|
|
for (let i = 0; i < len; i++) {
|
|
dotProduct += a[i] * b[i]
|
|
}
|
|
return -dotProduct
|
|
}
|
|
|
|
/**
|
|
* Batch distance calculation: the query vector against each candidate.
|
|
*
|
|
* With the distance functions now allocation-free indexed loops, this is a thin
|
|
* map over the (monomorphic, JIT-inlined) `distanceFunction` — no worker, no
|
|
* stringify/`new Function` reconstruction. Kept `async` for call-site
|
|
* compatibility with the HNSW search path.
|
|
*
|
|
* @param queryVector The query vector to compare against all candidates.
|
|
* @param vectors The candidate vectors.
|
|
* @param distanceFunction The distance function to use (default: Euclidean).
|
|
* @returns The distances, index-aligned with `vectors`.
|
|
*/
|
|
export async function calculateDistancesBatch(
|
|
queryVector: Vector,
|
|
vectors: Vector[],
|
|
distanceFunction: DistanceFunction = euclideanDistance
|
|
): Promise<number[]> {
|
|
const out = new Array<number>(vectors.length)
|
|
for (let i = 0; i < vectors.length; i++) {
|
|
out[i] = distanceFunction(queryVector, vectors[i])
|
|
}
|
|
return out
|
|
}
|