perf(8.0): HNSW removeItem is O(in-degree) via a reverse-adjacency index

removeItem() scanned the ENTIRE corpus on every delete to find nodes that
referenced the removed id (and to repair edges left asymmetric by pruning), so a
delete was O(N) and a bulk delete O(N²) — on the open-core JS vector path that
serves when no native provider is registered.

Maintain a reverse-adjacency index (`target → level → set of nodes that link to
target`), so removeItem touches only the removed node's actual in-neighbors:
O(in-degree) per delete, O(N·degree) for a bulk delete. The index is lazily
built, maintained incrementally at every forward-edge mutation (add-link and
prune), and invalidated (rebuilt on next use) by the bulk paths (cold-load
restore + clear). New tests assert the three invariants that matter for a
reverse index: no dangling references to deleted nodes, the incrementally-
maintained index exactly equals a fresh rebuild from the live adjacency, and
search still returns the survivors' true nearest neighbours (exact vs brute force).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/hnsw/hnswIndex.ts

@@ -32,6 +32,14 @@ const DEFAULT_CONFIG: HNSWConfig = {
  */
 export class JsHnswVectorIndex implements VectorIndexProvider {
   private nouns: Map<string, HNSWNoun> = new Map()
+  /**
+   * Reverse adjacency: `target id → (level → set of node ids that link TO target)`.
+   * Lets `removeItem` find a node's in-neighbors in O(in-degree) instead of scanning
+   * the whole corpus (which made bulk delete O(N²)). Lazily built on first need and
+   * maintained incrementally on link/prune/remove; `null` means "stale — rebuild on
+   * next use" (set by every bulk path: cold-load restore + `clear()`).
+   */
+  private incoming: Map<string, Map<number, Set<string>>> | null = null
   private entryPointId: string | null = null
   private maxLevel = 0
   // Track high-level nodes for O(1) entry point selection
@@ -245,6 +253,9 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
     hnswData: { level: number; connections: Record<string, string[]> },
     noun: HNSWNoun
   ): Promise<void> {
+    // Bulk load sets connections directly (bypassing the link path that maintains
+    // the reverse index) — mark it stale so the next removeItem rebuilds it.
+    this.incoming = null
     const storageWithBlob = this.storage as unknown as {
       loadBinaryBlob?: (key: string) => Promise<Buffer | null>
     }
@@ -489,12 +500,14 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
 
 
         noun.connections.get(level)!.add(neighborId)
+        this.addIncoming(neighborId, level, id) // forward edge id → neighborId
 
         // Add reverse connection
         if (!neighbor.connections.has(level)) {
           neighbor.connections.set(level, new Set<string>())
         }
         neighbor.connections.get(level)!.add(id)
+        this.addIncoming(id, level, neighborId) // forward edge neighborId → id
 
         // Ensure neighbor doesn't have too many connections
         if (neighbor.connections.get(level)!.size > this.config.M) {
@@ -841,6 +854,44 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
     return [...nearestNouns].slice(0, k)
   }
 
+  /**
+   * Build (or return the cached) reverse-adjacency index. O(N + E) the first time
+   * after a bulk load/clear; O(1) thereafter while it stays live.
+   */
+  private ensureIncoming(): Map<string, Map<number, Set<string>>> {
+    if (this.incoming !== null) return this.incoming
+    const inc = new Map<string, Map<number, Set<string>>>()
+    for (const [nodeId, node] of this.nouns) {
+      for (const [level, targets] of node.connections) {
+        for (const target of targets) {
+          let byLevel = inc.get(target)
+          if (!byLevel) { byLevel = new Map(); inc.set(target, byLevel) }
+          let set = byLevel.get(level)
+          if (!set) { set = new Set(); byLevel.set(level, set) }
+          set.add(nodeId)
+        }
+      }
+    }
+    this.incoming = inc
+    return inc
+  }
+
+  /** Record a new forward edge `source → target` at `level` (no-op while the index is stale). */
+  private addIncoming(target: string, level: number, source: string): void {
+    if (this.incoming === null) return
+    let byLevel = this.incoming.get(target)
+    if (!byLevel) { byLevel = new Map(); this.incoming.set(target, byLevel) }
+    let set = byLevel.get(level)
+    if (!set) { set = new Set(); byLevel.set(level, set) }
+    set.add(source)
+  }
+
+  /** Record that the forward edge `source → target` at `level` is gone. */
+  private removeIncoming(target: string, level: number, source: string): void {
+    if (this.incoming === null) return
+    this.incoming.get(target)?.get(level)?.delete(source)
+  }
+
   /**
    * Remove an item from the index
    */
@@ -852,41 +903,38 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
 
     const noun = this.nouns.get(id)!
 
-    // Remove connections to this noun from all neighbors
-    for (const [level, connections] of noun.connections.entries()) {
-      for (const neighborId of connections) {
-
-        const neighbor = this.nouns.get(neighborId)
-        if (!neighbor) {
-          // Skip neighbors that don't exist (expected during rapid additions/deletions)
-          continue
-        }
-        if (neighbor.connections.has(level)) {
-          neighbor.connections.get(level)!.delete(id)
-
-          // Prune connections after removing this noun to ensure consistency
-          await this.pruneConnections(neighbor, level)
+    // Reverse-adjacency lets us touch ONLY the nodes that actually reference `id`
+    // (its in-neighbors) rather than scanning the whole corpus — turning a delete
+    // from O(N) into O(in-degree) and a bulk delete from O(N²) into O(N·degree).
+    // Snapshot each referrer set because pruneConnections mutates the index.
+    const incoming = this.ensureIncoming()
+    const referrers = incoming.get(id)
+    if (referrers) {
+      for (const [level, refSet] of referrers) {
+        for (const refId of Array.from(refSet)) {
+          const ref = this.nouns.get(refId)
+          if (ref && ref.connections.has(level)) {
+            // Drop the forward edge ref → id, then re-prune ref so the graph stays
+            // navigable. (id's own reverse entry is dropped wholesale below, so we
+            // intentionally do not maintain incoming[id] inside this loop.)
+            ref.connections.get(level)!.delete(id)
+            await this.pruneConnections(ref, level)
+          }
         }
       }
     }
 
-    // Also check all other nouns for references to this noun and remove them
-    for (const [nounId, otherNoun] of this.nouns.entries()) {
-      if (nounId === id) continue // Skip the noun being removed
-
-
-      for (const [level, connections] of otherNoun.connections.entries()) {
-        if (connections.has(id)) {
-          connections.delete(id)
-
-          // Prune connections after removing this reference
-          await this.pruneConnections(otherNoun, level)
-        }
+    // id's OUTGOING edges disappear with it → drop id from each out-neighbor's
+    // reverse set so no stale referrer survives.
+    for (const [level, targets] of noun.connections) {
+      for (const target of targets) {
+        this.removeIncoming(target, level, id)
       }
     }
 
-    // Remove the noun
+    // Remove the noun + its reverse-index entry.
     this.nouns.delete(id)
+    this.incoming?.delete(id)
 
     // If we removed the entry point, find a new one
     if (this.entryPointId === id) {
@@ -966,6 +1014,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
    */
   public clear(): void {
     this.nouns.clear()
+    this.incoming = null // reverse index no longer reflects the (now empty) graph
     this.entryPointId = null
     this.maxLevel = 0
   }
@@ -1762,7 +1811,11 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
 
     // Only proceed if we have valid neighbors
     if (distances.size === 0) {
-      // If no valid neighbors, clear connections at this level
+      // If no valid neighbors, clear connections at this level. Every current
+      // connection is dropped — unlink each from the reverse index.
+      if (this.incoming !== null) {
+        for (const dropped of connections) this.removeIncoming(dropped, level, noun.id)
+      }
       noun.connections.set(level, new Set())
       return
     }
@@ -1774,8 +1827,16 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
       this.config.M
     )
 
-    // Update connections with only valid neighbors
-    noun.connections.set(level, new Set(selectedNeighbors.keys()))
+    // Update connections with only valid neighbors. Pruning only ever drops
+    // edges (the selected set is a subset of the current connections), so unlink
+    // each dropped neighbor from the reverse index.
+    const kept = new Set(selectedNeighbors.keys())
+    if (this.incoming !== null) {
+      for (const prev of connections) {
+        if (!kept.has(prev)) this.removeIncoming(prev, level, noun.id)
+      }
+    }
+    noun.connections.set(level, kept)
   }
 
   /**