chore(8.0): delete vectorStore:mmap wiring — dead in the 8.0 provider world
The mmap-vector backend was a 2.x-era accelerator for the JS HNSW read path: a native provider registered under 'vectorStore:mmap' supplied a single-file mmap'd vector store, and JsHnswVectorIndex tried it before per-entity storage reads with lazy write-back migration. In the 8.0 provider world nothing registers that key — the native plugin's 3.0 line replaces the entire vector index under the 'vector' provider key (vectors live inside its own index file), and the open-core path never had an mmap provider. The wiring is unreachable; per the no-unwired-code rule it goes away: - src/hnsw/mmapVectorBackend.ts deleted (175 LOC) - wireMmapVectorBackend() + its init call site removed from brainy.ts - VectorStoreMmapProvider + VectorStoreMmapInstance contracts removed from plugin.ts - JsHnswVectorIndex loses the vectorBackend field, setVectorBackend(), and the mmap-first branches in getVectorSafe/preloadVectors — the storage + UnifiedCache path is now the single read path - tests/unit/hnsw/mmap-vector-backend.test.ts deleted The 7.x line keeps the wiring and got the capacity-NaN bugfix as 7.31.3 (see the platform handoff mmap thread). EntityIdMapperProvider stays — the connections codec and the id mapper still implement it. 1403/1403 tests, build clean.
This commit is contained in:
parent
42159f2bd7
commit
62f6472fa0
5 changed files with 2 additions and 640 deletions
|
|
@ -17,7 +17,6 @@ import { prodLog } from '../utils/logger.js'
|
|||
import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js'
|
||||
import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js'
|
||||
import type { VectorIndexProvider } from '../plugin.js'
|
||||
import { MmapVectorBackend } from './mmapVectorBackend.js'
|
||||
import { ConnectionsCodec, compressedConnectionsKey } from './connectionsCodec.js'
|
||||
|
||||
// Default HNSW parameters
|
||||
|
|
@ -50,13 +49,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
private unifiedCache: UnifiedCache // Shared cache with Graph and Metadata indexes
|
||||
// Always-adaptive caching - no "mode" concept, system adapts automatically
|
||||
|
||||
// Optional mmap-vector backend (2.4.0 #2). When set, the read paths
|
||||
// (preloadVectors, getVectorSafe) try the mmap layer before storage, and
|
||||
// on a storage hit they write back into the mmap slot — a lazy migration
|
||||
// that converges upgraded installs to the fast path under live traffic.
|
||||
// Null on cloud storage adapters (no local path) and pre-injection init.
|
||||
private vectorBackend: MmapVectorBackend | null = null
|
||||
|
||||
// Optional connections codec (2.4.0 #3). When set, node-persist writes the
|
||||
// node's per-level connection sets as a single delta-varint-compressed
|
||||
// binary blob (typically ~4× smaller than the legacy JSON-UUID-array
|
||||
|
|
@ -119,22 +111,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
this.useParallelization = useParallelization
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Inject (or detach) the mmap-vector backend. When set, the
|
||||
* vector read paths (`preloadVectors`, `getVectorSafe`) try the mmap layer
|
||||
* before storage and lazily write back on storage hits — converging an
|
||||
* upgraded install to the zero-copy fast path without a migration step.
|
||||
* Setting to null reverts to the existing per-entity read path.
|
||||
*
|
||||
* Lifecycle: brainy.ts wires this after plugin activation when (a) the
|
||||
* `vectorStore:mmap` provider is registered AND (b) the storage adapter
|
||||
* resolves a real local path via `getBinaryBlobPath()`. Cloud adapters
|
||||
* leave it null; JsHnswVectorIndex's behaviour is then identical to pre-2.4.0.
|
||||
*/
|
||||
public setVectorBackend(backend: MmapVectorBackend | null): void {
|
||||
this.vectorBackend = backend
|
||||
}
|
||||
|
||||
/**
|
||||
* @description Inject (or detach) the connections codec. When set, node
|
||||
* persistence writes a delta-varint-compressed binary blob alongside an
|
||||
|
|
@ -1180,23 +1156,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
const cacheKey = `hnsw:vector:${noun.id}`
|
||||
|
||||
const vector = await this.unifiedCache.get(cacheKey, async () => {
|
||||
// Try the mmap-vector backend first when wired — zero-copy slot read,
|
||||
// no per-entity object hydration.
|
||||
if (this.vectorBackend) {
|
||||
const fromMmap = this.vectorBackend.readByUuid(noun.id)
|
||||
if (fromMmap) {
|
||||
this.unifiedCache.set(
|
||||
cacheKey,
|
||||
fromMmap,
|
||||
'vectors',
|
||||
fromMmap.length * 4,
|
||||
50
|
||||
)
|
||||
return fromMmap
|
||||
}
|
||||
}
|
||||
|
||||
// Storage fallback — the canonical source of truth.
|
||||
if (!this.storage) {
|
||||
throw new Error('Storage not available for vector loading')
|
||||
}
|
||||
|
|
@ -1206,18 +1165,6 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
throw new Error(`Vector not found for noun ${noun.id}`)
|
||||
}
|
||||
|
||||
// Lazy migration: write back into the mmap slot so the next read is
|
||||
// mmap-fast. Idempotent + resumable; a write failure is non-fatal —
|
||||
// the field still serves this read, and the next storage hit re-tries
|
||||
// the migration. (See mmapVectorBackend.ts for the contract.)
|
||||
if (this.vectorBackend) {
|
||||
try {
|
||||
this.vectorBackend.writeByUuid(noun.id, loaded)
|
||||
} catch (error) {
|
||||
prodLog.debug(`MmapVectorBackend write-back failed for ${noun.id}`, error)
|
||||
}
|
||||
}
|
||||
|
||||
// Add to UnifiedCache with cost-aware eviction
|
||||
// This competes fairly with Graph and Metadata indexes
|
||||
this.unifiedCache.set(
|
||||
|
|
@ -1275,50 +1222,7 @@ export class JsHnswVectorIndex implements VectorIndexProvider {
|
|||
private async preloadVectors(nodeIds: string[]): Promise<void> {
|
||||
if (nodeIds.length === 0) return
|
||||
|
||||
// mmap-vector backend fast path: batched O(1) slot reads + lazy
|
||||
// write-back migration on miss. Cache populated from both layers so
|
||||
// subsequent in-session reads are O(1) memory.
|
||||
if (this.vectorBackend) {
|
||||
// Filter out ids already cached so we never re-read.
|
||||
const uncachedIds: string[] = []
|
||||
for (const id of nodeIds) {
|
||||
if (this.unifiedCache.getSync(`hnsw:vector:${id}`) === undefined) {
|
||||
uncachedIds.push(id)
|
||||
}
|
||||
}
|
||||
if (uncachedIds.length === 0) return
|
||||
|
||||
const fromMmap = this.vectorBackend.readBatchByUuid(uncachedIds)
|
||||
const misses: string[] = []
|
||||
for (let i = 0; i < uncachedIds.length; i++) {
|
||||
const id = uncachedIds[i]
|
||||
const v = fromMmap[i]
|
||||
if (v) {
|
||||
this.unifiedCache.set(`vector:${id}`, v, 'vectors', v.length * 4, 50)
|
||||
} else {
|
||||
misses.push(id)
|
||||
}
|
||||
}
|
||||
if (misses.length === 0) return
|
||||
|
||||
// Storage fallback for the misses, with concurrent lazy write-back.
|
||||
await Promise.all(misses.map(async (id) => {
|
||||
if (!this.storage) return
|
||||
const vector = await this.storage.getNounVector(id)
|
||||
if (!vector) return
|
||||
if (this.vectorBackend) {
|
||||
try {
|
||||
this.vectorBackend.writeByUuid(id, vector)
|
||||
} catch (error) {
|
||||
prodLog.debug(`MmapVectorBackend write-back failed for ${id}`, error)
|
||||
}
|
||||
}
|
||||
this.unifiedCache.set(`vector:${id}`, vector, 'vectors', vector.length * 4, 50)
|
||||
}))
|
||||
return
|
||||
}
|
||||
|
||||
// Legacy path: per-entity storage load via UnifiedCache request coalescing.
|
||||
// Per-entity storage load via UnifiedCache request coalescing.
|
||||
const promises = nodeIds.map(async (id) => {
|
||||
const cacheKey = `hnsw:vector:${id}`
|
||||
return this.unifiedCache.get(cacheKey, async () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue