refactor(8.0): rename HNSWIndex class → JsHnswVectorIndex (scaffold step 3)
Step 3 of the brainy 8.0 rename scaffolding. The class name now matches its role: the JS HNSW implementation of the VectorIndexProvider contract. Per planning § 2.3: this is NOT cosmetic — leaving HNSWIndex when the public contract is VectorIndexProvider creates a confusing read at the implementation layer. CHANGES Repo-wide mechanical rename: HNSWIndex → JsHnswVectorIndex across 52 references in src/ + 5 references in tests/ via sed. Class declaration, imports, JSDoc, comments, and identifiers all updated. src/index.ts - Primary public export: JsHnswVectorIndex (the new name). - Backwards-compat alias: `export const HNSWIndex = JsHnswVectorIndex` with @deprecated note. Removed in the final 8.0 cleanup commit. NO-OP scope No behavioural change. The class is exactly the same; only the name moved. Tests + build green. VERIFICATION - npx tsc --noEmit: clean - npm test: 1468 / 1468 unit
This commit is contained in:
parent
8f87b35614
commit
f39d420cb4
11 changed files with 80 additions and 70 deletions
|
|
@ -6,8 +6,8 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { v4 as uuidv4 } from './universal/uuid.js'
|
import { v4 as uuidv4 } from './universal/uuid.js'
|
||||||
import { HNSWIndex } from './hnsw/hnswIndex.js'
|
import { JsHnswVectorIndex } from './hnsw/hnswIndex.js'
|
||||||
// TypeAwareHNSWIndex removed from default path — single unified HNSWIndex is faster
|
// TypeAwareHNSWIndex removed from default path — single unified JsHnswVectorIndex is faster
|
||||||
// for the 99% of queries that don't filter by type (avoids searching 42 separate graphs)
|
// for the 99% of queries that don't filter by type (avoids searching 42 separate graphs)
|
||||||
import { createStorage } from './storage/storageFactory.js'
|
import { createStorage } from './storage/storageFactory.js'
|
||||||
import { BaseStorage } from './storage/baseStorage.js'
|
import { BaseStorage } from './storage/baseStorage.js'
|
||||||
|
|
@ -156,7 +156,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
private static instances: Brainy[] = []
|
private static instances: Brainy[] = []
|
||||||
|
|
||||||
// Core components
|
// Core components
|
||||||
private index!: HNSWIndex
|
private index!: JsHnswVectorIndex
|
||||||
private storage!: BaseStorage
|
private storage!: BaseStorage
|
||||||
private metadataIndex!: MetadataIndexManager
|
private metadataIndex!: MetadataIndexManager
|
||||||
private graphIndex!: GraphAdjacencyIndex
|
private graphIndex!: GraphAdjacencyIndex
|
||||||
|
|
@ -606,8 +606,8 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
// Wire the mmap-vector backend (2.4.0 #2). When the vectorStore:mmap
|
// Wire the mmap-vector backend (2.4.0 #2). When the vectorStore:mmap
|
||||||
// provider is registered AND the storage adapter resolves a real local
|
// provider is registered AND the storage adapter resolves a real local
|
||||||
// path via getBinaryBlobPath(), open the mmap file there and inject the
|
// path via getBinaryBlobPath(), open the mmap file there and inject the
|
||||||
// backend into HNSWIndex. Cloud adapters return null and skip silently;
|
// backend into JsHnswVectorIndex. Cloud adapters return null and skip silently;
|
||||||
// HNSWIndex's behaviour is then identical to pre-2.4.0 (per-entity reads
|
// JsHnswVectorIndex's behaviour is then identical to pre-2.4.0 (per-entity reads
|
||||||
// from canonical storage). Failures are non-fatal — the index still
|
// from canonical storage). Failures are non-fatal — the index still
|
||||||
// works, just without the zero-copy fast path.
|
// works, just without the zero-copy fast path.
|
||||||
await this.wireMmapVectorBackend()
|
await this.wireMmapVectorBackend()
|
||||||
|
|
@ -8324,7 +8324,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* @param candidateIds Optional pre-resolved metadata filter IDs for metadata-first search.
|
* @param candidateIds Optional pre-resolved metadata filter IDs for metadata-first search.
|
||||||
* When provided, HNSW search is restricted to these candidates:
|
* When provided, HNSW search is restricted to these candidates:
|
||||||
* - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering)
|
* - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering)
|
||||||
* - JS HNSWIndex: converts to filter function with O(1) Set lookups
|
* - JS JsHnswVectorIndex: converts to filter function with O(1) Set lookups
|
||||||
*/
|
*/
|
||||||
private async executeVectorSearch(params: FindParams<T>, candidateIds?: string[]): Promise<Result<T>[]> {
|
private async executeVectorSearch(params: FindParams<T>, candidateIds?: string[]): Promise<Result<T>[]> {
|
||||||
const vector = params.vector || (await this.embed(params.query!))
|
const vector = params.vector || (await this.embed(params.query!))
|
||||||
|
|
@ -9070,7 +9070,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* - Cloud storage (GCS/S3/R2/Azure): 'deferred' for 30-50x faster adds
|
* - Cloud storage (GCS/S3/R2/Azure): 'deferred' for 30-50x faster adds
|
||||||
* - Local storage (FileSystem/Memory/OPFS): 'immediate' (already fast)
|
* - Local storage (FileSystem/Memory/OPFS): 'immediate' (already fast)
|
||||||
*/
|
*/
|
||||||
private setupIndex(): HNSWIndex {
|
private setupIndex(): JsHnswVectorIndex {
|
||||||
const indexConfig = {
|
const indexConfig = {
|
||||||
...this.config.index,
|
...this.config.index,
|
||||||
distanceFunction: this.distance,
|
distanceFunction: this.distance,
|
||||||
|
|
@ -9085,7 +9085,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
|
|
||||||
const persistMode = this.resolveHNSWPersistMode()
|
const persistMode = this.resolveHNSWPersistMode()
|
||||||
|
|
||||||
return new HNSWIndex(indexConfig as any, this.distance, {
|
return new JsHnswVectorIndex(indexConfig as any, this.distance, {
|
||||||
storage: this.storage,
|
storage: this.storage,
|
||||||
useParallelization: true,
|
useParallelization: true,
|
||||||
persistMode
|
persistMode
|
||||||
|
|
@ -9106,11 +9106,11 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* - The metadata index has a stable `idMapper`.
|
* - The metadata index has a stable `idMapper`.
|
||||||
* - `config.index.type !== 'hnsw'` (no explicit opt-out).
|
* - `config.index.type !== 'hnsw'` (no explicit opt-out).
|
||||||
* 3. `'hnsw'` provider if registered (cortex's native HNSW).
|
* 3. `'hnsw'` provider if registered (cortex's native HNSW).
|
||||||
* 4. Brainy's built-in TS HNSWIndex (the always-available fallback).
|
* 4. Brainy's built-in TS JsHnswVectorIndex (the always-available fallback).
|
||||||
*
|
*
|
||||||
* See ADR-002 in the cortex repo for the engagement rationale.
|
* See ADR-002 in the cortex repo for the engagement rationale.
|
||||||
*/
|
*/
|
||||||
private createIndex(): HNSWIndex {
|
private createIndex(): JsHnswVectorIndex {
|
||||||
const persistMode = this.resolveHNSWPersistMode()
|
const persistMode = this.resolveHNSWPersistMode()
|
||||||
const indexType = (this.config.index as any)?.type as 'hnsw' | 'diskann' | undefined
|
const indexType = (this.config.index as any)?.type as 'hnsw' | 'diskann' | undefined
|
||||||
|
|
||||||
|
|
@ -9170,7 +9170,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
private instantiateDiskAnn(
|
private instantiateDiskAnn(
|
||||||
factory: (config: any, distance: DistanceFunction, options: any) => any,
|
factory: (config: any, distance: DistanceFunction, options: any) => any,
|
||||||
persistMode: 'immediate' | 'deferred'
|
persistMode: 'immediate' | 'deferred'
|
||||||
): HNSWIndex {
|
): JsHnswVectorIndex {
|
||||||
const storageWithBlob = this.storage as unknown as {
|
const storageWithBlob = this.storage as unknown as {
|
||||||
getBinaryBlobPath?: (key: string) => string | null
|
getBinaryBlobPath?: (key: string) => string | null
|
||||||
}
|
}
|
||||||
|
|
@ -9377,7 +9377,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description Open the mmap-vector backend and inject it into HNSWIndex.
|
* @description Open the mmap-vector backend and inject it into JsHnswVectorIndex.
|
||||||
* Called once during init after plugin activation, metadataIndex init, and
|
* Called once during init after plugin activation, metadataIndex init, and
|
||||||
* graphIndex setup — so the idMapper is available and providers are wired.
|
* graphIndex setup — so the idMapper is available and providers are wired.
|
||||||
*
|
*
|
||||||
|
|
@ -9397,7 +9397,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
||||||
* (or a re-open) sees the correct value.
|
* (or a re-open) sees the correct value.
|
||||||
*
|
*
|
||||||
* Failures here are non-fatal: a debug-level log records the reason, and
|
* Failures here are non-fatal: a debug-level log records the reason, and
|
||||||
* HNSWIndex continues without the mmap fast path.
|
* JsHnswVectorIndex continues without the mmap fast path.
|
||||||
*/
|
*/
|
||||||
private async wireMmapVectorBackend(): Promise<void> {
|
private async wireMmapVectorBackend(): Promise<void> {
|
||||||
const provider = this.pluginRegistry.getProvider<VectorStoreMmapProvider>('vectorStore:mmap')
|
const provider = this.pluginRegistry.getProvider<VectorStoreMmapProvider>('vectorStore:mmap')
|
||||||
|
|
|
||||||
|
|
@ -31,10 +31,10 @@ const DEFAULT_CONFIG: HNSWConfig = {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls
|
* Implements {@link VectorIndexProvider}: the vector-index surface Brainy calls
|
||||||
* on whatever the `'vector'` factory returns (its own `HNSWIndex`, or a native
|
* on whatever the `'vector'` factory returns (its own `JsHnswVectorIndex`, or a native
|
||||||
* acceleration provider).
|
* acceleration provider).
|
||||||
*/
|
*/
|
||||||
export class HNSWIndex implements VectorIndexProvider {
|
export class JsHnswVectorIndex implements VectorIndexProvider {
|
||||||
private nouns: Map<string, HNSWNoun> = new Map()
|
private nouns: Map<string, HNSWNoun> = new Map()
|
||||||
private entryPointId: string | null = null
|
private entryPointId: string | null = null
|
||||||
private maxLevel = 0
|
private maxLevel = 0
|
||||||
|
|
@ -71,7 +71,7 @@ export class HNSWIndex implements VectorIndexProvider {
|
||||||
// COW (Copy-on-Write) support
|
// COW (Copy-on-Write) support
|
||||||
private cowEnabled: boolean = false
|
private cowEnabled: boolean = false
|
||||||
private cowModifiedNodes: Set<string> = new Set()
|
private cowModifiedNodes: Set<string> = new Set()
|
||||||
private cowParent: HNSWIndex | null = null
|
private cowParent: JsHnswVectorIndex | null = null
|
||||||
|
|
||||||
// Deferred HNSW persistence for cloud storage performance
|
// Deferred HNSW persistence for cloud storage performance
|
||||||
// In deferred mode, HNSW connections are only persisted on flush/close
|
// In deferred mode, HNSW connections are only persisted on flush/close
|
||||||
|
|
@ -130,7 +130,7 @@ export class HNSWIndex implements VectorIndexProvider {
|
||||||
* Lifecycle: brainy.ts wires this after plugin activation when (a) the
|
* Lifecycle: brainy.ts wires this after plugin activation when (a) the
|
||||||
* `vectorStore:mmap` provider is registered AND (b) the storage adapter
|
* `vectorStore:mmap` provider is registered AND (b) the storage adapter
|
||||||
* resolves a real local path via `getBinaryBlobPath()`. Cloud adapters
|
* resolves a real local path via `getBinaryBlobPath()`. Cloud adapters
|
||||||
* leave it null; HNSWIndex's behaviour is then identical to pre-2.4.0.
|
* leave it null; JsHnswVectorIndex's behaviour is then identical to pre-2.4.0.
|
||||||
*/
|
*/
|
||||||
public setVectorBackend(backend: MmapVectorBackend | null): void {
|
public setVectorBackend(backend: MmapVectorBackend | null): void {
|
||||||
this.vectorBackend = backend
|
this.vectorBackend = backend
|
||||||
|
|
@ -340,10 +340,10 @@ export class HNSWIndex implements VectorIndexProvider {
|
||||||
*
|
*
|
||||||
* @example
|
* @example
|
||||||
* ```typescript
|
* ```typescript
|
||||||
* const parent = new HNSWIndex(config)
|
* const parent = new JsHnswVectorIndex(config)
|
||||||
* // ... parent has 1M nodes ...
|
* // ... parent has 1M nodes ...
|
||||||
*
|
*
|
||||||
* const fork = new HNSWIndex(config)
|
* const fork = new JsHnswVectorIndex(config)
|
||||||
* fork.enableCOW(parent) // <10ms - instant!
|
* fork.enableCOW(parent) // <10ms - instant!
|
||||||
*
|
*
|
||||||
* // Reads share data
|
* // Reads share data
|
||||||
|
|
@ -353,7 +353,7 @@ export class HNSWIndex implements VectorIndexProvider {
|
||||||
* await fork.addItem(newItem) // Deep copies only modified nodes
|
* await fork.addItem(newItem) // Deep copies only modified nodes
|
||||||
* ```
|
* ```
|
||||||
*/
|
*/
|
||||||
public enableCOW(parent: HNSWIndex): void {
|
public enableCOW(parent: JsHnswVectorIndex): void {
|
||||||
this.cowEnabled = true
|
this.cowEnabled = true
|
||||||
this.cowParent = parent
|
this.cowParent = parent
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
/**
|
/**
|
||||||
* @module hnsw/mmapVectorBackend
|
* @module hnsw/mmapVectorBackend
|
||||||
* @description Disk-resident vector cache backing HNSWIndex, wrapping the
|
* @description Disk-resident vector cache backing JsHnswVectorIndex, wrapping the
|
||||||
* `vectorStore:mmap` provider (cortex's NativeMmapVectorStore in production).
|
* `vectorStore:mmap` provider (cortex's NativeMmapVectorStore in production).
|
||||||
*
|
*
|
||||||
* **Why this exists.** Brainy's per-entity vector storage hits two ceilings as
|
* **Why this exists.** Brainy's per-entity vector storage hits two ceilings as
|
||||||
* datasets grow: (1) every `HNSWIndex.preloadVectors` does a sequential
|
* datasets grow: (1) every `JsHnswVectorIndex.preloadVectors` does a sequential
|
||||||
* `storage.getNounVector` per id (no batch), and (2) cloud and FS storage both
|
* `storage.getNounVector` per id (no batch), and (2) cloud and FS storage both
|
||||||
* serialize each vector through JSON / msgpack, blocking zero-copy reads. An
|
* serialize each vector through JSON / msgpack, blocking zero-copy reads. An
|
||||||
* mmap-backed single-file vector store, addressed by the stable int from
|
* mmap-backed single-file vector store, addressed by the stable int from
|
||||||
|
|
@ -20,7 +20,7 @@
|
||||||
*
|
*
|
||||||
* **Local-FS only in 2.4.0.** Activated when the storage adapter exposes a
|
* **Local-FS only in 2.4.0.** Activated when the storage adapter exposes a
|
||||||
* real local path via `getBinaryBlobPath()`. Cloud adapters (S3 / R2 / GCS)
|
* real local path via `getBinaryBlobPath()`. Cloud adapters (S3 / R2 / GCS)
|
||||||
* return null and the backend is not constructed, so HNSWIndex falls back to
|
* return null and the backend is not constructed, so JsHnswVectorIndex falls back to
|
||||||
* the existing per-entity read path. Chunked vector segments for remote
|
* the existing per-entity read path. Chunked vector segments for remote
|
||||||
* storage are the planned 2.4.0 #2 follow-up.
|
* storage are the planned 2.4.0 #2 follow-up.
|
||||||
*/
|
*/
|
||||||
|
|
@ -38,7 +38,7 @@ import type {
|
||||||
* @description Bridge between brainy's UUID-keyed vector API and cortex's
|
* @description Bridge between brainy's UUID-keyed vector API and cortex's
|
||||||
* int-slot mmap vector store. Translates UUIDs to stable int slots via the
|
* int-slot mmap vector store. Translates UUIDs to stable int slots via the
|
||||||
* `EntityIdMapper`, grows the file on demand, and handles read misses by
|
* `EntityIdMapper`, grows the file on demand, and handles read misses by
|
||||||
* surfacing `null` (HNSWIndex's caller does the lazy storage fallback +
|
* surfacing `null` (JsHnswVectorIndex's caller does the lazy storage fallback +
|
||||||
* write-back). The class never touches per-entity storage directly — it owns
|
* write-back). The class never touches per-entity storage directly — it owns
|
||||||
* only the mmap layer.
|
* only the mmap layer.
|
||||||
*/
|
*/
|
||||||
|
|
|
||||||
|
|
@ -7,13 +7,13 @@
|
||||||
* - Storage: Already type-first from Phase 1a
|
* - Storage: Already type-first from Phase 1a
|
||||||
*
|
*
|
||||||
* Architecture:
|
* Architecture:
|
||||||
* - One HNSWIndex per NounType (42 total)
|
* - One JsHnswVectorIndex per NounType (42 total)
|
||||||
* - Lazy initialization (indexes created on first use)
|
* - Lazy initialization (indexes created on first use)
|
||||||
* - Type routing for optimal performance
|
* - Type routing for optimal performance
|
||||||
* - Falls back to multi-type search when type unknown
|
* - Falls back to multi-type search when type unknown
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { HNSWIndex } from './hnswIndex.js'
|
import { JsHnswVectorIndex } from './hnswIndex.js'
|
||||||
import {
|
import {
|
||||||
DistanceFunction,
|
DistanceFunction,
|
||||||
HNSWConfig,
|
HNSWConfig,
|
||||||
|
|
@ -25,7 +25,7 @@ import { euclideanDistance } from '../utils/index.js'
|
||||||
import type { BaseStorage } from '../storage/baseStorage.js'
|
import type { BaseStorage } from '../storage/baseStorage.js'
|
||||||
import { prodLog } from '../utils/logger.js'
|
import { prodLog } from '../utils/logger.js'
|
||||||
|
|
||||||
// Default HNSW parameters (same as HNSWIndex)
|
// Default HNSW parameters (same as JsHnswVectorIndex)
|
||||||
const DEFAULT_CONFIG: HNSWConfig = {
|
const DEFAULT_CONFIG: HNSWConfig = {
|
||||||
M: 16,
|
M: 16,
|
||||||
efConstruction: 200,
|
efConstruction: 200,
|
||||||
|
|
@ -58,7 +58,7 @@ export interface TypeAwareHNSWStats {
|
||||||
*/
|
*/
|
||||||
export class TypeAwareHNSWIndex {
|
export class TypeAwareHNSWIndex {
|
||||||
// One HNSW index per noun type (lazy initialization)
|
// One HNSW index per noun type (lazy initialization)
|
||||||
private indexes: Map<NounType, HNSWIndex> = new Map()
|
private indexes: Map<NounType, JsHnswVectorIndex> = new Map()
|
||||||
|
|
||||||
// Configuration
|
// Configuration
|
||||||
private config: HNSWConfig
|
private config: HNSWConfig
|
||||||
|
|
@ -105,7 +105,7 @@ export class TypeAwareHNSWIndex {
|
||||||
|
|
||||||
// Enable COW on each underlying type-specific index
|
// Enable COW on each underlying type-specific index
|
||||||
for (const [type, parentIndex] of parent.indexes.entries()) {
|
for (const [type, parentIndex] of parent.indexes.entries()) {
|
||||||
const childIndex = new HNSWIndex(this.config, this.distanceFunction, {
|
const childIndex = new JsHnswVectorIndex(this.config, this.distanceFunction, {
|
||||||
useParallelization: this.useParallelization,
|
useParallelization: this.useParallelization,
|
||||||
storage: this.storage || undefined,
|
storage: this.storage || undefined,
|
||||||
persistMode: this.persistMode
|
persistMode: this.persistMode
|
||||||
|
|
@ -183,9 +183,9 @@ export class TypeAwareHNSWIndex {
|
||||||
* Only types with entities get an index.
|
* Only types with entities get an index.
|
||||||
*
|
*
|
||||||
* @param type The noun type
|
* @param type The noun type
|
||||||
* @returns HNSWIndex for this type
|
* @returns JsHnswVectorIndex for this type
|
||||||
*/
|
*/
|
||||||
private getIndexForType(type: NounType): HNSWIndex {
|
private getIndexForType(type: NounType): JsHnswVectorIndex {
|
||||||
// Validate type is a valid NounType
|
// Validate type is a valid NounType
|
||||||
const typeIndex = TypeUtils.getNounIndex(type)
|
const typeIndex = TypeUtils.getNounIndex(type)
|
||||||
if (typeIndex === undefined || typeIndex === null || typeIndex < 0) {
|
if (typeIndex === undefined || typeIndex === null || typeIndex < 0) {
|
||||||
|
|
@ -197,7 +197,7 @@ export class TypeAwareHNSWIndex {
|
||||||
if (!this.indexes.has(type)) {
|
if (!this.indexes.has(type)) {
|
||||||
prodLog.info(`Creating HNSW index for type: ${type}`)
|
prodLog.info(`Creating HNSW index for type: ${type}`)
|
||||||
|
|
||||||
const index = new HNSWIndex(this.config, this.distanceFunction, {
|
const index = new JsHnswVectorIndex(this.config, this.distanceFunction, {
|
||||||
useParallelization: this.useParallelization,
|
useParallelization: this.useParallelization,
|
||||||
storage: this.storage || undefined,
|
storage: this.storage || undefined,
|
||||||
persistMode: this.persistMode
|
persistMode: this.persistMode
|
||||||
|
|
|
||||||
16
src/index.ts
16
src/index.ts
|
|
@ -296,10 +296,20 @@ import type {
|
||||||
StorageAdapter
|
StorageAdapter
|
||||||
} from './coreTypes.js'
|
} from './coreTypes.js'
|
||||||
|
|
||||||
// Export HNSW index
|
// Export vector index implementation (the JS HNSW path)
|
||||||
import { HNSWIndex } from './hnsw/hnswIndex.js'
|
import { JsHnswVectorIndex } from './hnsw/hnswIndex.js'
|
||||||
|
|
||||||
export { HNSWIndex }
|
export { JsHnswVectorIndex }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @deprecated Renamed to {@link JsHnswVectorIndex} in Brainy 8.0. The
|
||||||
|
* implementation is unchanged — the new name disambiguates the JS HNSW
|
||||||
|
* path from any native vector-index provider (DiskANN, etc.) that may be
|
||||||
|
* loaded via the `'vector'` provider key. This alias is provided as a
|
||||||
|
* compatibility shim for code mid-migration and is removed in the 8.0
|
||||||
|
* final cleanup commit.
|
||||||
|
*/
|
||||||
|
export const HNSWIndex = JsHnswVectorIndex
|
||||||
|
|
||||||
export type {
|
export type {
|
||||||
Vector,
|
Vector,
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,7 @@ export interface BrainyPluginContext {
|
||||||
* - 'graphIndex' — GraphAdjacencyIndex replacement
|
* - 'graphIndex' — GraphAdjacencyIndex replacement
|
||||||
* - 'entityIdMapper' — EntityIdMapper replacement
|
* - 'entityIdMapper' — EntityIdMapper replacement
|
||||||
* - 'cache' — UnifiedCache replacement
|
* - 'cache' — UnifiedCache replacement
|
||||||
* - 'hnsw' — HNSWIndex replacement
|
* - 'hnsw' — JsHnswVectorIndex replacement
|
||||||
* - 'roaring' — RoaringBitmap32 replacement
|
* - 'roaring' — RoaringBitmap32 replacement
|
||||||
* - 'embeddings' — Embedding engine replacement (single text)
|
* - 'embeddings' — Embedding engine replacement (single text)
|
||||||
* - 'embedBatch' — Batch embedding engine (texts[] → vectors[])
|
* - 'embedBatch' — Batch embedding engine (texts[] → vectors[])
|
||||||
|
|
@ -88,7 +88,7 @@ export interface BrainyPluginContext {
|
||||||
// (Cortex) that declares `implements MetadataIndexProvider` gets a compile
|
// (Cortex) that declares `implements MetadataIndexProvider` gets a compile
|
||||||
// error the moment a method Brainy depends on is dropped or its signature
|
// error the moment a method Brainy depends on is dropped or its signature
|
||||||
// drifts. Brainy's own baseline classes (`MetadataIndexManager`,
|
// drifts. Brainy's own baseline classes (`MetadataIndexManager`,
|
||||||
// `GraphAdjacencyIndex`, `HNSWIndex`, `EntityIdMapper`, `UnifiedCache`)
|
// `GraphAdjacencyIndex`, `JsHnswVectorIndex`, `EntityIdMapper`, `UnifiedCache`)
|
||||||
// implement them too, so the contract can never silently diverge from the
|
// implement them too, so the contract can never silently diverge from the
|
||||||
// thing Brainy ships.
|
// thing Brainy ships.
|
||||||
//
|
//
|
||||||
|
|
@ -281,11 +281,11 @@ export interface CacheProvider {
|
||||||
* connection lists as compact delta-varint byte sequences (cortex's
|
* connection lists as compact delta-varint byte sequences (cortex's
|
||||||
* `encodeConnections` / `decodeConnections`).
|
* `encodeConnections` / `decodeConnections`).
|
||||||
*
|
*
|
||||||
* Brainy's `HNSWIndex` consumes this via a `ConnectionsCodec` that translates
|
* Brainy's `JsHnswVectorIndex` consumes this via a `ConnectionsCodec` that translates
|
||||||
* UUIDs to stable int slots via the `EntityIdMapper`, encodes, and persists
|
* UUIDs to stable int slots via the `EntityIdMapper`, encodes, and persists
|
||||||
* the compressed bytes through the binary-blob primitive. On load, the blob
|
* the compressed bytes through the binary-blob primitive. On load, the blob
|
||||||
* is fetched + decoded back into UUID sets — `setConnectionsCodec()` on
|
* is fetched + decoded back into UUID sets — `setConnectionsCodec()` on
|
||||||
* `HNSWIndex` is the injection point. Read path is dual-format: when no blob
|
* `JsHnswVectorIndex` is the injection point. Read path is dual-format: when no blob
|
||||||
* exists for a node, the connections fall back to the legacy JSON-array path
|
* exists for a node, the connections fall back to the legacy JSON-array path
|
||||||
* embedded in `saveHNSWData`, so pre-2.4.0 indexes keep loading unchanged
|
* embedded in `saveHNSWData`, so pre-2.4.0 indexes keep loading unchanged
|
||||||
* and convergence to the compressed form happens lazily on next save.
|
* and convergence to the compressed form happens lazily on next save.
|
||||||
|
|
@ -304,7 +304,7 @@ export interface GraphCompressionProvider {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed
|
* The `'vectorStore:mmap'` provider — a static factory class for an mmap-backed
|
||||||
* vector file. Brainy's HNSWIndex uses the static `.create()` / `.open()`
|
* vector file. Brainy's JsHnswVectorIndex uses the static `.create()` / `.open()`
|
||||||
* factories to open a single-file store at a derived path on disk-resident
|
* factories to open a single-file store at a derived path on disk-resident
|
||||||
* storage adapters; non-FS adapters with no local-path support skip the mmap
|
* storage adapters; non-FS adapters with no local-path support skip the mmap
|
||||||
* layer and fall back to per-entity vector reads.
|
* layer and fall back to per-entity vector reads.
|
||||||
|
|
@ -325,7 +325,7 @@ export interface VectorStoreMmapProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An open mmap-vector-store instance — the surface brainy's HNSWIndex consumes
|
* An open mmap-vector-store instance — the surface brainy's JsHnswVectorIndex consumes
|
||||||
* for batch vector reads + lazy migration from per-entity storage on miss.
|
* for batch vector reads + lazy migration from per-entity storage on miss.
|
||||||
*/
|
*/
|
||||||
export interface VectorStoreMmapInstance {
|
export interface VectorStoreMmapInstance {
|
||||||
|
|
|
||||||
|
|
@ -2,14 +2,14 @@
|
||||||
* Index Operations with Rollback Support
|
* Index Operations with Rollback Support
|
||||||
*
|
*
|
||||||
* Provides transactional operations for all indexes:
|
* Provides transactional operations for all indexes:
|
||||||
* - HNSWIndex (unified vector index)
|
* - JsHnswVectorIndex (unified vector index)
|
||||||
* - MetadataIndexManager (roaring bitmap filtering)
|
* - MetadataIndexManager (roaring bitmap filtering)
|
||||||
* - GraphAdjacencyIndex (LSM-tree graph storage)
|
* - GraphAdjacencyIndex (LSM-tree graph storage)
|
||||||
*
|
*
|
||||||
* Each operation can be executed and rolled back atomically.
|
* Each operation can be executed and rolled back atomically.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { HNSWIndex } from '../../hnsw/hnswIndex.js'
|
import type { JsHnswVectorIndex } from '../../hnsw/hnswIndex.js'
|
||||||
import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
|
import type { MetadataIndexManager } from '../../utils/metadataIndex.js'
|
||||||
import type { GraphAdjacencyIndex } from '../../graph/graphAdjacencyIndex.js'
|
import type { GraphAdjacencyIndex } from '../../graph/graphAdjacencyIndex.js'
|
||||||
import type { GraphVerb } from '../../coreTypes.js'
|
import type { GraphVerb } from '../../coreTypes.js'
|
||||||
|
|
@ -21,13 +21,13 @@ import type { Operation, RollbackAction } from '../types.js'
|
||||||
* Rollback strategy:
|
* Rollback strategy:
|
||||||
* - Remove item from index
|
* - Remove item from index
|
||||||
*
|
*
|
||||||
* Note: Works with both HNSWIndex and TypeAwareHNSWIndex
|
* Note: Works with both JsHnswVectorIndex and TypeAwareHNSWIndex
|
||||||
*/
|
*/
|
||||||
export class AddToHNSWOperation implements Operation {
|
export class AddToHNSWOperation implements Operation {
|
||||||
readonly name = 'AddToHNSW'
|
readonly name = 'AddToHNSW'
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly index: HNSWIndex,
|
private readonly index: JsHnswVectorIndex,
|
||||||
private readonly id: string,
|
private readonly id: string,
|
||||||
private readonly vector: number[]
|
private readonly vector: number[]
|
||||||
) {}
|
) {}
|
||||||
|
|
@ -76,7 +76,7 @@ export class RemoveFromHNSWOperation implements Operation {
|
||||||
readonly name = 'RemoveFromHNSW'
|
readonly name = 'RemoveFromHNSW'
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly index: HNSWIndex,
|
private readonly index: JsHnswVectorIndex,
|
||||||
private readonly id: string,
|
private readonly id: string,
|
||||||
private readonly vector: number[] // Required for rollback
|
private readonly vector: number[] // Required for rollback
|
||||||
) {}
|
) {}
|
||||||
|
|
@ -211,7 +211,7 @@ export class BatchAddToHNSWOperation implements Operation {
|
||||||
private operations: AddToHNSWOperation[]
|
private operations: AddToHNSWOperation[]
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
index: HNSWIndex,
|
index: JsHnswVectorIndex,
|
||||||
items: Array<{ id: string; vector: number[] }>
|
items: Array<{ id: string; vector: number[] }>
|
||||||
) {
|
) {
|
||||||
this.operations = items.map(
|
this.operations = items.map(
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@
|
||||||
* - Fusion: O(k log k) where k = result count
|
* - Fusion: O(k log k) where k = result count
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { HNSWIndex } from '../hnsw/hnswIndex.js'
|
import { JsHnswVectorIndex } from '../hnsw/hnswIndex.js'
|
||||||
import { MetadataIndexManager } from '../utils/metadataIndex.js'
|
import { MetadataIndexManager } from '../utils/metadataIndex.js'
|
||||||
import { Vector } from '../coreTypes.js'
|
import { Vector } from '../coreTypes.js'
|
||||||
import { NounType } from '../types/graphTypes.js'
|
import { NounType } from '../types/graphTypes.js'
|
||||||
|
|
@ -230,7 +230,7 @@ class QueryPlanner {
|
||||||
*/
|
*/
|
||||||
export class TripleIntelligenceSystem {
|
export class TripleIntelligenceSystem {
|
||||||
private metadataIndex: MetadataIndexManager
|
private metadataIndex: MetadataIndexManager
|
||||||
private hnswIndex: HNSWIndex
|
private hnswIndex: JsHnswVectorIndex
|
||||||
private graphIndex: GraphAdjacencyIndex
|
private graphIndex: GraphAdjacencyIndex
|
||||||
private metrics: PerformanceMetrics
|
private metrics: PerformanceMetrics
|
||||||
private planner: QueryPlanner
|
private planner: QueryPlanner
|
||||||
|
|
@ -239,7 +239,7 @@ export class TripleIntelligenceSystem {
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
metadataIndex: MetadataIndexManager,
|
metadataIndex: MetadataIndexManager,
|
||||||
hnswIndex: HNSWIndex,
|
hnswIndex: JsHnswVectorIndex,
|
||||||
graphIndex: GraphAdjacencyIndex,
|
graphIndex: GraphAdjacencyIndex,
|
||||||
embedder: (text: string) => Promise<Vector>,
|
embedder: (text: string) => Promise<Vector>,
|
||||||
storage: any
|
storage: any
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@
|
||||||
import { prodLog } from './logger.js'
|
import { prodLog } from './logger.js'
|
||||||
import { DELETED_FIELD, isDeleted } from './metadataNamespace.js'
|
import { DELETED_FIELD, isDeleted } from './metadataNamespace.js'
|
||||||
import type { StorageAdapter } from '../coreTypes.js'
|
import type { StorageAdapter } from '../coreTypes.js'
|
||||||
import type { HNSWIndex } from '../hnsw/hnswIndex.js'
|
import type { JsHnswVectorIndex } from '../hnsw/hnswIndex.js'
|
||||||
import type { MetadataIndexManager } from './metadataIndex.js'
|
import type { MetadataIndexManager } from './metadataIndex.js'
|
||||||
|
|
||||||
export interface CleanupConfig {
|
export interface CleanupConfig {
|
||||||
|
|
@ -44,7 +44,7 @@ export interface CleanupStats {
|
||||||
*/
|
*/
|
||||||
export class PeriodicCleanup {
|
export class PeriodicCleanup {
|
||||||
private storage: StorageAdapter
|
private storage: StorageAdapter
|
||||||
private hnswIndex: HNSWIndex
|
private hnswIndex: JsHnswVectorIndex
|
||||||
private metadataIndex: MetadataIndexManager | null
|
private metadataIndex: MetadataIndexManager | null
|
||||||
private config: CleanupConfig
|
private config: CleanupConfig
|
||||||
private stats: CleanupStats
|
private stats: CleanupStats
|
||||||
|
|
@ -53,7 +53,7 @@ export class PeriodicCleanup {
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
storage: StorageAdapter,
|
storage: StorageAdapter,
|
||||||
hnswIndex: HNSWIndex,
|
hnswIndex: JsHnswVectorIndex,
|
||||||
metadataIndex: MetadataIndexManager | null,
|
metadataIndex: MetadataIndexManager | null,
|
||||||
config: Partial<CleanupConfig> = {}
|
config: Partial<CleanupConfig> = {}
|
||||||
) {
|
) {
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@
|
||||||
|
|
||||||
import { describe, it, expect, beforeEach } from 'vitest'
|
import { describe, it, expect, beforeEach } from 'vitest'
|
||||||
import { v4 as uuidv4 } from 'uuid'
|
import { v4 as uuidv4 } from 'uuid'
|
||||||
import { HNSWIndex } from '../../../src/hnsw/hnswIndex.js'
|
import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js'
|
||||||
import { euclideanDistance } from '../../../src/utils/index.js'
|
import { euclideanDistance } from '../../../src/utils/index.js'
|
||||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||||
|
|
||||||
|
|
@ -43,12 +43,12 @@ describe('Lazy Vector Loading (B2)', () => {
|
||||||
// 1. LAZY MODE: VECTOR EVICTION
|
// 1. LAZY MODE: VECTOR EVICTION
|
||||||
// =================================================================
|
// =================================================================
|
||||||
describe('vector eviction in lazy mode', () => {
|
describe('vector eviction in lazy mode', () => {
|
||||||
let index: HNSWIndex
|
let index: JsHnswVectorIndex
|
||||||
let storage: MemoryStorage
|
let storage: MemoryStorage
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
storage = new MemoryStorage()
|
storage = new MemoryStorage()
|
||||||
index = new HNSWIndex(
|
index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 8,
|
M: 8,
|
||||||
efConstruction: 100,
|
efConstruction: 100,
|
||||||
|
|
@ -116,7 +116,7 @@ describe('Lazy Vector Loading (B2)', () => {
|
||||||
describe('memory mode retains vectors (default)', () => {
|
describe('memory mode retains vectors (default)', () => {
|
||||||
it('should keep vectors in memory by default', async () => {
|
it('should keep vectors in memory by default', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{ M: 4, efConstruction: 50, efSearch: 20 },
|
{ M: 4, efConstruction: 50, efSearch: 20 },
|
||||||
euclideanDistance,
|
euclideanDistance,
|
||||||
{ useParallelization: false, storage }
|
{ useParallelization: false, storage }
|
||||||
|
|
@ -135,7 +135,7 @@ describe('Lazy Vector Loading (B2)', () => {
|
||||||
|
|
||||||
it('should work without storage adapter in memory mode', async () => {
|
it('should work without storage adapter in memory mode', async () => {
|
||||||
// No storage adapter provided
|
// No storage adapter provided
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{ M: 4, efConstruction: 50, efSearch: 20 },
|
{ M: 4, efConstruction: 50, efSearch: 20 },
|
||||||
euclideanDistance,
|
euclideanDistance,
|
||||||
{ useParallelization: false }
|
{ useParallelization: false }
|
||||||
|
|
@ -157,7 +157,7 @@ describe('Lazy Vector Loading (B2)', () => {
|
||||||
describe('lazy mode without storage', () => {
|
describe('lazy mode without storage', () => {
|
||||||
it('should not evict vectors when no storage adapter is configured', async () => {
|
it('should not evict vectors when no storage adapter is configured', async () => {
|
||||||
// vectorStorage: 'lazy' but no storage — vectors should stay in memory
|
// vectorStorage: 'lazy' but no storage — vectors should stay in memory
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 4,
|
M: 4,
|
||||||
efConstruction: 50,
|
efConstruction: 50,
|
||||||
|
|
@ -185,7 +185,7 @@ describe('Lazy Vector Loading (B2)', () => {
|
||||||
describe('lazy mode with quantization', () => {
|
describe('lazy mode with quantization', () => {
|
||||||
it('should use SQ8 for traversal and load full vectors for rerank', async () => {
|
it('should use SQ8 for traversal and load full vectors for rerank', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 8,
|
M: 8,
|
||||||
efConstruction: 100,
|
efConstruction: 100,
|
||||||
|
|
@ -221,7 +221,7 @@ describe('Lazy Vector Loading (B2)', () => {
|
||||||
|
|
||||||
it('should return results sorted by exact distance after rerank', async () => {
|
it('should return results sorted by exact distance after rerank', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 8,
|
M: 8,
|
||||||
efConstruction: 100,
|
efConstruction: 100,
|
||||||
|
|
@ -257,7 +257,7 @@ describe('Lazy Vector Loading (B2)', () => {
|
||||||
describe('multiple searches in lazy mode', () => {
|
describe('multiple searches in lazy mode', () => {
|
||||||
it('should handle repeated searches correctly', async () => {
|
it('should handle repeated searches correctly', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 8,
|
M: 8,
|
||||||
efConstruction: 100,
|
efConstruction: 100,
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ import {
|
||||||
serializeSQ8,
|
serializeSQ8,
|
||||||
deserializeSQ8
|
deserializeSQ8
|
||||||
} from '../../../src/utils/vectorQuantization.js'
|
} from '../../../src/utils/vectorQuantization.js'
|
||||||
import { HNSWIndex } from '../../../src/hnsw/hnswIndex.js'
|
import { JsHnswVectorIndex } from '../../../src/hnsw/hnswIndex.js'
|
||||||
import { euclideanDistance } from '../../../src/utils/index.js'
|
import { euclideanDistance } from '../../../src/utils/index.js'
|
||||||
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
import { MemoryStorage } from '../../../src/storage/adapters/memoryStorage.js'
|
||||||
|
|
||||||
|
|
@ -286,14 +286,14 @@ describe('SQ8 Quantization', () => {
|
||||||
// 4. HNSW SEARCH WITH QUANTIZATION
|
// 4. HNSW SEARCH WITH QUANTIZATION
|
||||||
// =================================================================
|
// =================================================================
|
||||||
describe('HNSW search with quantization', () => {
|
describe('HNSW search with quantization', () => {
|
||||||
let index: HNSWIndex
|
let index: JsHnswVectorIndex
|
||||||
let storage: MemoryStorage
|
let storage: MemoryStorage
|
||||||
const dim = 32 // Small dimension for fast tests
|
const dim = 32 // Small dimension for fast tests
|
||||||
let ids: string[]
|
let ids: string[]
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
storage = new MemoryStorage()
|
storage = new MemoryStorage()
|
||||||
index = new HNSWIndex(
|
index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 8,
|
M: 8,
|
||||||
efConstruction: 100,
|
efConstruction: 100,
|
||||||
|
|
@ -365,7 +365,7 @@ describe('SQ8 Quantization', () => {
|
||||||
describe('two-phase rerank', () => {
|
describe('two-phase rerank', () => {
|
||||||
it('should accept rerank options in search', async () => {
|
it('should accept rerank options in search', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 8,
|
M: 8,
|
||||||
efConstruction: 100,
|
efConstruction: 100,
|
||||||
|
|
@ -399,7 +399,7 @@ describe('SQ8 Quantization', () => {
|
||||||
|
|
||||||
it('reranked results should be sorted by exact distance', async () => {
|
it('reranked results should be sorted by exact distance', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 8,
|
M: 8,
|
||||||
efConstruction: 100,
|
efConstruction: 100,
|
||||||
|
|
@ -432,7 +432,7 @@ describe('SQ8 Quantization', () => {
|
||||||
describe('configuration defaults', () => {
|
describe('configuration defaults', () => {
|
||||||
it('should disable quantization by default', async () => {
|
it('should disable quantization by default', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const defaultIndex = new HNSWIndex(
|
const defaultIndex = new JsHnswVectorIndex(
|
||||||
{ M: 4, efConstruction: 50, efSearch: 20 },
|
{ M: 4, efConstruction: 50, efSearch: 20 },
|
||||||
euclideanDistance,
|
euclideanDistance,
|
||||||
{ useParallelization: false, storage }
|
{ useParallelization: false, storage }
|
||||||
|
|
@ -456,7 +456,7 @@ describe('SQ8 Quantization', () => {
|
||||||
|
|
||||||
it('should use default rerankMultiplier of 3', async () => {
|
it('should use default rerankMultiplier of 3', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{
|
{
|
||||||
M: 4,
|
M: 4,
|
||||||
efConstruction: 50,
|
efConstruction: 50,
|
||||||
|
|
@ -480,7 +480,7 @@ describe('SQ8 Quantization', () => {
|
||||||
|
|
||||||
it('should default vectorStorage to memory mode', async () => {
|
it('should default vectorStorage to memory mode', async () => {
|
||||||
const storage = new MemoryStorage()
|
const storage = new MemoryStorage()
|
||||||
const index = new HNSWIndex(
|
const index = new JsHnswVectorIndex(
|
||||||
{ M: 4, efConstruction: 50, efSearch: 20 },
|
{ M: 4, efConstruction: 50, efSearch: 20 },
|
||||||
euclideanDistance,
|
euclideanDistance,
|
||||||
{ useParallelization: false, storage }
|
{ useParallelization: false, storage }
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue