feat(plugin): DiskAnnProvider contract + HNSWConfig.type/diskann knobs
Adds the plugin-side surface for the new billion-scale index option:
- plugin.ts: DiskAnnProvider type alias (mirrors HnswProvider's shape
so cortex's DiskANN wrapper is a drop-in for the 'hnsw' factory).
- coreTypes.ts: HNSWConfig.type ('hnsw' | 'diskann') for explicit
engine selection, plus HNSWConfig.diskann for the build-time tuning
knobs (pqM, pqKsub, maxDegree, searchListSize, alpha, mmap
adjacency).
No algorithm code here — the implementation lives in the cortex
plugin. Brainy without cortex continues to use HNSW exactly as before.
This commit is contained in:
parent
4927d49e87
commit
f885f813fe
2 changed files with 57 additions and 0 deletions
|
|
@ -369,6 +369,15 @@ export interface GraphVerb {
|
||||||
* HNSW index configuration
|
* HNSW index configuration
|
||||||
*/
|
*/
|
||||||
export interface HNSWConfig {
|
export interface HNSWConfig {
|
||||||
|
/**
|
||||||
|
* Index engine selector. Defaults to auto-detect: cortex's DiskANN
|
||||||
|
* engages when registered + storage is local + a stable idMapper is
|
||||||
|
* available (ADR-002); otherwise HNSW. Set explicitly to override:
|
||||||
|
* - `'hnsw'` — force the in-memory HNSW index (the historical default).
|
||||||
|
* - `'diskann'` — require cortex's DiskANN; throw if conditions
|
||||||
|
* aren't met rather than silently falling back.
|
||||||
|
*/
|
||||||
|
type?: 'hnsw' | 'diskann'
|
||||||
M: number // Maximum number of connections per noun
|
M: number // Maximum number of connections per noun
|
||||||
efConstruction: number // Size of the dynamic candidate list during construction
|
efConstruction: number // Size of the dynamic candidate list during construction
|
||||||
efSearch: number // Size of the dynamic candidate list during search
|
efSearch: number // Size of the dynamic candidate list during search
|
||||||
|
|
@ -383,6 +392,29 @@ export interface HNSWConfig {
|
||||||
}
|
}
|
||||||
// Vector storage mode
|
// Vector storage mode
|
||||||
vectorStorage?: 'memory' | 'lazy' // default: 'memory' — 'lazy' evicts vectors after insert
|
vectorStorage?: 'memory' | 'lazy' // default: 'memory' — 'lazy' evicts vectors after insert
|
||||||
|
/**
|
||||||
|
* DiskANN-specific tuning. Only consulted when `type === 'diskann'`
|
||||||
|
* or auto-engagement selects DiskANN. Sensible defaults match the
|
||||||
|
* published Vamana paper (R=64, L=100, α=1.2, M=16, ksub=256).
|
||||||
|
*/
|
||||||
|
diskann?: {
|
||||||
|
/** PQ subspaces. dim must be divisible by m. Default 16. */
|
||||||
|
pqM?: number
|
||||||
|
/** Centroids per subspace. Default 256 (8-bit codes). */
|
||||||
|
pqKsub?: number
|
||||||
|
/** Vamana max degree (R). Default 64. */
|
||||||
|
maxDegree?: number
|
||||||
|
/** Build-time candidate list size (L). Default 100. */
|
||||||
|
searchListSize?: number
|
||||||
|
/** α-pruning density factor. Default 1.2. */
|
||||||
|
alpha?: number
|
||||||
|
/** Use file-backed adjacency during build. Required >~100M nodes. */
|
||||||
|
useMmapAdjacency?: boolean
|
||||||
|
/** Scratch file path when useMmapAdjacency is true. */
|
||||||
|
mmapAdjacencyPath?: string
|
||||||
|
/** Index file path. Defaults to `<storageRoot>/_diskann/main.bin`. */
|
||||||
|
indexPath?: string
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -216,6 +216,31 @@ export interface HnswProvider {
|
||||||
getPersistMode(): 'immediate' | 'deferred'
|
getPersistMode(): 'immediate' | 'deferred'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The `'diskann'` provider — a billion-scale alternative to HNSW backed
|
||||||
|
* by cortex's pure-Rust Vamana + Product Quantization implementation
|
||||||
|
* (see ADR-002 in the cortex repo).
|
||||||
|
*
|
||||||
|
* Structurally a drop-in for [`HnswProvider`]: brainy calls the same
|
||||||
|
* `addItem` / `search` / `rebuild` surface and never needs to know
|
||||||
|
* which index is underneath. The differences are operational:
|
||||||
|
*
|
||||||
|
* - **Memory footprint**: ~16 GB RAM at 1 B vectors (PQ codes only).
|
||||||
|
* HNSW would need ~1.5 TB to keep the full vectors resident.
|
||||||
|
* - **Build model**: build-once, query-many. Dynamic insertions buffer
|
||||||
|
* to an in-memory delta brute-forced alongside the main index;
|
||||||
|
* `rebuild()` folds them in.
|
||||||
|
* - **Storage**: requires a local filesystem path (NVMe SSD for the
|
||||||
|
* published latency numbers). Cloud-storage adapters continue using
|
||||||
|
* HNSW.
|
||||||
|
*
|
||||||
|
* Engagement is automatic when the cortex provider is registered, the
|
||||||
|
* storage adapter exposes `getBinaryBlobPath`, and the metadata index
|
||||||
|
* has a stable `idMapper`. Users force the legacy index via
|
||||||
|
* `config.index.type = 'hnsw'`.
|
||||||
|
*/
|
||||||
|
export type DiskAnnProvider = HnswProvider
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The `'entityIdMapper'` provider — a drop-in for `EntityIdMapper`. Injected
|
* The `'entityIdMapper'` provider — a drop-in for `EntityIdMapper`. Injected
|
||||||
* into the TypeScript `MetadataIndexManager` when a native metadata index is
|
* into the TypeScript `MetadataIndexManager` when a native metadata index is
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue