feat(plugin): DiskAnnProvider contract + HNSWConfig.type/diskann knobs
Adds the plugin-side surface for the new billion-scale index option:
- plugin.ts: DiskAnnProvider type alias (mirrors HnswProvider's shape
so cortex's DiskANN wrapper is a drop-in for the 'hnsw' factory).
- coreTypes.ts: HNSWConfig.type ('hnsw' | 'diskann') for explicit
engine selection, plus HNSWConfig.diskann for the build-time tuning
knobs (pqM, pqKsub, maxDegree, searchListSize, alpha, mmap
adjacency).
No algorithm code here — the implementation lives in the cortex
plugin. Brainy without cortex continues to use HNSW exactly as before.
This commit is contained in:
parent
4927d49e87
commit
f885f813fe
2 changed files with 57 additions and 0 deletions
|
|
@ -369,6 +369,15 @@ export interface GraphVerb {
|
|||
* HNSW index configuration
|
||||
*/
|
||||
export interface HNSWConfig {
|
||||
/**
|
||||
* Index engine selector. Defaults to auto-detect: cortex's DiskANN
|
||||
* engages when registered + storage is local + a stable idMapper is
|
||||
* available (ADR-002); otherwise HNSW. Set explicitly to override:
|
||||
* - `'hnsw'` — force the in-memory HNSW index (the historical default).
|
||||
* - `'diskann'` — require cortex's DiskANN; throw if conditions
|
||||
* aren't met rather than silently falling back.
|
||||
*/
|
||||
type?: 'hnsw' | 'diskann'
|
||||
M: number // Maximum number of connections per noun
|
||||
efConstruction: number // Size of the dynamic candidate list during construction
|
||||
efSearch: number // Size of the dynamic candidate list during search
|
||||
|
|
@ -383,6 +392,29 @@ export interface HNSWConfig {
|
|||
}
|
||||
// Vector storage mode
|
||||
vectorStorage?: 'memory' | 'lazy' // default: 'memory' — 'lazy' evicts vectors after insert
|
||||
/**
|
||||
* DiskANN-specific tuning. Only consulted when `type === 'diskann'`
|
||||
* or auto-engagement selects DiskANN. Sensible defaults match the
|
||||
* published Vamana paper (R=64, L=100, α=1.2, M=16, ksub=256).
|
||||
*/
|
||||
diskann?: {
|
||||
/** PQ subspaces. dim must be divisible by m. Default 16. */
|
||||
pqM?: number
|
||||
/** Centroids per subspace. Default 256 (8-bit codes). */
|
||||
pqKsub?: number
|
||||
/** Vamana max degree (R). Default 64. */
|
||||
maxDegree?: number
|
||||
/** Build-time candidate list size (L). Default 100. */
|
||||
searchListSize?: number
|
||||
/** α-pruning density factor. Default 1.2. */
|
||||
alpha?: number
|
||||
/** Use file-backed adjacency during build. Required >~100M nodes. */
|
||||
useMmapAdjacency?: boolean
|
||||
/** Scratch file path when useMmapAdjacency is true. */
|
||||
mmapAdjacencyPath?: string
|
||||
/** Index file path. Defaults to `<storageRoot>/_diskann/main.bin`. */
|
||||
indexPath?: string
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -216,6 +216,31 @@ export interface HnswProvider {
|
|||
getPersistMode(): 'immediate' | 'deferred'
|
||||
}
|
||||
|
||||
/**
|
||||
* The `'diskann'` provider — a billion-scale alternative to HNSW backed
|
||||
* by cortex's pure-Rust Vamana + Product Quantization implementation
|
||||
* (see ADR-002 in the cortex repo).
|
||||
*
|
||||
* Structurally a drop-in for [`HnswProvider`]: brainy calls the same
|
||||
* `addItem` / `search` / `rebuild` surface and never needs to know
|
||||
* which index is underneath. The differences are operational:
|
||||
*
|
||||
* - **Memory footprint**: ~16 GB RAM at 1 B vectors (PQ codes only).
|
||||
* HNSW would need ~1.5 TB to keep the full vectors resident.
|
||||
* - **Build model**: build-once, query-many. Dynamic insertions buffer
|
||||
* to an in-memory delta brute-forced alongside the main index;
|
||||
* `rebuild()` folds them in.
|
||||
* - **Storage**: requires a local filesystem path (NVMe SSD for the
|
||||
* published latency numbers). Cloud-storage adapters continue using
|
||||
* HNSW.
|
||||
*
|
||||
* Engagement is automatic when the cortex provider is registered, the
|
||||
* storage adapter exposes `getBinaryBlobPath`, and the metadata index
|
||||
* has a stable `idMapper`. Users force the legacy index via
|
||||
* `config.index.type = 'hnsw'`.
|
||||
*/
|
||||
export type DiskAnnProvider = HnswProvider
|
||||
|
||||
/**
|
||||
* The `'entityIdMapper'` provider — a drop-in for `EntityIdMapper`. Injected
|
||||
* into the TypeScript `MetadataIndexManager` when a native metadata index is
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue