diff --git a/src/coreTypes.ts b/src/coreTypes.ts index d6b7a9e6..c5389c6d 100644 --- a/src/coreTypes.ts +++ b/src/coreTypes.ts @@ -369,6 +369,15 @@ export interface GraphVerb { * HNSW index configuration */ export interface HNSWConfig { + /** + * Index engine selector. Defaults to auto-detect: cortex's DiskANN + * engages when registered + storage is local + a stable idMapper is + * available (ADR-002); otherwise HNSW. Set explicitly to override: + * - `'hnsw'` — force the in-memory HNSW index (the historical default). + * - `'diskann'` — require cortex's DiskANN; throw if conditions + * aren't met rather than silently falling back. + */ + type?: 'hnsw' | 'diskann' M: number // Maximum number of connections per noun efConstruction: number // Size of the dynamic candidate list during construction efSearch: number // Size of the dynamic candidate list during search @@ -383,6 +392,29 @@ export interface HNSWConfig { } // Vector storage mode vectorStorage?: 'memory' | 'lazy' // default: 'memory' — 'lazy' evicts vectors after insert + /** + * DiskANN-specific tuning. Only consulted when `type === 'diskann'` + * or auto-engagement selects DiskANN. Sensible defaults match the + * published Vamana paper (R=64, L=100, α=1.2, M=16, ksub=256). + */ + diskann?: { + /** PQ subspaces. dim must be divisible by m. Default 16. */ + pqM?: number + /** Centroids per subspace. Default 256 (8-bit codes). */ + pqKsub?: number + /** Vamana max degree (R). Default 64. */ + maxDegree?: number + /** Build-time candidate list size (L). Default 100. */ + searchListSize?: number + /** α-pruning density factor. Default 1.2. */ + alpha?: number + /** Use file-backed adjacency during build. Required >~100M nodes. */ + useMmapAdjacency?: boolean + /** Scratch file path when useMmapAdjacency is true. */ + mmapAdjacencyPath?: string + /** Index file path. Defaults to `/_diskann/main.bin`. */ + indexPath?: string + } } /** diff --git a/src/plugin.ts b/src/plugin.ts index 89e2bed4..4145daf3 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -216,6 +216,31 @@ export interface HnswProvider { getPersistMode(): 'immediate' | 'deferred' } +/** + * The `'diskann'` provider — a billion-scale alternative to HNSW backed + * by cortex's pure-Rust Vamana + Product Quantization implementation + * (see ADR-002 in the cortex repo). + * + * Structurally a drop-in for [`HnswProvider`]: brainy calls the same + * `addItem` / `search` / `rebuild` surface and never needs to know + * which index is underneath. The differences are operational: + * + * - **Memory footprint**: ~16 GB RAM at 1 B vectors (PQ codes only). + * HNSW would need ~1.5 TB to keep the full vectors resident. + * - **Build model**: build-once, query-many. Dynamic insertions buffer + * to an in-memory delta brute-forced alongside the main index; + * `rebuild()` folds them in. + * - **Storage**: requires a local filesystem path (NVMe SSD for the + * published latency numbers). Cloud-storage adapters continue using + * HNSW. + * + * Engagement is automatic when the cortex provider is registered, the + * storage adapter exposes `getBinaryBlobPath`, and the metadata index + * has a stable `idMapper`. Users force the legacy index via + * `config.index.type = 'hnsw'`. + */ +export type DiskAnnProvider = HnswProvider + /** * The `'entityIdMapper'` provider — a drop-in for `EntityIdMapper`. Injected * into the TypeScript `MetadataIndexManager` when a native metadata index is