fix(8.0): createIndex resolves the canonical 'vector' provider key — drop diskann/hnsw key lookups + legacy migration APIs
createIndex() still resolved the retired 'diskann' and 'hnsw' provider
keys and never looked up 'vector', so an 8.0-era plugin registering its
vector engine under the canonical 'vector' key silently never engaged
and Brainy always fell back to the built-in JS HNSW index. createIndex()
now resolves only getProvider('vector') (same factory call shape as the
old hnsw path) and falls back to setupIndex().
The brainy-side mode/migration machinery is obsolete in the 8.0 provider
world — the registered engine adapts internally (in-memory / hybrid /
on-disk selection is the provider's job, not Brainy's):
- delete migrateToDiskAnn() / migrateToHnsw() and the ADR-002
index-engine migration block
- delete diskAnnAutoEngageConditionsMet() / instantiateDiskAnn()
- narrow HNSWConfig.type to 'vector' and drop the diskann tuning block
(coreTypes.ts)
- well-known provider key lists + diagnostics now report 'vector'
instead of the never-registered 'hnsw' key
- plugin.ts docs: 'vector' is the only vector-index key consulted; the
pre-8.0 'hnsw'/'diskann' keys are retired
The schema-migration machinery (migrate(), checkMigrations(),
MigrationRunner, autoMigrate) serves data migrations and is untouched.
This commit is contained in:
parent
2427bb7960
commit
49e49483d1
3 changed files with 22 additions and 290 deletions
269
src/brainy.ts
269
src/brainy.ts
|
|
@ -659,7 +659,7 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
if (this.pluginRegistry.hasActivePlugins() && !this.config.silent) {
|
||||
const wellKnownKeys = [
|
||||
'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache',
|
||||
'hnsw', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack'
|
||||
'vector', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack'
|
||||
]
|
||||
const native = wellKnownKeys.filter(k => this.pluginRegistry.hasProvider(k))
|
||||
const fallback = wellKnownKeys.filter(k => !this.pluginRegistry.hasProvider(k))
|
||||
|
|
@ -4861,169 +4861,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return this.migrateInternal(runner, options)
|
||||
}
|
||||
|
||||
// ─── Index engine migration (ADR-002) ────────────────────────────
|
||||
|
||||
/**
|
||||
* Convert the current HNSW index to DiskANN for billion-scale
|
||||
* search.
|
||||
*
|
||||
* The migration runs in three phases:
|
||||
* 1. **Build**: stream every live noun's vector into a new DiskANN
|
||||
* file at the configured path. Existing HNSW continues serving
|
||||
* queries until the swap.
|
||||
* 2. **Verify**: sample queries against both indexes; require recall
|
||||
* parity within the configured threshold before swapping.
|
||||
* 3. **Swap**: atomically replace `this.index` with the DiskANN
|
||||
* wrapper.
|
||||
*
|
||||
* Reversible via {@link migrateToHnsw}. Both paths preserve the
|
||||
* underlying canonical storage (entity JSON, metadata index) — only
|
||||
* the search engine changes.
|
||||
*
|
||||
* @param options - Migration tuning. Defaults match ADR-002.
|
||||
* @throws If the `'diskann'` provider isn't registered (load cortex
|
||||
* as a plugin) or if recall verification fails.
|
||||
*
|
||||
* @example
|
||||
* ```typescript
|
||||
* await brain.migrateToDiskAnn({ recallTarget: 0.95, paddingFactor: 1.2 })
|
||||
* ```
|
||||
*/
|
||||
async migrateToDiskAnn(options?: {
|
||||
/** Minimum recall@10 vs HNSW required to accept the swap. Default 0.95. */
|
||||
recallTarget?: number
|
||||
/** PaddingFactor handed to DiskANN search at verify time. Default 1.2. */
|
||||
paddingFactor?: number
|
||||
/** Number of sampled queries used for recall verification. Default 100. */
|
||||
verifySampleSize?: number
|
||||
/** Build in parallel while old index serves queries. Default true. */
|
||||
parallel?: boolean
|
||||
}): Promise<{ recall: number; sampledQueries: number; newIndexPath: string }> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const recallTarget = options?.recallTarget ?? 0.95
|
||||
const paddingFactor = options?.paddingFactor ?? 1.2
|
||||
const sampleSize = options?.verifySampleSize ?? 100
|
||||
|
||||
const diskannFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('diskann')
|
||||
if (!diskannFactory) {
|
||||
throw new Error(
|
||||
'migrateToDiskAnn requires the cortex DiskANN provider. ' +
|
||||
'Install + load @soulcraft/cortex as a brainy plugin.'
|
||||
)
|
||||
}
|
||||
if (!this.diskAnnAutoEngageConditionsMet()) {
|
||||
throw new Error(
|
||||
'migrateToDiskAnn requires a local filesystem storage adapter ' +
|
||||
'and a stable idMapper. See ADR-002.'
|
||||
)
|
||||
}
|
||||
|
||||
const oldIndex = this.index
|
||||
const newIndex = this.instantiateDiskAnn(diskannFactory, this.resolveHNSWPersistMode())
|
||||
|
||||
// Phase 1: stream every live noun's vector into the new index's delta.
|
||||
let vectorCount = 0
|
||||
const idMapper = this.metadataIndex.getIdMapper?.()
|
||||
if (!idMapper) {
|
||||
throw new Error('migrateToDiskAnn: metadata index has no idMapper')
|
||||
}
|
||||
for (const intId of idMapper.getAllIntIds()) {
|
||||
const uuid = idMapper.getUuid(intId)
|
||||
if (!uuid) continue
|
||||
const noun = await this.storage.getNoun(uuid)
|
||||
const vec = noun?.vector
|
||||
if (!vec) continue
|
||||
await newIndex.addItem({ id: uuid, vector: vec } as any)
|
||||
vectorCount++
|
||||
}
|
||||
await newIndex.rebuild()
|
||||
|
||||
// Phase 2: recall verification. Sample random vectors from the live
|
||||
// set; for each, search both indexes at k=10 and compute Jaccard.
|
||||
let totalHits = 0
|
||||
let queriesRun = 0
|
||||
const allIds = idMapper.getAllIntIds()
|
||||
const stride = Math.max(1, Math.floor(allIds.length / sampleSize))
|
||||
for (let i = 0; i < allIds.length && queriesRun < sampleSize; i += stride) {
|
||||
const uuid = idMapper.getUuid(allIds[i])
|
||||
if (!uuid) continue
|
||||
const noun = await this.storage.getNoun(uuid)
|
||||
const vec = noun?.vector
|
||||
if (!vec) continue
|
||||
const truth = await oldIndex.search(vec, 10)
|
||||
const got = await newIndex.search(vec, 10, undefined, { rerank: { multiplier: paddingFactor } })
|
||||
const truthSet = new Set(truth.map(([id]) => id))
|
||||
const overlap = got.filter(([id]) => truthSet.has(id)).length
|
||||
totalHits += overlap / Math.max(1, truth.length)
|
||||
queriesRun++
|
||||
}
|
||||
const recall = queriesRun > 0 ? totalHits / queriesRun : 0
|
||||
|
||||
if (recall < recallTarget) {
|
||||
throw new Error(
|
||||
`migrateToDiskAnn aborted: recall ${recall.toFixed(3)} < target ${recallTarget}. ` +
|
||||
'Old HNSW index left in place. Retry with looser params (larger ' +
|
||||
'paddingFactor, larger searchListSize) or accept lower recall.'
|
||||
)
|
||||
}
|
||||
|
||||
// Phase 3: atomic swap.
|
||||
this.index = newIndex as any
|
||||
if (!this.config.silent) {
|
||||
console.log(
|
||||
`[brainy] migrated to DiskANN — ${vectorCount} vectors, ` +
|
||||
`recall=${recall.toFixed(3)} on ${queriesRun} sample queries`
|
||||
)
|
||||
}
|
||||
return {
|
||||
recall,
|
||||
sampledQueries: queriesRun,
|
||||
newIndexPath: (newIndex as any).config?.indexPath ?? '',
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reverse {@link migrateToDiskAnn}: rebuild the in-memory HNSW index
|
||||
* from the current vector set and swap it back in. Contract from
|
||||
* ADR-002: this is always available so users can roll back from
|
||||
* production issues.
|
||||
*/
|
||||
async migrateToHnsw(): Promise<{ vectorCount: number }> {
|
||||
await this.ensureInitialized()
|
||||
|
||||
const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw')
|
||||
const persistMode = this.resolveHNSWPersistMode()
|
||||
const newIndex = hnswFactory
|
||||
? hnswFactory(
|
||||
{ ...this.config.index, distanceFunction: this.distance },
|
||||
this.distance,
|
||||
{ storage: this.storage, persistMode }
|
||||
)
|
||||
: this.setupIndex()
|
||||
|
||||
let vectorCount = 0
|
||||
const idMapper = this.metadataIndex.getIdMapper?.()
|
||||
if (!idMapper) {
|
||||
throw new Error('migrateToHnsw: metadata index has no idMapper')
|
||||
}
|
||||
for (const intId of idMapper.getAllIntIds()) {
|
||||
const uuid = idMapper.getUuid(intId)
|
||||
if (!uuid) continue
|
||||
const noun = await this.storage.getNoun(uuid)
|
||||
const vec = noun?.vector
|
||||
if (!vec) continue
|
||||
await newIndex.addItem({ id: uuid, vector: vec } as any)
|
||||
vectorCount++
|
||||
}
|
||||
|
||||
this.index = newIndex as any
|
||||
if (!this.config.silent) {
|
||||
console.log(`[brainy] migrated to HNSW — ${vectorCount} vectors`)
|
||||
}
|
||||
return { vectorCount }
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for pending migrations during init().
|
||||
* Runs inline for small datasets if autoMigrate is enabled,
|
||||
|
|
@ -6465,14 +6302,14 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
* @example
|
||||
* ```typescript
|
||||
* const diag = brain.diagnostics()
|
||||
* console.log(diag.providers) // { hnsw: { source: 'plugin' }, ... }
|
||||
* console.log(diag.providers) // { vector: { source: 'plugin' }, ... }
|
||||
* console.log(diag.indexes.graph.wiredToStorage) // true
|
||||
* ```
|
||||
*/
|
||||
diagnostics(): DiagnosticsResult {
|
||||
const wellKnownKeys = [
|
||||
'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache',
|
||||
'hnsw', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack'
|
||||
'vector', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack'
|
||||
] as const
|
||||
|
||||
const providers: Record<string, { source: 'plugin' | 'default' }> = {}
|
||||
|
|
@ -9340,51 +9177,21 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
}
|
||||
|
||||
/**
|
||||
* Create the vector index, using plugin factories when available.
|
||||
* Create the vector index, using the plugin-provided engine when available.
|
||||
* Shared by init(), fork(), checkout(), and clear() to avoid duplication.
|
||||
*
|
||||
* Selection order (first match wins):
|
||||
* 1. `config.index.type === 'diskann'` (explicit opt-in):
|
||||
* require the `'diskann'` provider; throw if absent.
|
||||
* 2. Auto-engage DiskANN when ALL of these hold:
|
||||
* - `'diskann'` provider is registered.
|
||||
* - The storage adapter exposes a local path via
|
||||
* `getBinaryBlobPath('_diskann/main')`.
|
||||
* - The metadata index has a stable `idMapper`.
|
||||
* - `config.index.type !== 'hnsw'` (no explicit opt-out).
|
||||
* 3. `'hnsw'` provider if registered (cortex's native HNSW).
|
||||
* 4. Brainy's built-in TS JsHnswVectorIndex (the always-available fallback).
|
||||
*
|
||||
* See ADR-002 in the cortex repo for the engagement rationale.
|
||||
* 1. `'vector'` provider if registered — the canonical 8.0 provider key.
|
||||
* A native plugin registers one internally-adaptive engine here;
|
||||
* in-memory / hybrid / on-disk selection is the provider's job, not
|
||||
* Brainy's, so there is no engine-specific branching on this side.
|
||||
* 2. Brainy's built-in TS JsHnswVectorIndex (the always-available fallback).
|
||||
*/
|
||||
private createIndex(): JsHnswVectorIndex {
|
||||
const persistMode = this.resolveHNSWPersistMode()
|
||||
const indexType = (this.config.index as any)?.type as 'hnsw' | 'diskann' | undefined
|
||||
|
||||
const diskannFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('diskann')
|
||||
|
||||
if (indexType === 'diskann') {
|
||||
if (!diskannFactory) {
|
||||
throw new Error(
|
||||
"config.index.type='diskann' was requested but the 'diskann' provider " +
|
||||
'is not registered. Load cortex as a plugin (it ships the DiskANN engine) ' +
|
||||
'or remove the explicit type to fall back to HNSW.'
|
||||
)
|
||||
}
|
||||
return this.instantiateDiskAnn(diskannFactory, persistMode)
|
||||
}
|
||||
|
||||
if (
|
||||
indexType !== 'hnsw' &&
|
||||
diskannFactory &&
|
||||
this.diskAnnAutoEngageConditionsMet()
|
||||
) {
|
||||
return this.instantiateDiskAnn(diskannFactory, persistMode)
|
||||
}
|
||||
|
||||
const hnswFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('hnsw')
|
||||
if (hnswFactory) {
|
||||
return hnswFactory(
|
||||
const vectorFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('vector')
|
||||
if (vectorFactory) {
|
||||
return vectorFactory(
|
||||
{ ...this.config.index, distanceFunction: this.distance },
|
||||
this.distance,
|
||||
{ storage: this.storage, persistMode }
|
||||
|
|
@ -9393,58 +9200,6 @@ export class Brainy<T = any> implements BrainyInterface<T> {
|
|||
return this.setupIndex()
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-engagement guard for DiskANN (see {@link createIndex}).
|
||||
* Conservative on purpose — we only engage when the deployment shape
|
||||
* is the one DiskANN was designed for (local SSD + stable idMapper).
|
||||
*/
|
||||
private diskAnnAutoEngageConditionsMet(): boolean {
|
||||
const storageWithBlob = this.storage as unknown as {
|
||||
getBinaryBlobPath?: (key: string) => string | null
|
||||
}
|
||||
const indexPath = storageWithBlob.getBinaryBlobPath?.('_diskann/main') ?? null
|
||||
if (!indexPath) return false
|
||||
const idMapper = this.metadataIndex?.getIdMapper?.()
|
||||
if (!idMapper) return false
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Call the DiskANN factory with the right config shape. Pulled out of
|
||||
* {@link createIndex} so the explicit-opt-in and auto-engage paths
|
||||
* share one construction site.
|
||||
*/
|
||||
private instantiateDiskAnn(
|
||||
factory: (config: any, distance: DistanceFunction, options: any) => any,
|
||||
persistMode: 'immediate' | 'deferred'
|
||||
): JsHnswVectorIndex {
|
||||
const storageWithBlob = this.storage as unknown as {
|
||||
getBinaryBlobPath?: (key: string) => string | null
|
||||
}
|
||||
const indexPath =
|
||||
(this.config.index as any)?.diskann?.indexPath ??
|
||||
storageWithBlob.getBinaryBlobPath?.('_diskann/main') ??
|
||||
'_diskann/main.bin'
|
||||
const dim = this.dimensions ?? 384
|
||||
const cfgIn = (this.config.index as any)?.diskann ?? {}
|
||||
const cfg = {
|
||||
dimensions: dim,
|
||||
indexPath,
|
||||
distanceFunction: this.distance,
|
||||
...cfgIn,
|
||||
}
|
||||
if (!this.config.silent) {
|
||||
console.log(
|
||||
`[brainy] DiskANN engaged (path=${indexPath}, dim=${dim}). ` +
|
||||
'Override with config.index.type=\'hnsw\' to disable.'
|
||||
)
|
||||
}
|
||||
return factory(cfg, this.distance, {
|
||||
storage: this.storage,
|
||||
persistMode,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve HNSW persistence mode.
|
||||
* Extracted so both setupIndex() and the HNSW plugin factory path can use it.
|
||||
|
|
|
|||
|
|
@ -462,14 +462,13 @@ export interface GraphVerb {
|
|||
*/
|
||||
export interface HNSWConfig {
|
||||
/**
|
||||
* Index engine selector. Defaults to auto-detect: cortex's DiskANN
|
||||
* engages when registered + storage is local + a stable idMapper is
|
||||
* available (ADR-002); otherwise HNSW. Set explicitly to override:
|
||||
* - `'hnsw'` — force the in-memory HNSW index (the historical default).
|
||||
* - `'diskann'` — require cortex's DiskANN; throw if conditions
|
||||
* aren't met rather than silently falling back.
|
||||
* Index engine selector. `'vector'` is the only admissible value in 8.0
|
||||
* and means the same thing as omitting the field: Brainy resolves the
|
||||
* `'vector'` provider when a plugin registers one and falls back to its
|
||||
* built-in JS HNSW index otherwise. Engine specifics (in-memory vs hybrid
|
||||
* vs on-disk) are the provider's internal decision, not a config choice.
|
||||
*/
|
||||
type?: 'hnsw' | 'diskann'
|
||||
type?: 'vector'
|
||||
M: number // Maximum number of connections per noun
|
||||
efConstruction: number // Size of the dynamic candidate list during construction
|
||||
efSearch: number // Size of the dynamic candidate list during search
|
||||
|
|
@ -484,29 +483,6 @@ export interface HNSWConfig {
|
|||
}
|
||||
// Vector storage mode
|
||||
vectorStorage?: 'memory' | 'lazy' // default: 'memory' — 'lazy' evicts vectors after insert
|
||||
/**
|
||||
* DiskANN-specific tuning. Only consulted when `type === 'diskann'`
|
||||
* or auto-engagement selects DiskANN. Sensible defaults match the
|
||||
* published Vamana paper (R=64, L=100, α=1.2, M=16, ksub=256).
|
||||
*/
|
||||
diskann?: {
|
||||
/** PQ subspaces. dim must be divisible by m. Default 16. */
|
||||
pqM?: number
|
||||
/** Centroids per subspace. Default 256 (8-bit codes). */
|
||||
pqKsub?: number
|
||||
/** Vamana max degree (R). Default 64. */
|
||||
maxDegree?: number
|
||||
/** Build-time candidate list size (L). Default 100. */
|
||||
searchListSize?: number
|
||||
/** α-pruning density factor. Default 1.2. */
|
||||
alpha?: number
|
||||
/** Use file-backed adjacency during build. Required >~100M nodes. */
|
||||
useMmapAdjacency?: boolean
|
||||
/** Scratch file path when useMmapAdjacency is true. */
|
||||
mmapAdjacencyPath?: string
|
||||
/** Index file path. Defaults to `<storageRoot>/_diskann/main.bin`. */
|
||||
indexPath?: string
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ export interface BrainyPluginContext {
|
|||
* - 'graphIndex' — GraphAdjacencyIndex replacement
|
||||
* - 'entityIdMapper' — EntityIdMapper replacement
|
||||
* - 'cache' — UnifiedCache replacement
|
||||
* - 'hnsw' — JsHnswVectorIndex replacement
|
||||
* - 'vector' — JsHnswVectorIndex replacement (vector index engine)
|
||||
* - 'roaring' — RoaringBitmap32 replacement
|
||||
* - 'embeddings' — Embedding engine replacement (single text)
|
||||
* - 'embedBatch' — Batch embedding engine (texts[] → vectors[])
|
||||
|
|
@ -270,8 +270,9 @@ export interface GraphIndexProvider {
|
|||
* so they are optional and not part of the required contract (Brainy's own JS
|
||||
* HNSW index omits `setPersistMode`, for instance).
|
||||
*
|
||||
* **Provider key:** registered under `'vector'`. The legacy `'hnsw'` key is
|
||||
* accepted as a compat shim in 8.0 and retired in a future release.
|
||||
* **Provider key:** registered under `'vector'` — the only key Brainy
|
||||
* consults for the vector index. The pre-8.0 `'hnsw'` and `'diskann'` keys
|
||||
* are retired and never looked up.
|
||||
*/
|
||||
export interface VectorIndexProvider {
|
||||
addItem(item: VectorDocument): Promise<string>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue