diff --git a/src/graph/graphAdjacencyIndex.ts b/src/graph/graphAdjacencyIndex.ts index 59daac7f..a5776990 100644 --- a/src/graph/graphAdjacencyIndex.ts +++ b/src/graph/graphAdjacencyIndex.ts @@ -12,6 +12,7 @@ import { GraphVerb, StorageAdapter } from '../coreTypes.js' import { UnifiedCache, getGlobalCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' import { LSMTree } from './lsm/LSMTree.js' +import type { GraphIndexProvider } from '../plugin.js' export interface GraphIndexConfig { maxIndexSize?: number // Default: 100000 @@ -36,7 +37,7 @@ export interface GraphIndexStats { * Memory efficient: 385x less memory (1.3GB vs 500GB for 1B relationships) * Performance: Sub-5ms neighbor lookups with bloom filter optimization */ -export class GraphAdjacencyIndex { +export class GraphAdjacencyIndex implements GraphIndexProvider { // LSM-tree storage for outgoing and incoming edges private lsmTreeSource: LSMTree // sourceId -> targetIds (outgoing edges) private lsmTreeTarget: LSMTree // targetId -> sourceIds (incoming edges) diff --git a/src/hnsw/hnswIndex.ts b/src/hnsw/hnswIndex.ts index 85f65ce5..f5af9137 100644 --- a/src/hnsw/hnswIndex.ts +++ b/src/hnsw/hnswIndex.ts @@ -17,6 +17,7 @@ import { getGlobalCache, UnifiedCache } from '../utils/unifiedCache.js' import { prodLog } from '../utils/logger.js' import { quantizeSQ8, distanceSQ8 } from '../utils/vectorQuantization.js' import type { SQ8QuantizedVector } from '../utils/vectorQuantization.js' +import type { HnswProvider } from '../plugin.js' // Default HNSW parameters const DEFAULT_CONFIG: HNSWConfig = { @@ -26,7 +27,12 @@ const DEFAULT_CONFIG: HNSWConfig = { ml: 16 // Max level } -export class HNSWIndex { +/** + * Implements {@link HnswProvider}: the vector-index surface Brainy calls on + * whatever the `'hnsw'` factory returns (its own `HNSWIndex`, or Cortex's + * native engine). + */ +export class HNSWIndex implements HnswProvider { private nouns: Map = new Map() private entryPointId: string | null = null private maxLevel = 0 diff --git a/src/plugin.ts b/src/plugin.ts index a75ddd49..18652a65 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -8,7 +8,28 @@ * Plugins are auto-detected by package name or registered manually. */ -import type { StorageAdapter } from './coreTypes.js' +import type { + StorageAdapter, + Vector, + VectorDocument, + GraphVerb, +} from './coreTypes.js' +import type { NounType, VerbType } from './types/graphTypes.js' +import type { MetadataIndexStats } from './utils/metadataIndex.js' +import type { GraphIndexStats } from './graph/graphAdjacencyIndex.js' + +// Re-export the provider contracts that already live closer to their +// implementations so a plugin author (Cortex) can import the *entire* +// provider surface from one stable entrypoint: `@soulcraft/brainy/plugin`. +export type { ColumnStoreProvider } from './indexes/columnStore/types.js' +export type { + AggregationProvider, + AggregateDefinition, + AggregateGroupState, + AggregateResult, + AggregateQueryParams, + GroupByDimension, +} from './types/brainy.types.js' /** * Plugin interface — all brainy plugins must implement this. @@ -58,6 +79,180 @@ export interface BrainyPluginContext { readonly version: string } +// =========================================================================== +// Provider contracts — the EXACT surface Brainy calls on each registered +// provider. +// +// These interfaces are the type-level half of the provider-parity guarantee. +// They capture only what Brainy actually invokes, so a native accelerator +// (Cortex) that declares `implements MetadataIndexProvider` gets a compile +// error the moment a method Brainy depends on is dropped or its signature +// drifts. Brainy's own baseline classes (`MetadataIndexManager`, +// `GraphAdjacencyIndex`, `HNSWIndex`, `EntityIdMapper`, `UnifiedCache`) +// implement them too, so the contract can never silently diverge from the +// thing Brainy ships. +// +// Keep these in lockstep with the call sites in `brainy.ts` and the index +// classes. When Brainy starts calling a new member, add it here — every +// implementation then fails to compile until it provides the member. +// =========================================================================== + +/** + * The `'metadataIndex'` provider — a drop-in for `MetadataIndexManager`. + * Brainy calls this surface via `this.metadataIndex.*` (see `brainy.ts`) and + * the transactional add/remove operations. + */ +export interface MetadataIndexProvider { + init(): Promise + flush(): Promise + rebuild(): Promise + + addToIndex(id: string, entityOrMetadata: any, skipFlush?: boolean, deferWrites?: boolean): Promise + removeFromIndex(id: string, metadata?: any): Promise + + getIds(field: string, value: any): Promise + getIdsForFilter(filter: any): Promise + getIdsForTextQuery(query: string): Promise> + getSortedIdsForFilter(filter: any, orderBy: string, order?: 'asc' | 'desc'): Promise + getFilterValues(field: string): Promise + getFilterFields(): Promise + getFieldValueForEntity(entityId: string, field: string): Promise + getFieldsForType(nounType: NounType): Promise> + getFieldStatistics(): Promise> + getFieldsWithCardinality(): Promise> + getOptimalQueryPlan(filters: Record): Promise + + /** Report which index path a `where` clause on `field` will hit (drives `brain.explain()`). */ + explainField(field: string): Promise<{ path: 'column-store' | 'sparse-chunked' | 'none'; notes?: string }> + + getCountForCriteria(field: string, value: any): Promise + getEntityCountByType(type: string): number + getEntityCountByTypeEnum(type: NounType): number + getTotalEntityCount(): number + getAllEntityCounts(): Map + getTopNounTypes(n: number): NounType[] + getTopVerbTypes(n: number): VerbType[] + getAllNounTypeCounts(): Map + getAllVerbTypeCounts(): Map + getAllVFSEntityCounts(): Promise> + + detectAndRepairCorruption(): Promise + validateConsistency(): Promise<{ + healthy: boolean + avgEntriesPerEntity: number + entityCount: number + indexEntryCount: number + recommendation: string | null + }> + + tokenize(text: string): string[] + extractTextContent(data: any): string + + getStats(): Promise + + /** The shared UUID ↔ int mapper. Brainy reads `.getUuid(intId)` off the result. */ + getIdMapper(): { getUuid(intId: number): string | undefined } + + /** The column store the coordinator delegates `where`/`orderBy` to. */ + readonly columnStore: import('./indexes/columnStore/types.js').ColumnStoreProvider +} + +/** + * The `'graphIndex'` provider — a drop-in for `GraphAdjacencyIndex`. + * Brainy calls this surface via `this.graphIndex.*` (some optional-chained). + */ +export interface GraphIndexProvider { + /** `false` until the index has loaded; Brainy probes this before fast paths. */ + readonly isInitialized: boolean + + getNeighbors( + id: string, + optionsOrDirection?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number } | 'in' | 'out' | 'both' + ): Promise + getVerbIdsBySource(sourceId: string, options?: { limit?: number; offset?: number }): Promise + getVerbIdsByTarget(targetId: string, options?: { limit?: number; offset?: number }): Promise + getVerbsBatchCached(verbIds: string[]): Promise> + + rebuild(): Promise + flush(): Promise + close(): Promise + size(): number + + getStats(): GraphIndexStats + getRelationshipStats(): { + totalRelationships: number + relationshipsByType: Record + uniqueSourceNodes: number + uniqueTargetNodes: number + totalNodes: number + } + getRelationshipCountByType(type: string): number + getTotalRelationshipCount(): number + getAllRelationshipCounts(): Map +} + +/** + * The object returned by the `'hnsw'` provider factory — a drop-in for + * `HNSWIndex`. Brainy calls this surface via `this.index.*` plus the + * transactional add/remove operations. `enableCOW`, `getItem`, and + * `setPersistMode` are intentionally absent: Brainy guards each with + * feature-detection (`typeof x === 'function'`), so they are optional and not + * part of the required contract (Brainy's own `HNSWIndex` omits + * `setPersistMode`, for instance). + */ +export interface HnswProvider { + addItem(item: VectorDocument): Promise + removeItem(id: string): Promise + search( + queryVector: Vector, + k?: number, + filter?: (id: string) => Promise, + options?: { rerank?: { multiplier: number }; candidateIds?: string[] } + ): Promise> + size(): number + clear(): void + rebuild(options?: any): Promise + flush(): Promise + getPersistMode(): 'immediate' | 'deferred' +} + +/** + * The `'entityIdMapper'` provider — a drop-in for `EntityIdMapper`. Injected + * into the TypeScript `MetadataIndexManager` when a native metadata index is + * not also registered; that coordinator calls this full surface (incl. + * `getAllIntIds`, the all-ids universe for negation / `exists:false` filters). + */ +export interface EntityIdMapperProvider { + init(): Promise + getOrAssign(uuid: string): number + getUuid(intId: number): string | undefined + getInt(uuid: string): number | undefined + remove(uuid: string): boolean + flush(): Promise + clear(): Promise + getAllIntIds(): number[] + intsIterableToUuids(ints: Iterable): string[] + readonly size: number +} + +/** + * The `'cache'` provider — a drop-in for `UnifiedCache`. Brainy installs it as + * the global cache (`setGlobalCache`) and calls this surface via + * `getGlobalCache()`. + */ +export interface CacheProvider { + getSync(key: string): any | undefined + set(key: string, data: any, type: 'hnsw' | 'metadata' | 'embedding' | 'other', size: number, rebuildCost?: number): void + delete(key: string): boolean + deleteByPrefix(prefix: string): number + clear(type?: 'hnsw' | 'metadata' | 'embedding' | 'other'): void +} + +// The `'embeddings'` / `'embedBatch'` providers are function-shaped and already +// typed by the existing `EmbeddingFunction` (see `coreTypes.ts`), which Brainy +// uses at the `getProvider('embeddings')` call site. No separate interface is +// added here to avoid a duplicate, unwired contract. + /** * Storage adapter factory — plugins register these to provide * new storage backends that users reference by name. diff --git a/src/utils/entityIdMapper.ts b/src/utils/entityIdMapper.ts index 78eadea8..ff67a4e9 100644 --- a/src/utils/entityIdMapper.ts +++ b/src/utils/entityIdMapper.ts @@ -14,6 +14,7 @@ */ import type { StorageAdapter } from '../coreTypes.js' +import type { EntityIdMapperProvider } from '../plugin.js' export interface EntityIdMapperOptions { storage: StorageAdapter @@ -27,9 +28,12 @@ export interface EntityIdMapperData { } /** - * Maps entity UUIDs to integer IDs for use with Roaring Bitmaps + * Maps entity UUIDs to integer IDs for use with Roaring Bitmaps. + * + * Implements {@link EntityIdMapperProvider}: the surface a registered + * `'entityIdMapper'` provider (e.g. Cortex's native mapper) must also satisfy. */ -export class EntityIdMapper { +export class EntityIdMapper implements EntityIdMapperProvider { private storage: StorageAdapter private storageKey: string diff --git a/src/utils/metadataIndex.ts b/src/utils/metadataIndex.ts index 7db9e5fa..a83d2e2a 100644 --- a/src/utils/metadataIndex.ts +++ b/src/utils/metadataIndex.ts @@ -6,6 +6,7 @@ import { StorageAdapter, resolveEntityField } from '../coreTypes.js' import { ColumnStore } from '../indexes/columnStore/ColumnStore.js' +import type { MetadataIndexProvider } from '../plugin.js' import { MetadataIndexCache, MetadataIndexCacheConfig } from './metadataIndexCache.js' import { compareCodePoints } from './collation.js' import { prodLog } from './logger.js' @@ -103,7 +104,12 @@ interface FieldStats { normalizationStrategy?: 'none' | 'precision' | 'bucket' } -export class MetadataIndexManager { +/** + * Implements {@link MetadataIndexProvider}: the metadata-index surface Brainy + * calls on whatever the `'metadataIndex'` provider resolves to (its own + * manager, or Cortex's native Rust engine). + */ +export class MetadataIndexManager implements MetadataIndexProvider { private storage: StorageAdapter private config: Required private isRebuilding = false diff --git a/src/utils/unifiedCache.ts b/src/utils/unifiedCache.ts index 1ee7f1e7..e497c421 100644 --- a/src/utils/unifiedCache.ts +++ b/src/utils/unifiedCache.ts @@ -17,6 +17,7 @@ import { type MemoryInfo, type CacheAllocationStrategy } from './memoryDetection.js' +import type { CacheProvider } from '../plugin.js' export interface CacheItem { key: string @@ -57,7 +58,14 @@ export interface UnifiedCacheConfig { memoryCheckInterval?: number } -export class UnifiedCache { +/** + * Single cost-aware cache for HNSW and MetadataIndex. + * + * Implements {@link CacheProvider}: the surface Brainy calls on whatever cache + * is installed as the global cache (its own instance, or a registered + * `'cache'` provider such as Cortex's native eviction engine). + */ +export class UnifiedCache implements CacheProvider { private cache = new Map() private access = new Map() // Access counts private loadingPromises = new Map>()