/** * @module brainy * @description The `Brainy` class — the Triple-Intelligence database that unifies * vector similarity, graph traversal, and metadata filtering behind a single API * (`add`/`find`/`relate`/`related`/`get`/`similar`/`graph.*`/`now`/`asOf`/`transact`/ * `export`/`import`). Native acceleration is feature-detected through the provider * boundary; when no provider is registered, the built-in JS engines serve everything. */ import { v4 as uuidv4, v7 as uuidv7 } from './universal/uuid.js' import { coerceNewEntityId, resolveEntityId, ORIGINAL_ID_KEY } from './utils/idNormalization.js' import { JsHnswVectorIndex } from './hnsw/hnswIndex.js' // TypeAwareHNSWIndex removed from default path — single unified JsHnswVectorIndex is faster // for the 99% of queries that don't filter by type (avoids searching 42 separate graphs) import { createStorage, resolveFilesystemRoot } from './storage/storageFactory.js' import type { StorageOptions } from './storage/storageFactory.js' import { rebuildCounts } from './utils/rebuildCounts.js' import type { MetadataWriteBuffer } from './utils/metadataWriteBuffer.js' import { BaseStorage } from './storage/baseStorage.js' import { CURRENT_DATA_FORMAT, EXPECTED_INDEX_EPOCH, readBrainFormat, writeBrainFormat } from './storage/brainFormat.js' import type { BrainFormat } from './storage/brainFormat.js' import { StorageAdapter, Vector, DistanceFunction, EmbeddingFunction, GraphVerb, STANDARD_ENTITY_FIELDS } from './coreTypes.js' import type { HNSWNounWithMetadata, HNSWVerbWithMetadata, EntityVisibility } from './coreTypes.js' import { defaultEmbeddingFunction, cosineDistance, getBrainyVersion } from './utils/index.js' import { embeddingManager } from './embeddings/EmbeddingManager.js' import { matchesMetadataFilter, validateWhereFilter } from './utils/metadataFilter.js' import { NaturalLanguageProcessor } from './neural/naturalLanguageProcessor.js' import { NeuralEntityExtractor, ExtractedEntity } from './neural/entityExtractor.js' import { TripleIntelligenceSystem } from './triple/TripleIntelligenceSystem.js' import { VirtualFileSystem } from './vfs/VirtualFileSystem.js' import { MetadataIndexManager } from './utils/metadataIndex.js' import { detectContentType, extractForHighlighting } from './utils/contentExtractor.js' import { GraphAdjacencyIndex } from './graph/graphAdjacencyIndex.js' import { connectedComponents, stronglyConnectedComponents, pageRank, MinHeap } from './graph/analyticsFallback.js' import { runGraphAudit, type GraphAuditReport } from './graph/graphAudit.js' import { createPipeline } from './streaming/pipeline.js' import { configureLogger, LogLevel, prodLog } from './utils/logger.js' import { warnOnLowOsLimits } from './utils/osLimits.js' import { setGlobalCache } from './utils/unifiedCache.js' import type { UnifiedCache } from './utils/unifiedCache.js' import { rankIndicesByScore, reorderByIndices } from './utils/resultRanking.js' import { PluginRegistry, isVersionedIndexProvider, isGraphAccelerationProvider } from './plugin.js' import type { GraphAccelerationProvider, TraverseOptions, Subgraph, RankOptions, CommunitiesOptions, PathOptions, MetadataIndexProvider, OpaqueIdSet, AtGenerationVectors } from './plugin.js' import type { BrainyPlugin, BrainyPluginContext, GraphCompressionProvider } from './plugin.js' import { ConnectionsCodec } from './hnsw/connectionsCodec.js' import { TransactionManager } from './transaction/TransactionManager.js' import { transactTimeoutBudget } from './transaction/Transaction.js' import { RevisionConflictError } from './transaction/RevisionConflictError.js' import { EntityNotFoundError, RelationNotFoundError } from './errors/notFound.js' import { ValidationConfig, validateAddParams, validateUpdateParams, validateRelateParams, validateUpdateRelationParams, validateFindParams, recordQueryPerformance } from './utils/paramValidation.js' import { findCallerLocation } from './utils/callerLocation.js' import { SaveNounMetadataOperation, SaveNounOperation, AddToHNSWOperation, AddToMetadataIndexOperation, SaveVerbMetadataOperation, SaveVerbOperation, AddToGraphIndexOperation, RemoveFromHNSWOperation, RemoveFromMetadataIndexOperation, RemoveFromGraphIndexOperation, UpdateNounMetadataOperation, UpdateVerbMetadataOperation, DeleteNounMetadataOperation, DeleteVerbMetadataOperation } from './transaction/operations/index.js' import { BaseOperationalMode, ReaderMode, HybridMode } from './storage/operationalModes.js' import { Entity, Relation, Result, AddParams, UpdateParams, RelateParams, UpdateRelationParams, FindParams, SimilarParams, RelatedParams, GraphApi, GraphView, GraphNode, SubgraphOptions, SubgraphSelector, GraphExportOptions, GraphRankOptions, GraphRankEntry, GraphCommunitiesOptions, GraphCommunitiesResult, GraphPathOptions, GraphPathResult, GetOptions, AddManyParams, RemoveManyParams, UpdateManyParams, RelateManyParams, BatchResult, BrainyConfig, BrainyStats, ScoreExplanation, FillSubtypeRule, FillSubtypeRules, FillSubtypesResult } from './types/brainy.types.js' import { NounType, VerbType, TypeUtils } from './types/graphTypes.js' import { splitNounMetadataRecord, splitVerbMetadataRecord } from './types/reservedFields.js' import { BrainyInterface } from './types/brainyInterface.js' import type { IntegrationHub, IntegrationHubConfig } from './integrations/core/IntegrationHub.js' import { MigrationRunner } from './migration/MigrationRunner.js' import type { MigrationPreview, MigrationResult, MigrateOptions } from './migration/types.js' import { AggregationIndex } from './aggregation/AggregationIndex.js' import { AggregateMaterializer } from './aggregation/materializer.js' import type { AggregateDefinition, AggregateQueryParams, AggregateResult } from './types/brainy.types.js' import type { MigrationProgress } from './types/brainy.types.js' import { resolveJsHnswConfig, DEFAULT_RECALL } from './utils/recallPreset.js' import * as fs from 'node:fs' import * as os from 'node:os' import { Db, type DbHost, type HistoricalQueryHandle } from './db/db.js' import { entityMatchesFind } from './db/whereMatcher.js' import { importGraph, isPortableGraph, type PortableGraph, type ExportSelector, type ExportOptions, type ImportOptions, type ImportResult } from './db/portableGraph.js' import { GenerationStore, type CommitBeforeImages } from './db/generationStore.js' import type { FactScanHandle } from './db/factLog.js' import { ENTITY_TREE_STAMP_PATH, readFamilyStamp, verifyFamilyStamp, writeFamilyStamp, type FamilyStamp } from './db/familyStamp.js' import { ChangeFeed, type BrainyChangeEvent, type ChangeListener, type PendingChangeEvent } from './events/changeFeed.js' import { isDeterministicEmbedMode } from './embeddings/deterministicEmbedMode.js' import { GenerationConflictError, StoreInconsistentError } from './db/errors.js' import { BrainyError, GraphIndexNotReadyError, MetadataIndexNotReadyError, MigrationInProgressError, VectorIndexNotReadyError } from './errors/brainyError.js' import { assessIndexReadiness } from './utils/indexReadiness.js' import { MemoryStorage } from './storage/adapters/memoryStorage.js' import type { CompactHistoryOptions, CompactHistoryResult, TransactOptions, TransactReceipt, TxLogEntry, TxOperation, DiffResult, EntityHistory, HistoryVersion } from './db/types.js' import { stableDeepEqual } from './db/stableEqual.js' import type { VersionedIndexProvider, ProviderInvariantReport } from './plugin.js' import type { Operation, TransactionFunction } from './transaction/types.js' /** * Stopwords for semantic highlighting * These common words are skipped when highlighting individual words * to focus on meaningful content words. */ const STOPWORDS = new Set([ 'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'between', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just', 'and', 'but', 'or', 'if', 'because', 'until', 'while', 'although', 'though', 'this', 'that', 'these', 'those', 'it', 'its', 'i', 'me', 'my', 'you', 'your', 'he', 'him', 'his', 'she', 'her', 'we', 'us', 'our', 'they', 'them', 'their', 'what', 'which', 'who', 'whom', 'whose', 'am' ]) /** * Optional capabilities a plugin-provided `'vector'` engine may expose beyond * the built-in {@link JsHnswVectorIndex} surface ({@link VectorIndexProvider} * declares neither). Every call site feature-detects with * `typeof x === 'function'` before invoking, so an engine without these hooks * is fully supported. */ interface VectorIndexOptionalHooks { /** Switch persistence mode at runtime (bulk-ingest optimization). */ setPersistMode?: (mode: 'immediate' | 'deferred') => void /** Release engine resources (timers, file handles, native memory). */ close?: () => Promise | void } /** * Optional lifecycle hook a plugin-provided `'metadataIndex'` implementation * may expose beyond the built-in {@link MetadataIndexManager} surface. * Feature-detected with `typeof x === 'function'` before invoking. */ interface MetadataIndexOptionalHooks { /** Release index resources (timers, native memory). */ close?: () => Promise | void } /** * Brainy's internal resolved configuration. `normalizeConfig()` defaults every * field at construction except the genuinely-optional ones listed in the * `Pick`, which legitimately stay `undefined` when the consumer omitted them — * every read site handles that explicitly (`?.` chains and * `!== undefined` guards). */ type ResolvedBrainyConfig = Required< Omit< BrainyConfig, | 'maxQueryLimit' | 'reservedQueryMemory' | 'vector' | 'plugins' | 'integrations' | 'retention' | 'eagerEmbeddings' | 'migrationWaitTimeoutMs' > > & Pick< BrainyConfig, | 'maxQueryLimit' | 'reservedQueryMemory' | 'vector' | 'plugins' | 'integrations' | 'retention' | 'eagerEmbeddings' | 'migrationWaitTimeoutMs' > /** * Result type for brain.diagnostics() */ export interface DiagnosticsResult { version: string plugins: { active: string[], count: number } providers: Record indexes: { hnsw: { size: number, type: string } metadata: { type: string, initialized: boolean } graph: { type: string, initialized: boolean, wiredToStorage: boolean } } } /** * In-flight planning state for one `brain.transact()` batch (8.0 MVCC). * * Operations inside a batch may reference ids written by EARLIER operations * of the same batch (`add` an entity, then `relate` to it) — but nothing has * touched storage yet during planning. This state carries the * would-be-written stored objects so later planners read * "current state + batch so far", exactly what the executed batch produces. */ interface TxPlanState { /** Stored metadata + vector of entities written earlier in this batch. */ nouns: Map; vector: Vector }> /** Entity ids removed earlier in this batch. */ removedNouns: Set /** Verbs created earlier in this batch (keyed by relationship id). */ verbs: Map /** Relationship ids removed earlier in this batch. */ removedVerbs: Set } /** * The fully planned form of one `brain.transact()` batch: the ordered * `TransactionManager` operations, the resolved id per input operation (the * receipt), the touched-id sets the generation store stages before-images * for, and the post-commit hooks (aggregation-index maintenance — derived * data, applied outside the atomic batch exactly as the single-op methods * do). */ interface PlannedTransact { /** Ordered operations for one atomic TransactionManager execution. */ operations: Operation[] /** Resolved id per input operation, in input order. */ ids: string[] /** Entity ids the batch writes or deletes. */ touchedNouns: string[] /** Relationship ids the batch writes or deletes. */ touchedVerbs: string[] /** Aggregation-index hooks to run after the commit point. */ postCommit: Array<() => void> /** * One entry per staged `{ op: 'update' }`, re-verified UNDER the commit * mutex (the generation store's `precommit`) so per-op `ifRev` CAS is * atomic with the apply — planning-time checks can interleave with * concurrent writers. `updatedMetadata` is the staged metadata object (held * by reference by the staged operation), re-stamped there with the * authoritative `_rev`. A conflict rejects the WHOLE batch. */ casUpdates: Array<{ id: string ifRev?: number updatedMetadata: { _rev?: number } }> /** * Ids whose revision baseline THIS batch resets via an `add` op (a fresh * create, or add's overwrite semantics which restamp `_rev: 1`). Updates to * these ids sequence against in-batch state no concurrent writer can touch, * so their plan-time CAS checks and rev stamps are already exact — the * commit precondition leaves them untouched. */ createdNouns: Set /** * Change-feed events, one per affected record in op order, populated by the * planners ONLY when a listener is subscribed. Stamped with the batch's * committed generation and emitted after `commitTransaction` returns — a * rejected batch (CAS conflict, failed apply) emits nothing. */ changeEvents: PendingChangeEvent[] } /** * Internal control-flow signal: an insert guarded by a must-be-absent * precondition (`add({ ifAbsent })` / `add({ upsert })`) found the entity * already present in the authoritative before-image under the commit mutex — * a concurrent writer created it after the planning-time absence check. * Never escapes `add()`: the caller converts it into the documented * resolution (ifAbsent → return the existing id without writing; upsert → * merge into the now-existing entity via `update()`). */ class InsertPreconditionExistsSignal extends Error { constructor(readonly id: string) { super(`insert precondition: entity ${id} already exists`) this.name = 'InsertPreconditionExistsSignal' } } /** * @description The derived-index families a read may depend on. A read that * consults none of them (a canonical-storage read: `get`, an entity * enumeration, a VFS content/dir read) is index-independent and must never * block on another family's one-time migration. Used by the family-scoped * migration gate ({@link Brainy.awaitMigrationLock}). */ export type IndexFamily = 'vector' | 'metadata' | 'graph' /** * How long a failed aggregation-backfill walk suppresses fresh walk attempts. * Within the window, queries rethrow the recorded failure instantly (loud, * cheap); after it, one new attempt is allowed. Bounds the damage of a * caller-side tight retry loop against a deterministically-failing store. */ const AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS = 30_000 /** * The main Brainy class - Clean, Beautiful, Powerful * REAL IMPLEMENTATION - No stubs, no mocks * * Implements BrainyInterface to ensure consistency across integrations */ export class Brainy implements BrainyInterface { // Static shutdown hook tracking (global, not per-instance) private static shutdownHooksRegisteredGlobally = false private static instances: Brainy[] = [] /** The globally-registered shutdown listeners, kept so the LAST close() can * deregister them. A process.on('SIGINT'/'SIGTERM') listener holds a ref'd * signal handle that keeps the Node event loop alive — a library that never * removes its listeners makes every bare script hang after close(). * See {@link registerShutdownHooks} / {@link deregisterShutdownHooksIfIdle}. */ private static sigtermListener?: () => void private static sigintListener?: () => void private static beforeExitListener?: () => void /** Poll cadence (ms) for the migration LOCK when a provider exposes no * event-driven `whenMigrationComplete()` signal. See {@link awaitMigrationLock}. */ private static readonly MIGRATION_POLL_INTERVAL_MS = 250 /** First-party accelerator packages probed by guarded auto-detection when * `plugins` is undefined (installing one IS the opt-in). See {@link loadPlugins}. */ private static readonly AUTO_DETECT_PLUGIN_PACKAGES = ['@soulcraft/cor'] // Core components private index!: JsHnswVectorIndex private storage!: BaseStorage private metadataIndex!: MetadataIndexManager private graphIndex!: GraphAdjacencyIndex private transactionManager: TransactionManager /** * 8.0 generational MVCC record layer (created in `init()`, before any index * is built — crash recovery may rewrite canonical entity files). Owns the * generation counter, the transact commit protocol, pin refcounts, and * point-in-time record resolution. Backs `now()`/`transact()`/`asOf()`/ * `compactHistory()` and the `Db` value type. */ private generationStore!: GenerationStore /** * Resolves the in-flight write generation as a BigInt, for stamping graph * edges via the {@link GraphIndexProvider} contract. Passed to every * `AddToGraphIndexOperation`/`RemoveFromGraphIndexOperation` and evaluated * when the operation executes — inside a `transact()` batch the generation * store has assigned the batch generation by then; for single-op writes it * reads the post-write watermark. The arrow body reads `generationStore` * lazily, so it is safe to define before `init()` assigns the store. */ private readonly graphWriteGeneration = (): bigint => BigInt(this.generationStore.generation()) /** Lazily built host surface shared by every `Db` value of this brain. */ private _dbHost?: DbHost /** * GC backstop for leaked `Db` pins: when an unreleased `Db` is collected, * its generation pin (store + versioned providers) is released and an * owned snapshot brain (from `Brainy.load()`) is closed. Explicit * `db.release()` unregisters first, so pins are never double-released. */ private _dbFinalizationRegistry?: FinalizationRegistry<{ generation: number closeOnRelease?: () => Promise }> private embedder: EmbeddingFunction private distance: DistanceFunction private config: ResolvedBrainyConfig /** * Bounded warm cache for verb-int → verb-id resolution (8.0 u64 contract). * Fed by `addVerb` returns and `verbIntsToIds` results; evicted in insertion * order once {@link Brainy.VERB_INT_WARM_CACHE_MAX} is exceeded. Pure * optimization — the graph-index provider owns the durable interning, so a * miss just means one extra `verbIntsToIds` round trip. */ private readonly verbIntWarmCache = new Map() /** Cap for {@link Brainy.verbIntWarmCache} (~100k entries ≈ a few MB). */ private static readonly VERB_INT_WARM_CACHE_MAX = 100_000 /** * Default cap on how many `find()` matches seed a `graph.subgraph(query)` when the * caller pins no explicit `limit` — bounds the materialized seed set on the JS / * non-opaque path (the native opaque path forwards the whole universe, uncapped). */ private static readonly QUERY_SEED_CAP = 10_000 // Silent mode state private originalConsole?: { log: typeof console.log info: typeof console.info warn: typeof console.warn error: typeof console.error } // Plugin system private pluginRegistry = new PluginRegistry() /** * Resolved native graph-acceleration provider, cached on first `brain.graph.*` * access. `undefined` = not yet resolved; `null` = resolved, none registered * (use the TS fallback); otherwise the provider instance. */ private _graphAccelProvider: GraphAccelerationProvider | null | undefined = undefined /** * Set when a non-fatal index rebuild fails at init() (the brain is allowed to * start, but in a degraded state where queries may be incomplete). Surfaced * through {@link checkHealth} so consumers can observe the failure * programmatically rather than only via the console. A storage *read* failure * during rebuild is NOT recorded here — it re-throws and aborts init() loudly. */ private _indexRebuildFailed: Error | null = null /** * Ids of records that committed via an adopt-forward failed-rollback recovery * ({@link GenerationStore.commitSingleOp} `degraded`): the canonical record is * durable but its derived index entry may be incomplete until * {@link repairIndex}. Non-empty = a queryable degraded state, folded into * {@link checkHealth}/{@link validateIndexConsistency} and warned on the read * paths. Cleared by {@link repairIndex}. */ private _indexDegradedIds: Set = new Set() /** One-shot guard so the degraded-reads warning fires once per degraded window * (reset when the degraded state clears). See {@link warnIfReadsDegraded}. */ private _degradedReadWarned = false /** One-shot guard so the metadata cold-open consistency probe runs once per brain. */ private _metadataConsistencyProbed = false /** Graph-adjacency cold-load consistency: verified-live this session (one-shot). */ private _graphAdjacencyVerified = false /** Re-entrancy guard: a verify (rebuild → reads) is in flight. */ private _graphAdjacencyVerifying = false /** Metadata field-index cold-read guard: verified-serving this session (one-shot). */ private _metadataVerified = false /** Re-entrancy guard for {@link verifyMetadataLive}. */ private _metadataVerifying = false /** Vector-index cold-read guard: verified-serving this session (one-shot). */ private _vectorVerified = false /** Re-entrancy guard for {@link verifyVectorLive}. */ private _vectorVerifying = false /** * Coordinated migration LOCK (#18): dedup guards so the "upgrading, blocking" * and "upgrade complete, resumed" lines each log once per migration window, * not once per blocked operation. See {@link awaitMigrationLock}. */ private _migrationBlockLogged = false private _migrationReleaseLogged = false /** Epoch ms when brainy first observed the migration LOCK held, or `null` when * not migrating — the fallback `elapsedMs` a provider that omits `migrationStatus()` * timing still gets in `getIndexStatus().migration`. See {@link migrationSnapshot}. */ private _migrationObservedAt: number | null = null /** Path of the pre-upgrade backup taken this open (default-on, opt-out via * `migrationBackup: false`), or `null` if none. Removed once the 7.x→8.0 * upgrade verifies + stamps; retained on failure. See {@link createMigrationBackupIfNeeded}. */ private _migrationBackupPath: string | null = null /** * The in-process change feed behind {@link onChange}. Emitted from the * commit seam ({@link persistSingleOp} / {@link transact}), so every * canonical mutation — any origin — produces exactly one post-commit event * per affected record. See src/events/changeFeed.ts for the delivery * contract. */ private readonly _changeFeed = new ChangeFeed() /** Set when the on-open VFS-blob adoption left one or more `_cow/` blobs it * could not fully adopt (bytes or metadata missing). While true, the * pre-upgrade backup is NOT auto-removed — the upgrade is not verifiably * complete for the VFS. See {@link autoAdoptLegacyVfsBlobsIfNeeded}. */ private _vfsBlobAdoptionIncomplete = false /** * 8.0 ⇄ native-provider version handshake — the on-disk {@link BrainFormat} * marker (`_system/brain-format.json`) as read during the store-open phase, * or `null` for a pre-handshake / brand-new brain. Populated BEFORE any * derived index or native provider is constructed, so {@link formatInfo} * answers synchronously when the provider reads it at its own init(). * Refreshed to the current marker once it is (re)stamped. */ private _brainFormat: BrainFormat | null = null /** * True when the on-disk {@link _brainFormat} is absent OR carries an * `indexEpoch` different from {@link EXPECTED_INDEX_EPOCH} — i.e. the derived * JS indexes on disk predate this build and must be rebuilt from the * canonical records (the epoch-drift rebuild trigger that closes the gap * where JS indexes rebuilt only on `size()===0`, never on a format change). * Cleared once the rebuild verifies and the marker is re-stamped. */ private _indexEpochStale = false // Sub-APIs (lazy-loaded) private _nlp?: NaturalLanguageProcessor private _extractor?: NeuralEntityExtractor private _tripleIntelligence?: TripleIntelligenceSystem private _vfs?: VirtualFileSystem private _vfsInitialized = false // Track VFS init completion separately /** * 8.0 MVCC: single-op writes create generations (Model-B) only AFTER init * completes. Init-time infrastructure writes (the VFS root directory, etc.) * form the generation-0 baseline of a freshly-materialized brain and are NOT * versioned — so a brand-new brain reports `generation() === 0` / empty * `transactionLog()`, and the first USER write is generation 1. Enabled at * the tail of `init()`; see {@link persistSingleOp}. */ private _generationStampingActive = false /** * 8.0 MVCC: the coordinator-driven adaptive history byte budget for this * brain (set via {@link setRetentionBudget}; e.g. cor's `ResourceManager` * fair-shares one box-wide budget across co-located instances). Overrides the * local `os.freemem` probe while `retention` is adaptive. `undefined` = use * the probe. See {@link adaptiveHistoryBudgetBytes}. */ private _retentionBudgetBytes?: number private _hub?: IntegrationHub // Integration Hub for external tools private _pendingMigrationRunner?: MigrationRunner // Deferred migration runner for large datasets private _aggregationIndex?: AggregationIndex // Incremental aggregation engine private _aggregationBackfillFlight: Promise | null = null // Single-flight backfill walk // A failed walk latches its error: retries within the cooldown rethrow it // instantly instead of re-walking, so a tight caller-side retry loop costs // one loud error per query, never a full store walk per query. private _aggregationBackfillFailure: { at: number; error: Error } | null = null private _materializer?: AggregateMaterializer // Debounced materialization of aggregate results /** * Fields registered via `brain.trackField()` — drives optional value validation on * `add()`/`update()` and powers `brain.counts.byField()`. Outer key is the field * name (`'status'`, `'paradigm'`, `'role'`, ...); inner record carries the per-NounType * flag and an optional value whitelist (when provided, writes with off-vocabulary * values are rejected). */ private _trackedFields: Map }> = new Map() /** * Per-NounType / per-VerbType subtype enforcement rules registered via * `brain.requireSubtype(type, options)`. When `required` is true the matching * write path throws if subtype is missing; when `values` is set the matching * write path throws if the value is off-vocabulary. Composes with the * brain-wide `requireSubtype` constructor flag. */ private _requiredSubtypes: Map }> = new Map() // State private initialized = false private dimensions?: number // Multi-process mode (writer is default; reader is set via mode: 'reader' or // Brainy.openReadOnly()). `operationalMode.validateOperation('write')` is // called at the top of every mutation method. Snapshots from `asOf()` also // set this to ReaderMode so historical instances are protected the same way. private operationalMode: BaseOperationalMode // Write-quarantine after a failed transaction rollback left the store // inconsistent (see StoreInconsistentError). Set at the moment the failure is // surfaced; every subsequent mutation is refused via assertWritable until // repairIndex() reconciles canonical vs derived state and clears it. Reads are // never affected. null = healthy. private storeInconsistency: StoreInconsistentError | null = null // Ready Promise state (Unified readiness API) // Allows consumers to await brain.ready for initialization completion private _readyPromise: Promise | null = null private _readyResolve: (() => void) | null = null private _readyReject: ((error: Error) => void) | null = null // In-flight init() promise. Guards against concurrent initialization when // multiple operations race to lazily initialize the same instance — all // racers await the single in-flight run. Reset on failure so init() can // be retried after the underlying cause (e.g. a held writer lock) clears. private initPromise: Promise | null = null // Set once close() has torn the instance down. close() is terminal: a closed // instance must NOT silently lazy-re-initialize on the next operation (that // would resurrect released indexes and timers). The lazy-init convenience // applies only to instances that were never closed. private closed = false // Lazy rebuild state (Production-scale lazy loading) // Prevents race conditions when multiple queries trigger rebuild simultaneously private lazyRebuildInProgress = false private lazyRebuildCompleted = false private lazyRebuildPromise: Promise | null = null constructor(config?: BrainyConfig) { // Normalize configuration with defaults this.config = this.normalizeConfig(config) // Multi-process mode — default 'writer' uses HybridMode (read + write), // 'reader' uses ReaderMode (read-only, all mutations throw). A writer needs // to read its own data, so the writer role is read-write rather than // write-only. this.operationalMode = this.config.mode === 'reader' ? new ReaderMode() : new HybridMode() // Configure memory limits // This must happen early, before any validation occurs if (this.config.maxQueryLimit !== undefined || this.config.reservedQueryMemory !== undefined) { ValidationConfig.reconfigure({ maxQueryLimit: this.config.maxQueryLimit, reservedQueryMemory: this.config.reservedQueryMemory }) } // Setup core components this.distance = cosineDistance this.embedder = this.setupEmbedder() this.transactionManager = new TransactionManager() // Initialize ready Promise // This allows consumers to await brain.ready before using the database this._readyPromise = new Promise((resolve, reject) => { this._readyResolve = resolve this._readyReject = reject }) // Attach a default no-op rejection handler so that init-failure cases // (e.g. another writer holds the lock) do not surface as Node // "unhandled promise rejection" warnings when callers don't `await brain.ready`. // Callers who DO await it still see the original rejection. this._readyPromise.catch(() => { /* observed by ready() consumers, if any */ }) // Track this instance for shutdown hooks Brainy.instances.push(this) // Index and storage are initialized in init() because they may need each other } /** * Open a Brainy store in read-only mode and initialize it. * * Convenience factory equivalent to * `new Brainy({ ...config, mode: 'reader' })` followed by `init()`. The * resulting instance: * * - Does NOT acquire the writer lock — coexists with a live writer process * and with any number of other readers on the same data directory. * - Throws `Cannot mutate a read-only Brainy instance` from every mutation * method (`add`, `addMany`, `update`, `remove`, `removeMany`, `relate`, * `unrelate`, `transact`, `restore`). * - Reflects the state of the writer's last successful `flush()`. To force * the writer to flush before opening, call `requestFlush()` on a separate * handle first, or pass `--fresh` to the `brainy inspect` CLI. * * Typical use cases: * - Operator diagnostics during incidents (`brainy inspect` uses this). * - Read-replica processes on the same machine. * - Long-running analytics scripts that should not contend with the writer. * * @param config Same options as `new Brainy(config)`. The `mode` field is * ignored and forced to `'reader'`. * @returns An initialized, read-only Brainy instance. * * @example * ```typescript * const reader = await Brainy.openReadOnly({ * storage: { type: 'filesystem', path: '/data/brainy-data/tenant' } * }) * const bookings = await reader.find({ where: { entityType: 'booking' } }) * await reader.close() * ``` */ static async openReadOnly(config: BrainyConfig): Promise> { const brain = new Brainy({ ...config, mode: 'reader' }) await brain.init() return brain } /** * Whether this instance is read-only (set via `mode: 'reader'`, * `Brainy.openReadOnly()`, or `asOf()`). When true, all mutation methods * throw. */ get isReadOnly(): boolean { return !this.operationalMode.canWrite } /** * Whether the active storage adapter (anywhere in its prototype chain) * implements a given optional method. * * Storage-adapter plugins (e.g. `@soulcraft/cor`'s `MmapFileSystemStorage * extends FileSystemStorage`) inherit new methods Brainy adds to * `FileSystemStorage` / `BaseStorage` automatically — `typeof` walks the * prototype chain, so there's no in-package version skew to worry about as * long as the plugin's own dist resolves `@soulcraft/brainy` dynamically * (which Cortex 2.2.x onward does — see * `node_modules/@soulcraft/cor/dist/storage/mmapFileSystemStorage.js`). * * This helper exists for the **build/install** failure modes the import * resolution can't catch: * - Stale `node_modules` left over from a prior `bun install` against * `@soulcraft/brainy ≤7.20.x`. * - Lockfile drift pinning brainy below the version that introduced the * method. * - Docker layer caches that reuse a `node_modules` from an earlier image. * - Bundlers (esbuild, webpack) that freeze the prototype chain at build * time and lose later prototype mutations. * In any of those, calling the new method unconditionally crashes boot with * `TypeError: storage.X is not a function`. The guard turns that into a * loud warning + graceful degradation; the operator's clue is the warning * naming the adapter class so they can re-run install / rebuild the image. * * Storage-adapter authors: see `docs/concepts/storage-adapters.md` for the * inheritance contract. Filesystem-backed adapters extending * `FileSystemStorage` inherit the 6 multi-process helpers for free; just * override `supportsMultiProcessLocking()` → `true` to activate enforcement. */ private hasStorageMethod(name: string): boolean { return !!this.storage && typeof (this.storage as unknown as Record)[name] === 'function' } /** * Throw if this instance cannot perform writes. Called at the top of every * mutation method. The check is cheap (a property lookup + boolean test) and * runs before any work, so callers get a clear, fast failure in read-only mode. */ private assertWritable(method: string): void { if (!this.operationalMode.canWrite) { throw new Error( `Cannot call ${method}() on a read-only Brainy instance. ` + `This instance was opened with mode: 'reader' (or via Brainy.openReadOnly() / asOf()). ` + `Open in writer mode to modify data.` ) } // Write-quarantine: a prior transaction's rollback failed and left the store // inconsistent. Refuse further writes (which would compound the damage) until // repairIndex() reconciles and lifts the quarantine. Reads still work. if (this.storeInconsistency) { throw new Error( `Cannot call ${method}() — the store is WRITE-QUARANTINED after a failed ` + `transaction rollback left it inconsistent. Reads still work; run repairIndex() ` + `to reconcile the derived indexes against canonical storage and lift the ` + `quarantine. Original inconsistency: ${this.storeInconsistency.message}` ) } } /** * Initialize Brainy. * * **Calling this explicitly is optional.** Every public operation lazily * initializes the instance on first use (`new Brainy()` followed directly * by `add()` / `find()` / `transact()` just works). Call `init()` yourself * when you want to control *when* the startup cost is paid (e.g. during * server boot rather than on the first request) or to pass init-time * configuration overrides. * * Idempotent and concurrency-safe: repeat calls after success return * immediately; calls racing an in-flight initialization await that single * run (overrides are only applied by the run that starts initialization). * * @param overrides Optional configuration overrides for init */ async init(overrides?: Partial): Promise { if (this.initialized) { return } if (this.initPromise) { return this.initPromise } const run = this.performInit(overrides) this.initPromise = run try { await run } finally { this.initPromise = null } } /** * The single initialization run guarded by `init()`. Never call directly — * always go through `init()` (or any public method, which lazily inits). */ private async performInit(overrides?: Partial): Promise { // Apply any init-time configuration overrides if (overrides) { const { dimensions, ...configOverrides } = overrides // Storage configs shallow-merge; a pre-constructed adapter instance // (on either side) is taken whole — spreading an instance would strip // its prototype methods. const baseStorage = this.config.storage const overrideStorage = configOverrides.storage const mergedStorage = isStorageAdapterInstance(baseStorage) || isStorageAdapterInstance(overrideStorage) ? (overrideStorage ?? baseStorage) : { ...baseStorage, ...overrideStorage } this.config = { ...this.config, ...configOverrides, storage: mergedStorage, verbose: configOverrides.verbose ?? this.config.verbose, silent: configOverrides.silent ?? this.config.silent } // Set dimensions if provided if (dimensions) { this.dimensions = dimensions } // Re-derive operationalMode if mode override changed it if (configOverrides.mode) { this.operationalMode = configOverrides.mode === 'reader' ? new ReaderMode() : new HybridMode() } } // Configure logging based on config options if (this.config.silent) { // Store original console methods for restoration this.originalConsole = { log: console.log, info: console.info, warn: console.warn, error: console.error } // Override all console methods to completely silence output console.log = () => {} console.info = () => {} console.warn = () => {} console.error = () => {} // Also configure logger for silent mode configureLogger({ level: LogLevel.SILENT }) // Suppress all logs } else if (this.config.verbose) { configureLogger({ level: LogLevel.DEBUG }) // Enable verbose logging } try { // Auto-detect and activate plugins BEFORE storage setup // so plugin-provided storage factories (e.g., filesystem override from cor) are available await this.loadPlugins() // 7.x → 8.0 layout migration, BEFORE storage setup: collapse a legacy // branch layout to the flat 8.0 layout on a built-in FileSystemStorage, so // setupStorage() (and any native plugin factory's own legacy guard) sees a // clean store. No-op for non-filesystem stores and already-flat brains. await this.legacyLayoutMigrationPhase() // Setup and initialize storage (checks plugin storage factories first) this.storage = await this.setupStorage() await this.storage.init() // OS-limit detection (once per process, Linux-only, measurement-only): // warn NOW about RLIMIT_NOFILE / vm.max_map_count values that will bite // at pool scale, instead of letting the operator meet them as EMFILE or // a failed mmap deep inside an index open. Fire-and-forget — the check // never affects open. void warnOnLowOsLimits() // Acquire the writer lock for filesystem (and other locking-capable) backends. // Skipped in reader mode and on backends that don't support multi-process locking. // Throws if another live writer holds the directory (unless force: true). // // Defensive call: older storage adapters (e.g. `@soulcraft/cor@2.2.0` // and earlier) bundle a pre-7.21 `BaseStorage` that doesn't define these // methods. We feature-detect each one rather than fail boot. if (this.config.mode !== 'reader') { const canLock = this.hasStorageMethod('supportsMultiProcessLocking') && this.storage.supportsMultiProcessLocking() if (canLock && this.hasStorageMethod('acquireWriterLock')) { await this.storage.acquireWriterLock({ force: this.config.force }) if (this.hasStorageMethod('startFlushRequestWatcher')) { this.storage.startFlushRequestWatcher(async () => { if (this.initialized) { await this.flush() } }) } } else if (!this.config.silent) { // Older adapter OR a backend that doesn't enforce locking (cloud / memory). // Surface this so operators know the multi-process protections aren't active. const backendName = this.storage.constructor.name || 'storage' if (backendName === 'MemoryStorage') { // Memory is single-process by construction — no warning needed. } else if (!this.hasStorageMethod('supportsMultiProcessLocking')) { // The multi-process methods are inherited from FileSystemStorage // when an adapter extends it. Reaching this branch means the // prototype chain doesn't resolve to a Brainy version that // defines them — almost always a build/install artifact rather // than the plugin lacking the code. Tell the operator what to // try first. console.warn( `[brainy] Storage adapter \`${backendName}\` is missing the ` + `multi-process methods on its prototype chain. Writer locking ` + `and the flush-request RPC are disabled for this directory. ` + `Likely fix: clean install (\`rm -rf node_modules bun.lockb && ` + `bun install\`) or rebuild your container image to refresh ` + `\`@soulcraft/brainy\` to ≥7.21. See docs/concepts/storage-adapters.md.` ) } else { console.warn( `[brainy] Multi-process writer protection is not enforced on ${backendName}. ` + `See docs/concepts/multi-process.md for the model.` ) } } } // 8.0 generational MVCC: open the record layer BEFORE any index is // created or loaded. Crash recovery may rewrite canonical entity files // (restoring before-images of an uncommitted transaction), and every // index below loads from those files — opening the store first // guarantees indexes never observe rolled-back state. Reader-mode // instances skip recovery (readers never write; the next writer // repairs). this.generationStore = new GenerationStore(this.storage) const generationOpenResult = await this.generationStore.open({ readOnly: this.config.mode === 'reader' }) // The generation fact log is CANONICAL state, not a derived index — no // sweeper, GC, or blob-lifecycle path may ever delete under it. Declare // its namespace as a protected family (rebuildable: false — a lost fact // segment is NOT reconstructable) so the storage layer REFUSES such // deletes; refusal beats trust. Feature-detected + idempotent per name. if ( this.config.mode !== 'reader' && this.generationStore.getFactLog() && typeof this.storage.registerDerivedFamily === 'function' ) { await this.storage.registerDerivedFamily({ name: 'generation-facts', members: ['_generations/facts/'], namespace: true, rebuildable: false }) } // Fact-scan capability: wire the storage seam through which index // providers (which hold only `storage`) reach the fact log. A closure // over the LIVE log — restore/reopen swaps the instance transparently — // so a provider's heal can switch from the enumeration walk to one // sequential fact scan whenever the log exists. if (typeof (this.storage as BaseStorage).setFactScanSource === 'function') { ;(this.storage as BaseStorage).setFactScanSource({ factLog: () => this.generationStore?.getFactLog() ?? null, // The committed watermark, exposed as a capability so providers // never parse the store's private manifest format. committedGeneration: () => this.generationStore?.committedGeneration() ?? 0 }) } // Entity-tree stamp coherence: compare the stamped sourceGeneration + // rollup invariants against the log head + live counters. Loud on // genuine incoherence (repairIndex heals), silent on absent/coherent, // benign-behind refreshes at the next flush. Never blocks open. await this.verifyEntityTreeStamp() // 8.0 ⇄ native-provider version handshake: load the on-disk brain-format // marker (`_system/brain-format.json`) into an in-memory field NOW — // after the store-open phase, but BEFORE any derived index or native // provider is constructed below — so `formatInfo()` is populated when the // provider reads it synchronously at its own init(). A drifted or absent // `indexEpoch` means the on-disk derived-index format predates this build // (the derived JS indexes are stale); `rebuildIndexesIfNeeded()` rebuilds // them from the canonical records and then re-stamps the marker AFTER the // rebuild verifies (non-destructive: a crash mid-rebuild leaves the old / // absent marker, so the next open idempotently re-rebuilds). this._brainFormat = await readBrainFormat(this.storage) this._indexEpochStale = this._brainFormat === null || this._brainFormat.indexEpoch !== EXPECTED_INDEX_EPOCH // Pre-upgrade backup (default-on, opt-out via `migrationBackup: false`): the // on-disk format is stale, so a one-time 7.x → 8.0 rebuild will run below. // Snapshot the brain dir NOW — before any provider (native or JS) rebuilds a // derived index — so a migration bug can be rolled back. Removed once the // upgrade verifies + stamps; retained on failure. No-op for a reader, for // non-filesystem storage, or for a brain with no persisted data. if (this._indexEpochStale && this.config.migrationBackup && !this.isReadOnly) { await this.createMigrationBackupIfNeeded() } // Provider: embeddings (reassign embedder if plugin provides one) const embeddingProvider = this.pluginRegistry.getProvider('embeddings') if (embeddingProvider) { this.embedder = embeddingProvider } // Provider: cache (replace global singleton before any consumer uses it) const cacheProvider = this.pluginRegistry.getProvider('cache') if (cacheProvider) { setGlobalCache(cacheProvider) } // Provider: roaring bitmaps (native CRoaring replacement for WASM) const roaringProvider = this.pluginRegistry.getProvider('roaring') if (roaringProvider) { const { setRoaringImplementation } = await import('./utils/roaring/index.js') setRoaringImplementation(roaringProvider) } // Provider: msgpack (native replacement for JS @msgpack/msgpack) const msgpackProvider = this.pluginRegistry.getProvider('msgpack') if (msgpackProvider) { const { setMsgpackImplementation } = await import('./graph/lsm/SSTable.js') setMsgpackImplementation(msgpackProvider) } // Provider: sort:topK (e.g. cor's native partial-sort / heap-select) — swaps the // JS result-ranking used by find() to pick the top `offset + limit` rows. The provider // returns indices into a scores array, ordered descending with stable ties, identical // to the JS sortTopKIndicesJs. rankIndicesByScore validates the provider's output and // falls back to JS on any inconsistency, so ranking is always correct. const sortTopKProvider = this.pluginRegistry.getProvider< (scores: number[], k: number, descending: boolean) => number[] >('sort:topK') if (sortTopKProvider) { const { setSortTopKImplementation } = await import('./utils/resultRanking.js') setSortTopKImplementation(sortTopKProvider) } // Provider: distance function (resolve BEFORE setupIndex — index uses this.distance) const nativeDistance = this.pluginRegistry.getProvider('distance') if (nativeDistance) { this.distance = nativeDistance } // Provider: HNSW index factory (plugin or JS fallback) this.index = this.createIndex() // Provider: metadata index factory const metadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') if (metadataFactory) { this.metadataIndex = metadataFactory(this.storage) } else { // JS fallback — inject native EntityIdMapper if cor provides one const entityIdMapperFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('entityIdMapper') this.metadataIndex = new MetadataIndexManager(this.storage, {}, { entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined, }) } // Provider: graph index factory const graphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') if (graphFactory) { this.graphIndex = graphFactory(this.storage) this.storage.setGraphIndex(this.graphIndex) await this.metadataIndex.init() } else { const [, graphIndex] = await Promise.all([ this.metadataIndex.init(), this.storage.getGraphIndex() ]) this.graphIndex = graphIndex } // Eager cold-load (readiness contract). A provider that persists its // derived state exposes init?(): trigger the load NOW — AFTER // metadataIndex.init() above (the id-mapper is hydrated first, so a // native int-keyed index resolves endpoints/slots through it — the // CTX-BR-RESTORE-REBUILD order) and BEFORE the rebuild gate — so a // durable index reports its real size()/isReady() at the gate instead // of eating a spurious rebuild-from-canonical on every open (§7.1 // rebuild()==0). Vector first, then graph. JS engines: the JS vector // index has no init() (rebuild() IS its load path); the JS graph's // init() already cold-loaded its LSM inside storage.getGraphIndex() // (idempotent here). const vectorWithInit = this.index as { init?: () => Promise } if (typeof vectorWithInit.init === 'function') { await vectorWithInit.init() } const graphWithInit = this.graphIndex as { init?: () => Promise } if (typeof graphWithInit.init === 'function') { await graphWithInit.init() } // 8.0 u64 contract: thread the shared UUID ↔ int resolver into the JS // graph index and the storage layer's verb read paths. this.wireGraphIdResolver() // Wire the connections codec (2.4.0 #3). When the graph:compression // provider is registered AND the metadata index exposes a stable // idMapper, inject a codec that encodes JS HNSW connections as // delta-varint blobs at save time and decodes on load. It rides the // storage adapter's generic binary-blob primitive (both the filesystem // and memory adapters implement it), so it is independent of any native // vector provider's single-file on-disk format. this.wireConnectionsCodec() // 8.0 generational MVCC: if crash recovery rolled back an uncommitted // transaction, every derived index is suspect — persisted index state // (flushed before the crash) may reference the rolled-back writes. // Rebuild all three from the repaired canonical records (the JS-index // equivalent of the locked design's "manifest-vs-index generation // comparison + replay on open"). if (generationOpenResult.rolledBackGenerations > 0) { prodLog.warn( `[Brainy] Rebuilding indexes after crash recovery rolled back ` + `${generationOpenResult.rolledBackGenerations} uncommitted transaction(s)` ) await Promise.all([ this.metadataIndex.rebuild(), this.index.rebuild(), this.graphIndex.rebuild() ]) } // 8.0 versioned-provider replay-gap check: a provider whose persisted // index generation is behind the storage layer's committed generation // replays the gap itself (post-commit applier contract) — surface the // gap for observability. for (const provider of this.versionedIndexProviders()) { const providerGen = provider.generation() const committed = BigInt(this.generationStore.committedGeneration()) if (providerGen < committed) { prodLog.info( `[Brainy] Versioned index provider is at generation ${providerGen} ` + `(storage committed: ${committed}) — provider replays the gap per ` + `the post-commit applier contract` ) } else if (providerGen > committed) { // The AHEAD direction is incoherence, not a replay gap: the provider's // persisted index claims writes the store no longer has — the signature // of a torn copy or a log truncation that pulled the committed // watermark back (crash recovery, byte-copy of a live store). A replay // can never converge on it and index answers may reference vanished // writes. Name it loudly at open so it is never diagnosed from a // silent journal; the provider's own coherence check / heal walk (or // brain.repairIndex()) is the cure. prodLog.warn( `[Brainy] Versioned index provider is AHEAD of the store: provider ` + `generation ${providerGen} vs committed ${committed}. This store was ` + `likely copied from a live service or truncated during crash recovery. ` + `Derived-index answers may reference rolled-back writes until the ` + `provider heals from canonical (brain.repairIndex() forces it).` ) } } // Recover VFS content blobs a 7→8 upgrade left stranded in the removed // branch system's copy-on-write area (`_cow/`). Runs BEFORE the rebuild // (whose success stamp removes the pre-upgrade backup) so the backup still // covers this step, and works on the raw object primitives (no blob store // needed yet). A cheap no-op on native-8.0 / fresh brains and on a brain // already healed. See adoptLegacyCowBlobs. await this.autoAdoptLegacyVfsBlobsIfNeeded() // Temporal-blob contract: one-time (marker-gated) backfill of blob // history reference counts for stores whose generation history predates // the contract — after it, compaction reclaims blob bytes exactly (zero // live AND zero history references). On a scrub failure the store runs // leak-safe (no blob reclamation) rather than risk a premature delete. if (typeof (this.storage as unknown as { backfillBlobHistoryRefCountsIfNeeded?: () => Promise }).backfillBlobHistoryRefCountsIfNeeded === 'function') { await (this.storage as unknown as { backfillBlobHistoryRefCountsIfNeeded: () => Promise }).backfillBlobHistoryRefCountsIfNeeded() } // Rebuild indexes if needed for existing data await this.rebuildIndexesIfNeeded() // Check for pending data migrations await this.checkMigrations() // Register shutdown hooks for graceful count flushing (once globally) if (!Brainy.shutdownHooksRegisteredGlobally) { this.registerShutdownHooks() Brainy.shutdownHooksRegisteredGlobally = true } // Initialize the content-addressed blob store before VFS — the VFS // stores all file content through it. if (typeof this.storage.initializeBlobStorage === 'function') { await this.storage.initializeBlobStorage() } // Log provider summary after all wiring is complete // Shows developers exactly what's native vs falling back to JS if (this.pluginRegistry.hasActivePlugins() && !this.config.silent) { const wellKnownKeys = [ 'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache', 'vector', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack' ] const native = wellKnownKeys.filter(k => this.pluginRegistry.hasProvider(k)) const fallback = wellKnownKeys.filter(k => !this.pluginRegistry.hasProvider(k)) const plugins = this.pluginRegistry.getActivePlugins().join(', ') if (fallback.length === 0) { console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins})`) } else { console.log(`[brainy] Providers: ${native.length}/${wellKnownKeys.length} native (${plugins}) | default: ${fallback.join(', ')}`) } // An activated accelerator that registered NOTHING is running as pure // decoration — every query silently serves from the JS engines. Most // often a licensing gate declining to engage. Say so, loudly, so an // evaluator never concludes the accelerator "does nothing". if (native.length === 0) { console.warn( `[brainy] ⚠ ${plugins} activated but registered 0 native providers — all queries are ` + `running on the default JS engines. Check the plugin's requirements (e.g. a license ` + `key) and its logs; see docs/PLUGINS.md.` ) } } // Mark as initialized BEFORE VFS init // VFS.init() needs brain to be marked initialized to call brain methods this.initialized = true // Migration LOCK (#18): if a native provider is running the one-time // 7.x → 8.0 rebuild, WAIT for it to finish HERE — before VFS bootstrap and // before the brain serves — so init-internal reads/writes (the VFS root // get/add/find below) run against the rebuilt indexes, never a half-built // one. This is the "not ready until upgraded" contract: a blocking upgrade // to a known-good state. Bounded by `migrationWaitTimeoutMs`: a brain whose // upgrade outlives the budget surfaces MigrationInProgressError here (raise // the budget, or run the offline migrator) instead of bootstrapping the VFS // against a partial index. A non-migrating brain returns instantly. await this.awaitMigrationLock() // Initialize VFS: Ensure VFS is ready when accessed as property // This eliminates need for separate vfs.init() calls - zero additional complexity this._vfs = new VirtualFileSystem(this) await this._vfs.init() this._vfsInitialized = true // Mark VFS as fully initialized // 8.0 MVCC: infrastructure bootstrap (VFS root, etc.) is now the // generation-0 baseline. From here, every single-op write is a Model-B // generation. (Writers only — readers never stamp; a no-op for them.) if (!this.isReadOnly) { this._generationStampingActive = true } // Eager embedding initialization. // // Adaptive default (8.0): the WASM embedding engine eagerly initializes // during init() WHENEVER it is the active embedder — i.e. no native // 'embeddings' provider has taken over — and the instance is a writer // (not reader-mode) outside of unit tests. The WASM module (≈93MB with // the embedded model) takes 90-140s to compile on throttled CPUs; paying // that during boot rather than on the first embed()-driven call is the // right default for the overwhelmingly common single-process server. // // Skipped automatically when: // - a native 'embeddings' provider is registered (it owns embeddings; // the WASM engine is dead weight), // - reader-mode (readers don't embed — they query existing vectors), // - unit-test mode (tests must stay fast and use the mock embedder). // // `eagerEmbeddings: false` is the explicit override to force lazy init // (first-embed) even when this instance is the active embedder. const isUnitTestMode = isDeterministicEmbedMode() const eager = this.config.eagerEmbeddings ?? true if ( eager && !this.pluginRegistry.hasProvider('embeddings') && this.config.mode !== 'reader' && !isUnitTestMode ) { console.log('Eager embedding initialization enabled...') await embeddingManager.init() console.log('Embedding engine ready') } // Integration Hub initialization // Creates the hub when integrations are enabled in config // Uses dynamic import for tree-shaking when integrations are disabled if (this.config.integrations) { const hubConfig = this.config.integrations === true ? { enable: 'all' as const } : this.config.integrations const { IntegrationHub } = await import('./integrations/core/IntegrationHub.js') this._hub = await IntegrationHub.create(this, { basePath: hubConfig.basePath, enable: hubConfig.enable, // Boundary: the public `IntegrationsConfig['config']` (docs-facing, // src/types/brainy.types.ts) and the hub's internal // `IntegrationHubConfig['config']` evolved as separate declarations. // The hub passes per-integration overrides through verbatim, so the // shapes are runtime-compatible, but TypeScript's weak-type check // rejects the bridge (e.g. `webhooks.maxRetries` exists only on the // public side). config: hubConfig.config as unknown as IntegrationHubConfig['config'] }) } // Resolve ready Promise - consumers awaiting brain.ready will now proceed if (this._readyResolve) { this._readyResolve() } } catch (error) { // Reject ready Promise - consumers awaiting brain.ready will receive error if (this._readyReject) { this._readyReject(error instanceof Error ? error : new Error(String(error))) } // Machine-readable init failures pass through UNWRAPPED — the writer-lock // conflict documents an err.code/err.lockInfo contract ("callers detect // this case via err.code"), and wrapping in a fresh Error silently // stripped both, leaving consumers only a message to regex against. if (error instanceof Error && (error as Error & { code?: string }).code === 'BRAINY_WRITER_LOCKED') { throw error } throw new Error(`Failed to initialize Brainy: ${error}`) } } /** * Register shutdown hooks for graceful count flushing * * Ensures pending count batches are persisted before container shutdown. * Critical for Cloud Run, Fargate, Lambda, and other containerized deployments. * * Handles: * - SIGTERM: Graceful termination (Cloud Run, Fargate, Lambda) * - SIGINT: Ctrl+C (development/local testing) * - beforeExit: Node.js cleanup hook (fallback) * * NOTE: Registers globally (once for all instances) to avoid MaxListenersExceededWarning */ private registerShutdownHooks(): void { const flushOnShutdown = async () => { console.log('Shutdown signal received - flushing pending data...') try { let flushedCount = 0 for (const instance of Brainy.instances) { if (instance.initialized) { // Flush all buffered data, then close to release resources (timers, handles) await Promise.all([ (async () => { if (instance.storage && typeof instance.storage.flushCounts === 'function') { await instance.storage.flushCounts() } })(), (async () => { if (instance.metadataIndex && typeof instance.metadataIndex.flush === 'function') { await instance.metadataIndex.flush() } })(), (async () => { if (instance.graphIndex && typeof instance.graphIndex.flush === 'function') { await instance.graphIndex.flush() } })(), (async () => { if (instance.index && typeof instance.index.flush === 'function') { await instance.index.flush() } })() ]) // Close components to stop timers that would prevent clean process exit await Promise.all([ (async () => { if (instance.graphIndex && typeof instance.graphIndex.close === 'function') { await instance.graphIndex.close() } })(), (async () => { const index = instance.index as JsHnswVectorIndex & VectorIndexOptionalHooks if (index && typeof index.close === 'function') { await index.close() } })(), (async () => { const metadataIndex = instance.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks if (metadataIndex && typeof metadataIndex.close === 'function') { await metadataIndex.close() } })(), // Release the writer lock so a successor process can take over. // No-op for readers and for backends without locking. (async () => { if (instance.storage && typeof instance.storage.releaseWriterLock === 'function') { await instance.storage.releaseWriterLock() } })(), // Stop the flush-request watcher to release its interval timer. (async () => { if (instance.storage && typeof instance.storage.stopFlushRequestWatcher === 'function') { instance.storage.stopFlushRequestWatcher() } })(), ]) flushedCount++ } } if (flushedCount > 0) { console.log(`Flushed successfully (${flushedCount} instance${flushedCount > 1 ? 's' : ''})`) } } catch (error) { console.error('Failed to flush on shutdown:', error) } } // Graceful shutdown signals (registered once globally). The listeners are // kept as statics so the last live instance's close() can deregister them // — the signal handles they hold are ref'd and would otherwise keep the // process alive forever after every brain is closed. Brainy.sigtermListener = async () => { await flushOnShutdown() process.exit(0) } Brainy.sigintListener = async () => { await flushOnShutdown() process.exit(0) } Brainy.beforeExitListener = async () => { // Self-deregister FIRST: Node re-emits 'beforeExit' after every event- // loop drain, and this flush schedules new async work — with the // listener still attached, a script that never calls close() would spin // flush → drain → flush forever and never exit. One flush, then the // next drain finds no listener and the process exits. if (Brainy.beforeExitListener) { process.off('beforeExit', Brainy.beforeExitListener) Brainy.beforeExitListener = undefined } await flushOnShutdown() } process.on('SIGTERM', Brainy.sigtermListener) process.on('SIGINT', Brainy.sigintListener) process.on('beforeExit', Brainy.beforeExitListener) } /** * Deregister the global shutdown hooks once NO live instance remains, so a * script that closed every brain exits on its own — a library must never * keep its host process alive. Re-initializing later re-registers them * (the `shutdownHooksRegisteredGlobally` flag resets here). */ private static deregisterShutdownHooksIfIdle(): void { if (Brainy.instances.length > 0 || !Brainy.shutdownHooksRegisteredGlobally) { return } if (Brainy.sigtermListener) process.off('SIGTERM', Brainy.sigtermListener) if (Brainy.sigintListener) process.off('SIGINT', Brainy.sigintListener) if (Brainy.beforeExitListener) process.off('beforeExit', Brainy.beforeExitListener) Brainy.sigtermListener = undefined Brainy.sigintListener = undefined Brainy.beforeExitListener = undefined Brainy.shutdownHooksRegisteredGlobally = false } /** * Ensure Brainy is initialized, lazily initializing on first use. * * Called at the top of every public operation, so `new Brainy()` followed * directly by `add()` / `find()` / any other call just works — forgetting * `init()` is not an error. Concurrent first operations share a single * initialization run (see `init()`); if initialization fails, the error * propagates to the operation that triggered it. * * `close()` is terminal: a closed instance throws here instead of silently * re-initializing — using a closed Brainy is a consumer bug, not a lazy-init * opportunity. */ private async ensureInitialized(opts?: { bypassMigrationLock?: boolean needs?: IndexFamily[] }): Promise { if (this.closed) { throw new Error('Brainy instance is not initialized: it was closed via close(). Create a new instance.') } if (!this.initialized) { await this.init() } // Coordinated migration LOCK (#18): every data-plane read and write funnels // through here, so this is the single choke point that holds operations while // a native provider runs its one-time 7.x → 8.0 rebuild-from-canonical — no // op touches a half-built index. The gate is FAMILY-SCOPED: `needs` names the // derived-index families this operation actually consults, so a read served // entirely from canonical storage (`needs: []`) or from a healthy family // never blocks on an UNRELATED family's migration. `needs` omitted = the // conservative whole-brain wait (writes, and any read not yet classified). // Observability (`health`/`checkHealth`) and the lock-clearing path // (`stampBrainFormat`, which does not route through here) opt out entirely so // an operator can always watch progress and a native provider can stamp. // A brain that never migrates pays one boolean check (see awaitMigrationLock). if (!opts?.bypassMigrationLock) { await this.awaitMigrationLock(opts?.needs) } } /** * Check if Brainy is initialized */ get isInitialized(): boolean { return this.initialized } /** * Promise that resolves when Brainy is fully initialized and ready to use * * This Promise is created in the constructor and resolves when init() completes. * It can be awaited multiple times safely - the result is cached. * * This enables reliable readiness detection for consumers — await it * before issuing queries when init() was fired without awaiting. * * @example Waiting for readiness before API calls * ```typescript * const brain = new Brainy() * brain.init() // Fire and forget * * // Elsewhere in your code (e.g., API handler) * await brain.ready * const results = await brain.find({ query: 'test' }) * ``` * * @example Server startup pattern * ```typescript * const brain = new Brainy() * await brain.init() * * // For health check endpoint * app.get('/health', async (req, res) => { * try { * await brain.ready * res.json({ status: 'ready' }) * } catch (error) { * res.status(503).json({ status: 'initializing', error: error.message }) * } * }) * ``` * * @returns Promise that resolves when init() completes, or rejects if init fails */ get ready(): Promise { if (!this._readyPromise) { // This should never happen if constructor ran, but handle gracefully return Promise.reject(new Error('Brainy not constructed properly')) } return this._readyPromise } // ============= CORE CRUD OPERATIONS ============= /** * Add an entity to the database * * **Data vs Metadata:** * - `data`: Content used for vector embeddings. Searchable via **semantic similarity** * (HNSW vector index). NOT queryable via `where` filters. Pass a string for text * embedding, or any value for opaque storage. * - `metadata`: Structured fields indexed by MetadataIndex. Queryable via `where` * filters in `find()`. Put anything you want to filter/query on here (tags, * categories, dates, flags, etc.). * * @param params - Parameters for adding the entity * @param params.data - Content to embed and store (required). Strings are auto-embedded. * @param params.type - NounType classification (required) * @param params.metadata - Custom queryable metadata (indexed, used in where filters) * @param params.id - Custom ID (auto-generated UUID if not provided) * @param params.vector - Pre-computed embedding vector (skips auto-embedding) * @param params.service - Service name for multi-tenancy * @param params.confidence - Type classification confidence (0-1) * @param params.weight - Entity importance/salience (0-1) * @returns Promise that resolves to the entity ID * * @example Basic entity creation * ```typescript * const id = await brain.add({ * data: "John Smith is a software engineer", * type: NounType.Person, * metadata: { role: "engineer", team: "backend" } * }) * console.log(`Created entity: ${id}`) * ``` * * @example Adding with confidence and weight * ```typescript * const id = await brain.add({ * data: "Machine learning model for sentiment analysis", * type: NounType.Concept, * metadata: { accuracy: 0.95, version: "2.1" }, * confidence: 0.92, // High confidence in Concept classification * weight: 0.85 // High importance entity * }) * ``` * * @example Adding with custom ID * ```typescript * const customId = await brain.add({ * id: "user-12345", * data: "Important document content", * type: NounType.Document, * metadata: { priority: "high", department: "legal" } * }) * ``` * * @example Using pre-computed vector (optimization) * ```typescript * const vector = await brain.embed("Optimized content") * const id = await brain.add({ * data: "Optimized content", * type: NounType.Document, * vector: vector, // Skip re-embedding * metadata: { optimized: true } * }) * ``` * * @example Multi-tenant usage * ```typescript * const id = await brain.add({ * data: "Customer feedback", * type: NounType.Message, * service: "customer-portal", // Multi-tenancy * metadata: { rating: 5, verified: true } * }) * ``` */ /** * @description Mint a fresh, time-ordered entity id (UUID v7) WITHOUT a write. * Assign ids client-side to reference an entity before it exists — forward * references in `transact()`, deterministic `relate()`, no write→get→relate * round-trip. Time-ordered, so ids sort by creation time. * @returns A new UUID v7 string. * @example * const id = brain.newId() * await brain.add({ id, type: NounType.Document, data: 'draft' }) */ newId(): string { return uuidv7() } /** * @description Persist one single-operation write (`add`/`update`/`remove`/ * `relate`/`updateRelation`/`unrelate` and the per-item calls of `addMany`/ * `updateMany`/`relateMany`) as its OWN immutable generation — the Model-B * "every write versioned" contract. (`removeMany` is the one exception: it * commits each delete *chunk* as a single generation for bulk-delete efficiency, * not one generation per removed id.) Wraps the write's existing `executeTransaction` batch in * {@link GenerationStore.commitSingleOp}, which captures byte-identical * before-images of `touched`, runs the batch with the generation watermark * pinned to the reserved generation (so this write's graph-index ops stamp cor * with that one distinct per-op generation, via the unchanged * {@link graphWriteGeneration}), and buffers the history for async * group-commit. A crash before the flush loses only that buffered history, not * the acknowledged live write (drop-without-restore recovery). * * @param touched - The entity/relationship ids the write creates, updates, or * deletes — the before-image + per-id-chain set. * @param run - The single-op's existing operation batch builder (the * `tx => {…}` body previously passed straight to `executeTransaction`). */ private async persistSingleOp( touched: { nouns?: string[]; verbs?: string[] }, run: TransactionFunction, precommit?: (before: CommitBeforeImages) => void, pendingEvents?: PendingChangeEvent[] ): Promise<{ generation?: number; timestamp: number; degraded?: string[] }> { // Change-feed capture: when this write will emit, hold a reference to the // commit's before-images so `remove` events can carry the record's last // committed state (free — the commit reads them anyway for Model B). let capturedBefore: CommitBeforeImages | undefined const captureAndCheck = pendingEvents && pendingEvents.length > 0 ? (before: CommitBeforeImages): void => { capturedBefore = before precommit?.(before) } : precommit if (!this._generationStampingActive) { // Init-time / infrastructure baseline write (e.g. the VFS root): apply // WITHOUT creating a generation. Generation 0 is the freshly-materialized // brain (bootstrap included); the first USER write is generation 1. // Bootstrap writes are single-threaded, so a supplied precondition is // honored against directly-read before-images (no commit mutex exists // on this path — nothing to race). if (captureAndCheck) { const nouns = new Map() for (const id of touched.nouns ?? []) { const prev = await this.storage.readNounRaw(id) nouns.set(id, { kind: 'noun', metadata: prev.metadata, vector: prev.vector }) } const verbs = new Map() for (const id of touched.verbs ?? []) { const prev = await this.storage.readVerbRaw(id) verbs.set(id, { kind: 'verb', metadata: prev.metadata, vector: prev.vector }) } captureAndCheck({ nouns, verbs } as CommitBeforeImages) } await this.generationStore.runWithoutGeneration(() => this.transactionManager.executeTransaction(run, { timeout: transactTimeoutBudget( (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0) ) }) ) const timestamp = Date.now() // Bootstrap writes are not generation-stamped; emit without one. this.emitCommitted(pendingEvents, capturedBefore, undefined, timestamp) return { timestamp } } let receipt try { receipt = await this.generationStore.commitSingleOp({ touched, precommit: captureAndCheck, execute: () => this.transactionManager.executeTransaction(run, { timeout: transactTimeoutBudget( (touched.nouns?.length ?? 0) + (touched.verbs?.length ?? 0) ) }) }) } catch (err) { // A failed rollback that left the store inconsistent (a remove/update // whose restore-undo failed) quarantines writes until repairIndex(). if (err instanceof StoreInconsistentError) this.storeInconsistency = err throw err } // POST-COMMIT ONLY: an aborted commit (CAS conflict, failed apply) throws // above and never reaches this line — the feed cannot announce a write // that did not become durable. (A degraded adopt-forward write DID commit — // it carries a generation and emits normally.) this.emitCommitted(pendingEvents, capturedBefore, receipt.generation, receipt.timestamp) // An adopt-forward failed-rollback recovery committed the record but may have // left its derived index incomplete (generationStore already warned once). // Record the ids so reads and checkHealth() surface the incompleteness rather // than silently returning partial data; repairIndex() clears them. if (receipt.degraded && receipt.degraded.length > 0) { for (const degradedId of receipt.degraded) this._indexDegradedIds.add(degradedId) this._degradedReadWarned = false // re-arm the read-path warning if (!this.config.silent) { prodLog.warn( `[Brainy] A single-op write committed in a DEGRADED state — the derived ` + `index may be incomplete for id(s) ${receipt.degraded.join(', ')}. ` + `Reads may return partial results until repairIndex() reconciles them.` ) } } return receipt } /** * @description Stamp pending change events with their commit receipt, * enrich entity `remove` events with the record's last committed state * (from the commit's before-images), and hand them to the change feed for * post-mutex dispatch. No-op when the write produced no events. * @param pending - Events the mutation method constructed pre-commit * (only when a listener is subscribed — see {@link ChangeFeed.hasListeners}). * @param before - The commit's before-images (delete-payload source). * @param generation - The committed generation (absent for bootstrap writes). * @param timestamp - The commit timestamp. */ private emitCommitted( pending: PendingChangeEvent[] | undefined, before: CommitBeforeImages | undefined, generation: number | undefined, timestamp: number ): void { if (!pending || pending.length === 0) return const events: BrainyChangeEvent[] = pending.map((p) => { let entity = p.entity if (!entity && p.kind === 'entity' && p.op === 'remove' && p.id && before) { const record = before.nouns.get(p.id)?.metadata as | Record | null | undefined if (record) { entity = this.entityViewFromRawRecord(p.id, record) } } return { ...p, ...(entity && { entity }), ...(generation !== undefined && { generation }), timestamp } }) this._changeFeed.emit(events) } /** * @description Build the change-event entity view from a stored flat * metadata record (the shape before-images and pre-delete reads carry), * using THE canonical reserved/custom split so the view can never drift * from the live read paths. * @param id - The entity's canonical UUID. * @param record - The stored flat metadata record. * @returns The `ChangeEventEntity` view (type, subtype, service, custom metadata). */ private entityViewFromRawRecord( id: string, record: Record ): NonNullable { const { reserved, custom } = splitNounMetadataRecord(record) return { id, type: String(reserved.noun ?? 'unknown'), ...(reserved.subtype !== undefined && { subtype: String(reserved.subtype) }), metadata: custom, ...(reserved.service !== undefined && { service: String(reserved.service) }) } } /** * @description Add an entity (noun) to the brain. Embeds `data` into a vector and * indexes the entity across all three intelligences — vector similarity, graph * adjacency, and metadata — then returns its id. When no `id` is supplied a fresh * time-ordered **UUID v7** is generated; a supplied natural-key string is * normalized to a stable **UUID v5**. Under Model-B every add is its own immutable * generation, so it is time-travelable via {@link asOf}. * @param params - The entity to add. `data` (embedded for similarity) and `type` * (a {@link NounType}) are the essentials; `subtype`, `metadata`, an explicit * `id`, `visibility`, `confidence`, and `weight` are optional. * @returns The entity's id — the supplied/normalized id, or a freshly generated UUID v7. * @throws If the brain is read-only (`mode: 'reader'`), or brain-wide `requireSubtype` * is on and the entity's type has no subtype. * @example * const id = await brain.add({ * data: 'Ada Lovelace', * type: NounType.Person, * metadata: { role: 'mathematician' } * }) */ async add(params: AddParams): Promise { this.assertWritable('add') await this.ensureInitialized() // Zero-config validation (static import for performance) validateAddParams(params) // Reserved fields arriving via the metadata bag (untyped callers — the // compile-time guard stops TypeScript callers) are normalized to their // canonical top-level location BEFORE any enforcement runs, so a // remapped subtype participates in subtype-pairing enforcement and the // indexed metadata bag carries only custom fields. params = this.remapReservedAddMetadata(params) // Tracked-field vocabulary enforcement (Layer 2). Walks both bags so a // tracked field declared at top level (e.g. 'subtype') and one declared in // metadata (e.g. 'status') both validate. this.enforceTrackedFieldValues(params.metadata as Record | undefined, 'metadata') this.enforceTrackedFieldValues( { subtype: params.subtype } as Record, 'top-level' ) // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered // via brain.requireSubtype() compose with the brain-wide strict-mode flag. // Metadata is passed so infrastructure writes (VFS) can bypass the // missing-subtype check via the `isVFSEntity` marker. this.enforceSubtypeOnAdd('add', params.type, params.subtype, params.metadata) // Id normalization (8.0): a natural string key (e.g. 'user-123') is coerced // to a STABLE UUID (v5) so the engine only ever sees a UUID, while the // caller's original string is preserved under ORIGINAL_ID_KEY for reads. // A real UUID passes through; no id mints a fresh v7. Done BEFORE the // ifAbsent check so a natural-key upsert is idempotent against the same key. const { id, originalId } = coerceNewEntityId(params.id) // ifAbsent and upsert are opposite resolutions of the same id collision — // ifAbsent SKIPS the existing entity, upsert MERGES into it. Allowing both // would be ambiguous, so it is a hard error. if (params.ifAbsent && params.upsert) { throw new Error( 'add(): ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.' ) } // ifAbsent (7.31.0) — by-ID idempotent insert. Only meaningful when a custom id // is supplied; a freshly generated UUID can never collide. Returns the existing // (canonical) id without writing if the entity is already present (no throw, // no overwrite). This check is a FAST-FAIL (skips the embedding cost); the // authoritative absence check runs in the insert's commit precondition // below, atomic with the apply, so a concurrent same-id create cannot make // both callers write. if (params.id && params.ifAbsent) { const existing = await this.storage.getNounMetadata(id) if (existing) return id } // upsert — by-ID create-or-update. Only meaningful when a custom id is supplied // (a freshly generated UUID can never collide). When the id already exists, // delegate to update() so the supplied fields MERGE into the existing entity // (metadata merge, re-embed on changed data, _rev bump, createdAt preserved) // instead of the default destructive overwrite. When the id is absent, fall // through to the guarded insert below (whose commit precondition converts a // concurrent same-id create into this same merge — never an overwrite). if (params.id && params.upsert) { const existing = await this.storage.getNounMetadata(id) if (existing) { await this.update(this.upsertMergeParams(id, params)) return id } } // Get or compute vector const vector = params.vector || (await this.embed(params.data)) // Ensure dimensions are set if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { throw new Error( `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}` ) } // Prepare metadata for storage // data is stored opaquely in the 'data' field - NOT spread into top-level metadata. // Only metadata fields are queryable via find({ where }). const storageMetadata = { ...params.metadata, // Preserve the caller's original (non-UUID) id when normalized, so reads // can surface it. A real UUID passes through with no _originalId. ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }), data: params.data, noun: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), // visibility: stored only when not 'public' (absent === public, keeps records lean) ...(params.visibility !== undefined && params.visibility !== 'public' && { visibility: params.visibility }), service: params.service, createdAt: Date.now(), updatedAt: Date.now(), _rev: 1, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), ...(params.createdBy && { createdBy: params.createdBy }) } // Build entity structure for indexing (NEW - with top-level fields) // Optional fields must use conditional spreading to match storageMetadata exactly. // If undefined values are included as explicit keys, extractIndexableFields indexes // them as '__NULL__' entries that removeFromIndex can never clean up (storageMetadata // omits those keys entirely via conditional spreading, so the fields don't match). const entityForIndexing = { id, vector, connections: new Map(), level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && params.visibility !== 'public' && { visibility: params.visibility }), ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), createdAt: Date.now(), updatedAt: Date.now(), service: params.service, data: params.data, ...(params.createdBy && { createdBy: params.createdBy }), // Only custom fields in metadata (plus the preserved original id, which // is a custom field — surfaced on read under ORIGINAL_ID_KEY). metadata: { ...(params.metadata || {}), ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) } } // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the absent/create sentinel). // All operations succeed or all rollback - prevents partial failures // ifAbsent/upsert insert leg: must-be-absent commit precondition, run // under the commit mutex against the authoritative before-image — the // planning-time absence checks above are fast-fails that can interleave // with a concurrent same-id create. Exactly one concurrent create wins; // every other caller takes its documented resolution instead of // silently overwriting the winner. const requireAbsent = Boolean(params.id && (params.ifAbsent || params.upsert)) const insertPrecommit = requireAbsent ? (before: CommitBeforeImages): void => { if (before.nouns.get(id)?.metadata) { throw new InsertPreconditionExistsSignal(id) } } : undefined const runInsert: TransactionFunction = async (tx) => { // Operation 1: Save metadata FIRST (TypeAwareStorage caching) // isNew=true: skip pre-read for rollback (entity doesn't exist yet) tx.addOperation( new SaveNounMetadataOperation(this.storage, id, storageMetadata, true) ) // Operation 2: Save vector data // isNew=true: skip pre-read for rollback (entity doesn't exist yet) tx.addOperation( new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, true) ) // Operation 3: Add to HNSW index (after entity saved) tx.addOperation( new AddToHNSWOperation(this.index, id, vector) ) // Operation 4: Add to metadata index tx.addOperation( new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing) ) } // Bounded retry closes the remaining interleavings without ever holding // a lock across the loop: a lost insert race resolves to skip (ifAbsent) // or merge (upsert); a merge that then hits a concurrent delete retries // the insert. Each persistSingleOp call constructs fresh operations, so // re-running the callback is safe. // Change feed: built only when someone is listening (zero-cost gate). const addEvents: PendingChangeEvent[] | undefined = this._changeFeed.hasListeners ? [ { kind: 'entity', op: 'add', id, entity: { id, type: String(params.type), ...(params.subtype !== undefined && { subtype: String(params.subtype) }), metadata: (params.metadata as Record) ?? {}, ...(params.service !== undefined && { service: String(params.service) }) } } ] : undefined const MAX_UPSERT_ATTEMPTS = 10 for (let attempt = 0; ; attempt++) { try { await this.persistSingleOp({ nouns: [id] }, runInsert, insertPrecommit, addEvents) break } catch (err) { if (!(err instanceof InsertPreconditionExistsSignal)) { throw err } // A concurrent writer created the entity between our absence check // and the commit. if (params.ifAbsent) { // Documented contract: return the existing id without writing. return id } // upsert: merge into the now-existing entity. A concurrent delete // between this conflict and the merge throws EntityNotFoundError — // loop back and retry the insert. try { await this.update(this.upsertMergeParams(id, params)) return id } catch (mergeErr) { if ( !(mergeErr instanceof EntityNotFoundError) || attempt >= MAX_UPSERT_ATTEMPTS ) { throw mergeErr } } } } // Aggregation hook (outside transaction — derived data, can be reconstructed) if (this._aggregationIndex) { this._aggregationIndex.onEntityAdded(id, entityForIndexing) } return id } /** * @description The `update()` param mapping `add({ upsert })` delegates to * when the target id already exists: every supplied field merges into the * existing entity (metadata merge, re-embed on changed data, `_rev` bump, * `createdAt` preserved); absent fields are left untouched. Shared by the * planning-time upsert branch and the commit-conflict retry so the two * paths can never drift. * @param id - The canonical entity id the upsert resolved to. * @param params - The original `add()` params carrying the fields to merge. * @returns The `UpdateParams` for the merging `update()` call. */ private upsertMergeParams(id: string, params: AddParams): UpdateParams { return { id, ...(params.data !== undefined && { data: params.data }), ...(params.type !== undefined && { type: params.type }), ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && { visibility: params.visibility }), ...(params.metadata !== undefined && { metadata: params.metadata }), ...(params.vector !== undefined && { vector: params.vector }), ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }) } as UpdateParams } /** * Get an entity by ID * * @param id - The unique identifier of the entity to retrieve * @returns Promise that resolves to the entity if found, null if not found * * **Entity includes:** * - `confidence` - Type classification confidence (0-1) if set * - `weight` - Entity importance/salience (0-1) if set * - All standard fields: id, type, data, metadata, vector, timestamps * * @example * // Basic entity retrieval * const entity = await brainy.get('user-123') * if (entity) { * console.log('Found entity:', entity.data) * console.log('Created at:', new Date(entity.createdAt)) * } else { * console.log('Entity not found') * } * * @example * // Accessing confidence and weight * const entity = await brainy.get('concept-456') * if (entity) { * console.log(`Type: ${entity.type}`) * console.log(`Confidence: ${entity.confidence ?? 'N/A'}`) * console.log(`Weight: ${entity.weight ?? 'N/A'}`) * } * * @example * // Working with typed entities * interface User { * name: string * email: string * } * * const brainy = new Brainy({ storage: 'filesystem' }) * const user = await brainy.get('user-456') * if (user) { * // TypeScript knows user.metadata is of type User * console.log(`Hello ${user.metadata.name}`) * } * * @example * // Safe retrieval with error handling * try { * const entity = await brainy.get('document-789') * if (!entity) { * throw new Error('Document not found') * } * * // Process the entity * return { * id: entity.id, * content: entity.data, * type: entity.type, * metadata: entity.metadata * } * } catch (error) { * console.error('Failed to retrieve entity:', error) * return null * } * * @example * // Batch retrieval pattern * const ids = ['doc-1', 'doc-2', 'doc-3'] * const entities = await Promise.all( * ids.map(id => brainy.get(id)) * ) * const foundEntities = entities.filter(entity => entity !== null) * console.log(`Found ${foundEntities.length} out of ${ids.length} entities`) * * @example * // Using with async iteration * const entityIds = ['user-1', 'user-2', 'user-3'] * * for (const id of entityIds) { * const entity = await brainy.get(id) * if (entity) { * console.log(`Processing ${entity.type}: ${id}`) * // Process entity... * } * } */ /** * Get an entity by ID * * **Performance**: Optimized for metadata-only reads by default * - **Default (metadata-only)**: 10ms, 300 bytes - 76-81% faster * - **Full entity (includeVectors: true)**: 43ms, 6KB - when vectors needed * * **When to use metadata-only (default)**: * - VFS operations (readFile, stat, readdir) - 100% of cases * - Existence checks: `if (await brain.get(id))` * - Metadata inspection: `entity.metadata`, `entity.data`, `entity.type` * - Relationship traversal: `brain.related({ from: id })` * * **When to include vectors**: * - Computing similarity on this specific entity: `brain.similar({ to: entity.vector })` * - Manual vector operations: `cosineSimilarity(entity.vector, otherVector)` * * @param id - Entity ID to retrieve * @param options - Retrieval options (includeVectors defaults to false) * @returns Entity or null if not found * * @example * ```typescript * // ✅ FAST: Metadata-only (default) - 10ms, 300 bytes * const entity = await brain.get(id) * console.log(entity.data, entity.metadata) // ✅ Available * console.log(entity.vector.length) // 0 (stub vector) * * // ✅ FULL: Include vectors when needed - 43ms, 6KB * const fullEntity = await brain.get(id, { includeVectors: true }) * const similarity = cosineSimilarity(fullEntity.vector, otherVector) * * // ✅ Existence check (metadata-only is perfect) * if (await brain.get(id)) { * console.log('Entity exists') * } * * // ✅ VFS automatically benefits (no code changes needed) * await vfs.readFile('/file.txt') // 53ms → 10ms (81% faster) * ``` * * @performance * - Metadata-only: 76-81% faster, 95% less bandwidth, 87% less memory * - Full entity: Same (no regression) * - VFS operations: 81% faster with zero code changes * */ async get(id: string, options?: GetOptions): Promise | null> { // Canonical read: a get resolves an entity by id straight from storage and // consults no derived index — it must not wait on any family's migration. await this.ensureInitialized({ needs: [] }) this.warnIfReadsDegraded('get') // Id normalization (8.0): a caller may read by their natural key — resolve // it to the same canonical UUID add() stored. A real UUID passes through. // Only a non-empty string is normalized; empty/null/undefined fall through // to the storage layer's structural-validation throw (preserved contract). if (typeof id === 'string' && id !== '') { id = resolveEntityId(id) } // Route to metadata-only or full entity based on options const includeVectors = options?.includeVectors ?? false // Default: metadata-only (fast) if (includeVectors) { // FULL PATH: Load vector + metadata (6KB, 43ms) // Used when: Computing similarity on this entity, manual vector operations const noun = await this.storage.getNoun(id) if (!noun) { return null } return this.convertNounToEntity(noun) } else { // FAST PATH: Metadata-only (300 bytes, 10ms) - DEFAULT // Used when: VFS operations, existence checks, metadata inspection (94% of calls) const metadata = await this.storage.getNounMetadata(id) if (!metadata) { return null } return this.convertMetadataToEntity(id, metadata) } } /** * Batch get multiple entities by IDs (Cloud Storage Optimization) * * **Performance**: Eliminates N+1 query pattern * - Current: N × get() = N × 300ms cloud latency = 3-6 seconds for 10-20 entities * - Batched: 1 × batchGet() = 1 × 300ms cloud latency = 0.3 seconds ✨ * * **Use cases:** * - VFS tree traversal (get all children at once) * - Relationship traversal (get all targets at once) * - Import operations (batch existence checks) * - Admin tools (fetch multiple entities for listing) * * @param ids Array of entity IDs to fetch * @param options Get options (includeVectors defaults to false for speed) * @returns Map of id → entity (only successfully fetched entities included) * * @example * ```typescript * // VFS getChildren optimization * const childIds = relations.map(r => r.to) * const childrenMap = await brain.batchGet(childIds) * const children = childIds.map(id => childrenMap.get(id)).filter(Boolean) * ``` */ async batchGet(ids: string[], options?: GetOptions): Promise>> { // Canonical read (see get): resolves by id from storage, no derived index. await this.ensureInitialized({ needs: [] }) // Id normalization (8.0): resolve each id to its canonical UUID so callers // can batch-read by natural key. resolveEntityId is idempotent on real // UUIDs, so internal callers passing canonical ids are unaffected. The // returned map is keyed by the canonical (stored) id. ids = ids.map((id) => resolveEntityId(id)) const results = new Map>() if (ids.length === 0) return results const includeVectors = options?.includeVectors ?? false if (includeVectors) { // FULL PATH optimized with batch vector loading (10x faster on GCS) // GCS: 10 entities with vectors = 1×50ms vs 10×50ms = 500ms (10x faster) const nounsMap = await this.storage.getNounBatch(ids) for (const [id, noun] of nounsMap.entries()) { const entity = await this.convertNounToEntity(noun) results.set(id, entity) } } else{ // FAST PATH: Metadata-only batch (default) - OPTIMIZED const metadataMap = await this.storage.getNounMetadataBatch(ids) for (const [id, metadata] of metadataMap.entries()) { const entity = await this.convertMetadataToEntity(id, metadata) results.set(id, entity) } } return results } /** * Create a flattened Result object from entity * Flattens commonly-used entity fields to top level for convenience */ private createResult(id: string, score: number, entity: Entity, explanation?: ScoreExplanation): Result { return { id, score, // Flatten common entity fields to top level type: entity.type, subtype: entity.subtype, visibility: entity.visibility, metadata: entity.metadata, data: entity.data, confidence: entity.confidence, weight: entity.weight, _rev: entity._rev, // Preserve full entity for backward compatibility entity, // Optional score explanation ...(explanation && { explanation }) } } /** * Convert a noun from storage to an entity (SIMPLIFIED!) * * Dramatically simplified - standard fields moved to top-level * - Extracts standard fields from metadata (storage format) * - Returns entity with standard fields at top-level (in-memory format) * - metadata contains ONLY custom user fields */ private async convertNounToEntity(noun: any): Promise> { // Storage adapters ALREADY extract standard fields to top-level! // Just read from top-level fields of HNSWNounWithMetadata // Clean structure with standard fields at top-level const entity: Entity = { id: noun.id, vector: noun.vector, type: noun.type || NounType.Thing, subtype: noun.subtype, visibility: noun.visibility, // Standard fields at top-level confidence: noun.confidence, weight: noun.weight, createdAt: noun.createdAt || Date.now(), updatedAt: noun.updatedAt || Date.now(), service: noun.service, data: noun.data, createdBy: noun.createdBy, // 7.31.0 — surface revision counter (defaults to 1 for pre-7.31.0 entities) _rev: typeof noun._rev === 'number' ? noun._rev : 1, // ONLY custom user fields in metadata (already separated by storage adapter) metadata: noun.metadata as T } return entity } /** * Convert metadata-only to entity (FAST PATH!) * * Used when vectors are NOT needed (94% of brain.get() calls): * - VFS operations (readFile, stat, readdir) * - Existence checks * - Metadata inspection * - Relationship traversal * * Performance: 76-81% faster, 95% less bandwidth, 87% less memory * - Metadata-only: 10ms, 300 bytes * - Full entity: 43ms, 6KB * * @param id - Entity ID * @param metadata - Metadata from storage.getNounMetadata() * @returns Entity with stub vector (Float32Array(0)) * */ private async convertMetadataToEntity(id: string, metadata: any): Promise> { // Metadata-only entity (no vector loading) // This is 76-81% faster for operations that don't need semantic similarity // Canonical reserved/custom split — same single source of truth as every // other read path (see src/types/reservedFields.ts). const { reserved, custom } = splitNounMetadataRecord(metadata) const entity: Entity = { id, vector: [], // Stub vector (empty array - vectors not loaded for metadata-only) type: (reserved.noun as NounType) || NounType.Thing, subtype: reserved.subtype as string | undefined, visibility: reserved.visibility as EntityVisibility | undefined, // Standard fields from metadata confidence: reserved.confidence as number | undefined, weight: reserved.weight as number | undefined, createdAt: (reserved.createdAt as number) || Date.now(), updatedAt: (reserved.updatedAt as number) || Date.now(), service: reserved.service as string | undefined, data: reserved.data, createdBy: reserved.createdBy as Entity['createdBy'], // 7.31.0 — surface revision counter (defaults to 1 for pre-7.31.0 entities) _rev: typeof reserved._rev === 'number' ? reserved._rev : 1, // Custom user fields (standard fields removed, only custom remain) metadata: custom as T } return entity } /** One-shot registry for reserved-field warnings (per process, per method+field). */ private static warnedReservedFields = new Set() /** * @description Resolve the human-readable "correct write path" guidance for a * reserved field on a given write method. Single source of truth shared by the * `'throw'` (Error message) and `'warn'` (one-shot warning) paths so the two * never drift. The trio `confidence` / `weight` / `subtype` and the * add()/relate()-time fields `service` / `createdBy` / `visibility` map to a * dedicated param; everything else is system-managed. * @param method - The public write method the bag arrived through. * @param field - The reserved field name found in the metadata bag. * @returns Guidance naming the correct way to set the field. */ private reservedWritePath( method: 'add' | 'update' | 'relate' | 'updateRelation', field: string ): string { const typeParam = "the top-level 'type' param" switch (field) { case 'noun': case 'verb': return typeParam case 'data': return "the top-level 'data' param" case 'confidence': return "the 'confidence' param" case 'weight': return "the 'weight' param" case 'subtype': return "the 'subtype' param" case 'visibility': return "the 'visibility' param ('public' | 'internal')" case 'service': return method === 'add' ? "the 'service' param of add()" : method === 'relate' ? "the 'service' param of relate()" : 'nothing — service is fixed at create time' case 'createdBy': return method === 'add' ? "the 'createdBy' param of add()" : 'nothing — createdBy is system-managed' case 'createdAt': return 'nothing — creation time is set automatically' case 'updatedAt': return 'nothing — set automatically on every write' case '_rev': return method === 'update' ? "the 'ifRev' param for optimistic concurrency" : 'nothing — revisions are system-managed' default: return 'a dedicated top-level param' } } /** * @description Enforce {@link BrainyConfig.reservedFieldPolicy} for reserved * fields found inside a metadata bag. Called by every write-path remap once * the bag has been split and at least one reserved key is present. * * - `'throw'` (default): throw a clear Error naming every offending key and * its correct write path. The caller never reaches the remap. * - `'warn'`: emit a ONE-SHOT (per method+field, per process) warning for * EVERY reserved key found — both the user-mutable fields that are about to * be remapped and the system-managed fields that are about to be dropped — * then fall through to the legacy remap. * - `'remap'`: silent legacy remap, no warning. * * @param method - The public write method the bag arrived through. * @param reserved - The reserved half of the split metadata bag (non-empty). * @param reservedListName - `'RESERVED_ENTITY_FIELDS'` or * `'RESERVED_RELATION_FIELDS'` — named in the thrown Error for discoverability. * @returns `true` when the caller should proceed with the legacy remap * (`'warn'` / `'remap'`); `'throw'` never returns (it throws first). * @throws {Error} When the policy is `'throw'` and any reserved key is present. */ private enforceReservedPolicy( method: 'add' | 'update' | 'relate' | 'updateRelation', reserved: Partial>, reservedListName: 'RESERVED_ENTITY_FIELDS' | 'RESERVED_RELATION_FIELDS' ): boolean { const policy = this.config.reservedFieldPolicy ?? 'throw' const keys = Object.keys(reserved) if (keys.length === 0) return true if (policy === 'throw') { const detail = keys .map((k) => { const path = this.reservedWritePath(method, k) // System-managed fields resolve to a "nothing — …" sentinel; phrase // those as "is system-managed" rather than "pass it as the nothing". return path.startsWith('nothing') ? `metadata.${k} is a reserved field (${path.replace(/^nothing\s*—\s*/, '')}) and cannot be set through ${method}()` : `metadata.${k} is a reserved field — pass it as ${path} to ${method}()` }) .join('; ') throw new Error( `${detail} (reserved: see ${reservedListName}). ` + `Set reservedFieldPolicy:'remap' to opt into legacy remapping, ` + `or reservedFieldPolicy:'warn' to remap with a warning.` ) } if (policy === 'warn') { // One-shot warning for EVERY reserved key (today only system-managed ones // warn — this closes that gap so user-mutable remaps are visible too). for (const k of keys) { this.warnReservedRemapped(method, k, this.reservedWritePath(method, k)) } } // 'warn' and 'remap' both fall through to the legacy remap. return true } /** * @description One-shot (per method+field, per process) warning that a * reserved field arrived inside a metadata bag under the `'warn'` policy. The * wording is neutral on "remapped vs dropped" — `reservedWritePath()` already * tells the caller where the value goes (a dedicated param, or "nothing"). * @param method - The public write method the bag arrived through. * @param field - The reserved field name found in the bag. * @param rightPath - Guidance naming the correct write path. */ private warnReservedRemapped(method: string, field: string, rightPath: string): void { const key = `${method}:${field}` if (Brainy.warnedReservedFields.has(key)) return Brainy.warnedReservedFields.add(key) // System-managed fields resolve to a "nothing — …" sentinel; phrase the // guidance so it reads cleanly in both the remapped and dropped cases. const guidance = rightPath.startsWith('nothing') ? `it is ${rightPath.replace(/^nothing\s*—\s*/, '')} and was dropped` : `set it via ${rightPath} instead` prodLog.warn( `[brainy] ${method}(): '${field}' is a reserved field and was found inside the ` + `metadata bag — ${guidance}. (Legacy remap applied because ` + `reservedFieldPolicy is 'warn'. This warning is shown once per field per process.)` ) } /** * @description Normalize an `add()` params object with respect to * Brainy-reserved fields arriving inside `metadata` (untyped callers only — * the compile-time guard on `AddParams.metadata` stops TypeScript callers). * Governed by {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): * `'throw'` rejects the write naming the offending key(s); `'warn'`/`'remap'` * fall through to the legacy remap, where fields with a dedicated `add()` * param (`confidence`, `weight`, `subtype`, `visibility`, `service`, * `createdBy`) are remapped to that param unless the caller also passed it * explicitly (top-level wins) and system-managed fields (`noun`, `data`, * `createdAt`, `updatedAt`, `_rev`) are dropped. A remapped `subtype` flows * through subtype-pairing enforcement exactly like a top-level one. * @param params - The caller's add params (not mutated). * @returns Params with reserved fields normalized out of `metadata`. * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. */ private remapReservedAddMetadata(params: AddParams): AddParams { const bag = params.metadata as Record | undefined if (!bag || typeof bag !== 'object') return params const { reserved, custom } = splitNounMetadataRecord(bag) if (Object.keys(reserved).length === 0) return params // Policy gate: 'throw' (default) throws here; 'warn' warns once per key then // remaps; 'remap' silently remaps. (Throw never returns.) this.enforceReservedPolicy('add', reserved, 'RESERVED_ENTITY_FIELDS') const createdBy = reserved.createdBy as { augmentation?: unknown; version?: unknown } | undefined const createdByValid = typeof createdBy === 'object' && createdBy !== null && typeof createdBy.augmentation === 'string' && typeof createdBy.version === 'string' return { ...params, metadata: custom as AddParams['metadata'], ...(params.confidence === undefined && typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), ...(params.weight === undefined && typeof reserved.weight === 'number' && { weight: reserved.weight }), ...(params.subtype === undefined && typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), ...(params.visibility === undefined && (reserved.visibility === 'public' || reserved.visibility === 'internal') && { visibility: reserved.visibility as 'public' | 'internal' }), ...(params.service === undefined && typeof reserved.service === 'string' && { service: reserved.service }), ...(params.createdBy === undefined && createdByValid && { createdBy: createdBy as { augmentation: string; version: string } }) } } /** * @description Normalize an `update()` params object with respect to * Brainy-reserved fields arriving inside the metadata patch — the `update()` * mirror of {@link remapReservedAddMetadata}, closing the historical trap * where `add({metadata:{confidence}})` lifted the field but * `update({metadata:{confidence}})` silently dropped it (the patch value * survived the merge and was then clobbered by the preserve-existing * spread; a production consumer's confidence-evolution writes no-oped until * read back). Governed by {@link BrainyConfig.reservedFieldPolicy} (default * `'throw'`): `'throw'` rejects the write; `'warn'`/`'remap'` remap * user-mutable fields (`confidence`, `weight`, `subtype`) to their dedicated * param unless the caller also passed it (top-level wins) and drop everything * else (`noun`, `data`, `createdAt`, `updatedAt`, `service`, `createdBy`, * `_rev`) as system-managed or fixed at `add()` time. * @param params - The caller's update params (not mutated). * @returns Params with reserved fields normalized out of `metadata`. * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. */ private remapReservedUpdateMetadata(params: UpdateParams): UpdateParams { const bag = params.metadata as Record | undefined if (!bag || typeof bag !== 'object') return params const { reserved, custom } = splitNounMetadataRecord(bag) if (Object.keys(reserved).length === 0) return params // Policy gate: 'throw' (default) throws; 'warn' warns once per key then // remaps; 'remap' silently remaps. this.enforceReservedPolicy('update', reserved, 'RESERVED_ENTITY_FIELDS') return { ...params, metadata: custom as UpdateParams['metadata'], ...(params.confidence === undefined && typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), ...(params.weight === undefined && typeof reserved.weight === 'number' && { weight: reserved.weight }), ...(params.subtype === undefined && typeof reserved.subtype === 'string' && { subtype: reserved.subtype }) } } /** * @description Normalize a `relate()` params object with respect to * Brainy-reserved fields arriving inside `metadata` — the relationship * mirror of {@link remapReservedAddMetadata}. Governed by * {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): `'throw'` * rejects the write; `'warn'`/`'remap'` remap fields with a dedicated * `relate()` param (`confidence`, `weight`, `subtype`, `visibility`, * `service`) to that param (top-level wins) and drop system-managed fields * (`verb`, `data`, `createdAt`, `updatedAt`, `createdBy`, `_rev`). * @param params - The caller's relate params (not mutated). * @returns Params with reserved fields normalized out of `metadata`. * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. */ private remapReservedRelateMetadata(params: RelateParams): RelateParams { const bag = params.metadata as Record | undefined if (!bag || typeof bag !== 'object') return params const { reserved, custom } = splitVerbMetadataRecord(bag) if (Object.keys(reserved).length === 0) return params // Policy gate: 'throw' (default) throws; 'warn' warns once per key then // remaps; 'remap' silently remaps. this.enforceReservedPolicy('relate', reserved, 'RESERVED_RELATION_FIELDS') return { ...params, metadata: custom as RelateParams['metadata'], ...(params.confidence === undefined && typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), ...(params.weight === undefined && typeof reserved.weight === 'number' && { weight: reserved.weight }), ...(params.subtype === undefined && typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), ...(params.visibility === undefined && (reserved.visibility === 'public' || reserved.visibility === 'internal') && { visibility: reserved.visibility as 'public' | 'internal' }), ...(params.service === undefined && typeof reserved.service === 'string' && { service: reserved.service }) } } /** * @description Normalize an `updateRelation()` params object with respect * to Brainy-reserved fields arriving inside the metadata patch — the * relationship mirror of {@link remapReservedUpdateMetadata}. Governed by * {@link BrainyConfig.reservedFieldPolicy} (default `'throw'`): `'throw'` * rejects the write; `'warn'`/`'remap'` remap user-mutable fields * (`confidence`, `weight`, `subtype`, `visibility`) to their dedicated param * (top-level wins) and drop everything else. * @param params - The caller's update-relation params (not mutated). * @returns Params with reserved fields normalized out of `metadata`. * @throws {Error} When `reservedFieldPolicy` is `'throw'` and the bag carries a reserved key. */ private remapReservedUpdateRelationMetadata( params: UpdateRelationParams ): UpdateRelationParams { const bag = params.metadata as Record | undefined if (!bag || typeof bag !== 'object') return params const { reserved, custom } = splitVerbMetadataRecord(bag) if (Object.keys(reserved).length === 0) return params // Policy gate: 'throw' (default) throws; 'warn' warns once per key then // remaps; 'remap' silently remaps. this.enforceReservedPolicy('updateRelation', reserved, 'RESERVED_RELATION_FIELDS') return { ...params, metadata: custom as UpdateRelationParams['metadata'], ...(params.confidence === undefined && typeof reserved.confidence === 'number' && { confidence: reserved.confidence }), ...(params.weight === undefined && typeof reserved.weight === 'number' && { weight: reserved.weight }), ...(params.subtype === undefined && typeof reserved.subtype === 'string' && { subtype: reserved.subtype }), ...(params.visibility === undefined && (reserved.visibility === 'public' || reserved.visibility === 'internal') && { visibility: reserved.visibility as 'public' | 'internal' }) } } /** * Update an existing entity * * Merges metadata by default — new fields are added, existing fields are overwritten, * and omitted fields are preserved. Set `merge: false` to replace metadata entirely. * If `data` is provided, the entity is re-embedded and re-indexed in HNSW. * * **Data vs Metadata:** * - `data`: Content used for vector embeddings (searchable via semantic similarity / HNSW). * NOT queryable via `where` filters. Pass a string for text search, or any value for storage. * - `metadata`: Structured fields indexed by MetadataIndex. Queryable via `where` filters * in `find()`. Put anything you want to filter/query on here. * * @param params - Update parameters * @param params.id - UUID of the entity to update (required) * @param params.data - New content to re-embed (triggers HNSW re-indexing) * @param params.type - Change entity type classification * @param params.metadata - Metadata fields to merge (or replace if merge=false) * @param params.merge - If true (default), merges metadata; if false, replaces it entirely * @param params.vector - Pre-computed vector (skips embedding) * @param params.confidence - Update type classification confidence (0-1) * @param params.weight - Update entity importance/salience (0-1) * * @example Update metadata (merge by default) * ```typescript * await brain.update({ * id: entityId, * metadata: { status: 'reviewed', rating: 4.5 } * // Existing metadata fields preserved, only status and rating changed * }) * ``` * * @example Update data (re-embeds and re-indexes) * ```typescript * await brain.update({ * id: entityId, * data: 'Updated description of the concept' * // Vector is recomputed, HNSW index updated * }) * ``` * * @example Replace metadata entirely * ```typescript * await brain.update({ * id: entityId, * metadata: { onlyThisField: true }, * merge: false // All previous metadata removed * }) * ``` */ async update(params: UpdateParams): Promise { this.assertWritable('update') await this.ensureInitialized() // Zero-config validation (static import for performance) validateUpdateParams(params) // Id normalization (8.0): resolve a natural key to the canonical UUID add() // stored, so update() targets the same entity. A real UUID passes through. params = { ...params, id: resolveEntityId(params.id) } // Reserved fields arriving via the metadata patch are remapped to their // canonical top-level location, mirroring add()'s lift. Without this the // patch value survived the merge but was then clobbered by the // preserve-existing spreads below — a silent no-op consumers could only // detect by reading values back. User-mutable fields (confidence, // weight, subtype) remap unless the same field was also passed top-level // (top-level wins); system-managed fields are dropped with a one-shot // warning naming the right path. params = this.remapReservedUpdateMetadata(params) // Tracked-field vocabulary enforcement (Layer 2). Same as add() — the // metadata bag carries fields registered via trackField(), and subtype is // a tracked top-level candidate. this.enforceTrackedFieldValues(params.metadata as Record | undefined, 'metadata') if (params.subtype !== undefined) { this.enforceTrackedFieldValues( { subtype: params.subtype } as Record, 'top-level' ) } // Subtype pairing enforcement on update (7.30.0). The effective NounType after // the update is `params.type ?? existing.type`; we look it up if needed. if (params.subtype !== undefined || params.type !== undefined) { const existing = await this.get(params.id) const effectiveType = params.type ?? existing?.type const effectiveSubtype = params.subtype !== undefined ? params.subtype : existing?.subtype const effectiveMetadata = params.metadata ?? existing?.metadata this.enforceSubtypeOnAdd('update', effectiveType, effectiveSubtype, effectiveMetadata) } // Get existing entity with vectors (fix for regression) // We need includeVectors: true because: // 1. SaveNounOperation requires the vector // 2. HNSW reindexing operations need the original vector const existing = await this.get(params.id, { includeVectors: true }) if (!existing) { throw new EntityNotFoundError(params.id) } // ifRev (7.31.0) — optimistic concurrency. If caller supplied ifRev, the persisted // _rev must match exactly. `_rev` is reserved: every read path surfaces it ONLY // top-level (entities without one are read as rev 1), so that is the one place // to look. This check is purely a FAST-FAIL (avoids paying the embedding // cost on an obviously stale expectation) — the authoritative check runs // in the commit precondition below, under the commit mutex, where it is // atomic with the apply. Concurrent same-rev updates all pass HERE but // exactly one survives THERE. const currentRev = typeof existing._rev === 'number' ? existing._rev : 1 if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) { throw new RevisionConflictError(params.id, params.ifRev, currentRev) } // Resolve the updated vector: an explicit `vector` always wins (the // UpdateParams contract — a new pre-computed vector, with or without // new `data`); otherwise new `data` re-embeds; otherwise the existing // vector is kept. Any vector change re-indexes HNSW below. let vector = existing.vector if (params.vector) { if (this.dimensions && params.vector.length !== this.dimensions) { throw new Error( `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` ) } vector = params.vector } else if (params.data) { vector = await this.embed(params.data) } const needsReindexing = Boolean(params.data || params.type || params.vector) // Always update the noun with new metadata const newMetadata = params.merge !== false ? { ...existing.metadata, ...params.metadata } : params.metadata || existing.metadata // Prepare updated metadata object // data is stored opaquely in the 'data' field - NOT spread into top-level metadata. const updatedMetadata = { ...newMetadata, data: params.data !== undefined ? params.data : existing.data, noun: params.type || existing.type, service: existing.service, createdAt: existing.createdAt, updatedAt: Date.now(), _rev: currentRev + 1, // Update confidence and weight if provided, otherwise preserve existing ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), ...(params.confidence === undefined && existing.confidence !== undefined && { confidence: existing.confidence }), ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), // Update subtype if provided, otherwise preserve existing ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), // Visibility: take the new value if provided, else preserve existing. Stored only // when the effective value is not 'public' (absent === public, keeps records lean). // A change to 'public' therefore drops the field entirely. ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { visibility: params.visibility ?? existing.visibility }) } // Build entity structure for metadata index (with top-level fields) const entityForIndexing = { id: params.id, vector, connections: new Map(), level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { visibility: params.visibility ?? existing.visibility }), confidence: params.confidence !== undefined ? params.confidence : existing.confidence, weight: params.weight !== undefined ? params.weight : existing.weight, createdAt: existing.createdAt, updatedAt: Date.now(), service: existing.service, data: params.data !== undefined ? params.data : existing.data, createdBy: existing.createdBy, // Only custom fields in metadata metadata: newMetadata } // Authoritative CAS + honest rev stamp, run by the generation store // UNDER the commit mutex against the just-read before-image — the one // point where check-and-apply is atomic (the fast-fail above can // interleave with concurrent writers; this cannot). Also re-stamps // `_rev` from the authoritative base so the counter stays monotonic // even for concurrent non-CAS updates. The staged operations below // capture `updatedMetadata` by reference, so the re-stamp lands. const casPrecommit = (before: CommitBeforeImages): void => { const beforeMeta = before.nouns.get(params.id)?.metadata as | { _rev?: unknown } | null | undefined if (!beforeMeta) { // A concurrent remove() deleted the entity after our read. A CAS // caller gets the truth; a plain update keeps its long-standing // last-writer-wins semantics (the staged write re-creates it). if (typeof params.ifRev === 'number') { throw new EntityNotFoundError(params.id) } return } const authoritativeRev = typeof beforeMeta._rev === 'number' ? beforeMeta._rev : 1 if (typeof params.ifRev === 'number' && params.ifRev !== authoritativeRev) { throw new RevisionConflictError(params.id, params.ifRev, authoritativeRev) } updatedMetadata._rev = authoritativeRev + 1 } // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the entity's prior state). await this.persistSingleOp({ nouns: [params.id] }, async (tx) => { // Operation 1: Update metadata FIRST (updates type cache) tx.addOperation( new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata) ) // Operation 2: Update vector data (will use updated type cache) tx.addOperation( new SaveNounOperation(this.storage, { id: params.id, vector, connections: new Map(), level: 0 }) ) // Operation 3-4: Update HNSW index (remove and re-add if reindexing needed) if (needsReindexing) { tx.addOperation( new RemoveFromHNSWOperation(this.index, params.id, existing.vector) ) tx.addOperation( new AddToHNSWOperation(this.index, params.id, vector) ) } // Operation 5-6: Update metadata index (remove old, add new) // FIX: Include ALL indexed fields in removalMetadata (not just type) // Previously, only metadata + type was removed, but entityForIndexing includes: // confidence, weight, createdAt, updatedAt, service, data, createdBy // This asymmetry caused 7 fields to accumulate on EVERY update, eventually // making queries return 0 results (77x overcounting at scale). // // DEBUG: Log what we're removing and adding // console.log('[UPDATE DEBUG] existing.metadata:', JSON.stringify(existing.metadata)) // console.log('[UPDATE DEBUG] entityForIndexing keys:', Object.keys(entityForIndexing)) // // FIX: removalMetadata must MATCH entityForIndexing structure // entityForIndexing has: { type, confidence, ..., metadata: {...} } // So removalMetadata must also have: { type, confidence, ..., metadata: {...} } const removalMetadata = { type: existing.type, confidence: existing.confidence, weight: existing.weight, createdAt: existing.createdAt, updatedAt: existing.updatedAt, // CRITICAL: removes old timestamp service: existing.service, data: existing.data, createdBy: existing.createdBy, metadata: existing.metadata // CRITICAL: keep as nested 'metadata' property! } tx.addOperation( new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata) ) tx.addOperation( new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing) ) }, casPrecommit, this._changeFeed.hasListeners ? [ { kind: 'entity', op: 'update', id: params.id, entity: { id: params.id, type: String(entityForIndexing.type), ...(entityForIndexing.subtype !== undefined && { subtype: String(entityForIndexing.subtype) }), metadata: (newMetadata as Record) ?? {}, ...(entityForIndexing.service !== undefined && { service: String(entityForIndexing.service) }) } } ] : undefined) // Aggregation hook (outside transaction — derived data) if (this._aggregationIndex) { const oldEntityForAgg = { type: existing.type, service: existing.service, data: existing.data, metadata: existing.metadata } this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) } } /** * Remove an entity and all its relationships * * Removes the entity from all indexes (HNSW vector index, MetadataIndex, * GraphAdjacencyIndex) and deletes all relationships where this entity * is the source or target. All operations are executed atomically. * * Named `remove` to match the transact operation vocabulary * (`{add, update, remove, relate, unrelate}`). * * @param id - UUID of the entity to remove. Silently returns for invalid/null IDs. * * @example * ```typescript * await brain.remove(entityId) * const entity = await brain.get(entityId) // null * ``` */ async remove(id: string): Promise { this.assertWritable('remove') // Handle invalid IDs gracefully if (!id || typeof id !== 'string') { return // Silently return for invalid IDs } await this.ensureInitialized() // Id normalization (8.0): resolve a natural key to the canonical UUID add() // stored, so remove() deletes the same entity. A real UUID passes through. id = resolveEntityId(id) // Get entity metadata and related verbs before deletion const metadata = await this.storage.getNounMetadata(id) const noun = await this.storage.getNoun(id) const verbs = await this.storage.getVerbsBySource(id) const targetVerbs = await this.storage.getVerbsByTarget(id) const allVerbs = [...verbs, ...targetVerbs] // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation covering the entity AND its cascade-deleted // relationships (each id's before-image = its prior state). await this.persistSingleOp( { nouns: [id], verbs: allVerbs.map((v) => v.id) }, async (tx) => { // Operation 1: Remove from vector index if (noun) { tx.addOperation( new RemoveFromHNSWOperation(this.index, id, noun.vector) ) } // Operation 2: Remove from metadata index if (metadata) { tx.addOperation( new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata) ) } // Operation 3: Delete noun (full removal). The pre-read metadata rides // along so the count decrement never depends on re-reading the record // being removed (a null re-read must not silently skip it). tx.addOperation( new DeleteNounMetadataOperation(this.storage, id, metadata) ) // Operations 4+: Delete all related verbs atomically for (const verb of allVerbs) { // Remove from graph index (endpoint ints resolved up front so a // rollback can re-add through the BigInt addVerb contract) const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) tx.addOperation( new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration) ) // Delete verb metadata tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) ) } }, undefined, this._changeFeed.hasListeners ? [ // The entity delete (payload = last state, from the pre-delete read) // plus one unrelate per cascade-deleted relationship. { kind: 'entity', op: 'remove', id, ...(metadata && { entity: this.entityViewFromRawRecord(id, metadata as Record) }) }, ...allVerbs.map( (v): PendingChangeEvent => ({ kind: 'relation', op: 'unrelate', id: v.id, relation: { id: v.id, from: v.sourceId, to: v.targetId, type: String(v.verb) } }) ) ] : undefined) // Aggregation hook (outside transaction — derived data) if (this._aggregationIndex && metadata) { // Reconstruct entity-like object from stored metadata via the // canonical reserved/custom split (the hand-rolled destructure here // missed subtype/_rev, leaking them into the aggregation view). const { reserved, custom } = splitNounMetadataRecord(metadata) const entityForAgg = { type: reserved.noun, service: reserved.service, data: reserved.data, metadata: custom } this._aggregationIndex.onEntityDeleted(id, entityForAgg) } } // ============= RELATIONSHIP OPERATIONS ============= // --- 8.0 u64 boundary helpers ------------------------------------------- // UUID ↔ int conversion happens ONCE here at the coordinator boundary: // `getOrAssign` on writes, `getInt` on reads (undefined → the entity was // never mapped, i.e. it has no relations — return empty without calling the // provider). Provider returns convert back via `getUuid` (entities) and // `verbIntsToIds` + the warm cache (verbs). `Number(bigint)` narrowing is // lossless under the shipped EntityIdSpaceExceeded u32 guard. /** * Resolve a UUID to its entity int for a READ — `undefined` means the * entity was never mapped and therefore has no relations. */ private graphEntityInt(uuid: string): bigint | undefined { const intId = this.metadataIndex.getIdMapper().getInt(uuid) return intId === undefined ? undefined : BigInt(intId) } /** * Resolve a verb's endpoint UUIDs to entity ints for a WRITE * (`getOrAssign`), mirroring them onto `verb.sourceInt`/`verb.targetInt` * (derived state — never persisted to storage JSON) before the verb is * handed to the graph-index provider. Accepts any verb shape carrying * endpoint UUIDs (`GraphVerb`, `HNSWVerbWithMetadata`, …). */ private resolveVerbEndpointInts( verb: Pick & { sourceInt?: bigint; targetInt?: bigint } ): { sourceInt: bigint; targetInt: bigint } { const idMapper = this.metadataIndex.getIdMapper() const sourceInt = BigInt(idMapper.getOrAssign(verb.sourceId)) const targetInt = BigInt(idMapper.getOrAssign(verb.targetId)) verb.sourceInt = sourceInt verb.targetInt = targetInt return { sourceInt, targetInt } } /** * Convert provider-returned entity ints back to UUIDs, dropping ints the * mapper no longer knows (deleted entities). */ private entityIntsToUuids(entityInts: bigint[]): string[] { const idMapper = this.metadataIndex.getIdMapper() const uuids: string[] = [] for (const entityInt of entityInts) { const uuid = idMapper.getUuid(Number(entityInt)) if (uuid !== undefined) uuids.push(uuid) } return uuids } /** * Record one verb-int → verb-id pair in the bounded warm cache, evicting * the oldest entries (insertion order) past the cap. */ private cacheVerbInt(verbInt: bigint, verbId: string): void { if (!this.verbIntWarmCache.has(verbInt) && this.verbIntWarmCache.size >= Brainy.VERB_INT_WARM_CACHE_MAX) { // Evict oldest insertions until under cap (single eviction in the // common case; loop guards against future cap reductions). for (const oldest of this.verbIntWarmCache.keys()) { this.verbIntWarmCache.delete(oldest) if (this.verbIntWarmCache.size < Brainy.VERB_INT_WARM_CACHE_MAX) break } } this.verbIntWarmCache.set(verbInt, verbId) } /** * Resolve provider-returned verb ints to verb-id strings: warm cache first, * then one batched `verbIntsToIds` call for the misses (which also refills * the cache). Unknown ints are dropped. */ private async resolveVerbIntsToIds(verbInts: bigint[]): Promise { if (verbInts.length === 0) return [] const resolved = new Array(verbInts.length) const missIndices: number[] = [] const missInts: bigint[] = [] for (let i = 0; i < verbInts.length; i++) { const cached = this.verbIntWarmCache.get(verbInts[i]) if (cached !== undefined) { resolved[i] = cached } else { missIndices.push(i) missInts.push(verbInts[i]) } } if (missInts.length > 0) { const ids = await this.graphIndex.verbIntsToIds(missInts) for (let j = 0; j < missInts.length; j++) { const id = ids[j] if (id !== null) { resolved[missIndices[j]] = id this.cacheVerbInt(missInts[j], id) } } } return resolved.filter((id): id is string => id !== undefined) } /** * UUID-level neighbor lookup over the BigInt provider contract: resolves * the anchor via `getInt` (unmapped → empty), then maps returned entity * ints back to UUIDs. Shared by the traversal paths and the * TripleIntelligenceSystem adapter. */ private async getNeighborUuids( uuid: string, options?: { direction?: 'in' | 'out' | 'both'; limit?: number; offset?: number } ): Promise { const entityInt = this.graphEntityInt(uuid) if (entityInt === undefined) return [] const neighborInts = await this.graphIndex.getNeighbors(entityInt, options) return this.entityIntsToUuids(neighborInts) } /** * @description Hydrate the entity id-mapper from persisted state BEFORE a graph * rebuild. A native int-keyed adjacency resolves every verb endpoint * (`sourceId`/`targetId` → stable int) through this mapper, so it must reflect * the persisted int assignments before `graphIndex.rebuild()` runs — otherwise * edges resolve to stale/missing ints and are silently dropped * (CTX-BR-RESTORE-REBUILD). The JS mapper has no `rebuild()` and needs no * reload (it is re-derived by `MetadataIndex.rebuild()` via append-only * `getOrAssign`), so this is a no-op for it. Mirrors the ordering in * {@link restore}. */ private async hydrateIdMapperForGraphRebuild(): Promise { const idMapper = this.metadataIndex.getIdMapper() as { rebuild?: () => Promise } if (typeof idMapper.rebuild === 'function') { await idMapper.rebuild() } } /** * @description Verify that the graph adjacency is actually LIVE before a graph read trusts * its result. A native graph index can load its relationship COUNT (manifest) on a cold open * of a LARGE brain (≥10k nouns, which skips the eager index rebuild) but NOT its * source→target adjacency, so `getNeighbors()` returns `[]` for EVERY source even though * edges are persisted — and `find({ connected })` / `neighbors()` / `related()` would serve * that `[]` as if it were truth. * * Two detection strategies, in order of honesty: * - **Preferred (8.0 contract):** the provider exposes a sync `isReady()` that is true ONLY * when the edges are loaded. `false` → hydrate the id-mapper (a native int adjacency * resolves endpoints through it), rebuild from storage, and re-check `isReady()`; if it is * still `false`, throw {@link GraphIndexNotReadyError} rather than returning `[]`. * - **Fallback (providers without `isReady()`):** a GLOBAL known-edge sample (a real * persisted verb's `sourceId`, which by definition HAS an outgoing edge) — NOT any queried * anchor, because brainy cannot cheaply tell "adjacency unloaded" from "this node is * genuinely edgeless" per-anchor. If that known-edge source resolves to no neighbors, the * adjacency did not load: rebuild and re-probe; if even that fails, throw. * * @returns `'live'` when the adjacency is already trustworthy (or there is genuinely nothing * to verify), or `'rebuilt'` when a cold-unloaded adjacency was just healed from storage — * in which case callers that observed an empty result must RE-RUN their collection. * @throws {GraphIndexNotReadyError} when the index claims edges but cannot serve a known * persisted edge (or stays not-ready) even after a rebuild. */ private async verifyGraphAdjacencyLive(): Promise<'live' | 'rebuilt'> { if (this._graphAdjacencyVerified) return 'live' // Coordinated migration LOCK (#18): while the graph provider owns a locked // rebuild-from-canonical, brainy must NOT fire its own graphIndex.rebuild() // on a read — that would race the provider's in-place rebuild. The data-plane // lock (awaitMigrationLock in ensureInitialized) already makes callers wait, // so this is normally unreachable mid-migration; the guard is defensive. It // deliberately does NOT set `_graphAdjacencyVerified`, so the real verify runs // once the migration clears. if (this.providerIsMigrating(this.graphIndex)) return 'live' // Re-entrancy: rebuild() can trigger reads (neighbors/related) that call back into this // guard. While a verify is in flight, short-circuit so we cannot recurse into rebuild(). if (this._graphAdjacencyVerifying) return 'live' this._graphAdjacencyVerifying = true try { const gi = this.graphIndex as GraphAdjacencyIndex & { isReady?: () => boolean } // ── Strategy 1: honest isReady() signal (cortex >= 2.7.8 / 3.0) ────────── if (typeof gi.isReady === 'function') { if (gi.isReady()) { this._graphAdjacencyVerified = true return 'live' } // Not ready: the edges did not load on open. Hydrate the id-mapper, then rebuild. if (!this.config.silent) { console.warn( `[Brainy] Graph adjacency reports not-ready (isReady() === false) — the persisted ` + `adjacency did not load on open. Rebuilding from storage…` ) } await this.hydrateIdMapperForGraphRebuild() await this.graphIndex.rebuild() if (gi.isReady()) { this._graphAdjacencyVerified = true return 'rebuilt' } throw new GraphIndexNotReadyError( `Graph adjacency index reports not-ready even after a rebuild — the persisted ` + `adjacency could not be loaded. find({ connected }), neighbors() and related() ` + `cannot be served reliably for this brain.` ) } // ── Strategy 2: known-edge-sample probe (providers without isReady()) ──── const claimed = await this.graphIndex.size() if (!claimed || claimed <= 0) return 'live' // no edges claimed — nothing to verify const sample = await this.storage.getVerbs({ pagination: { limit: 1 } }) const verb = sample.items?.[0] if (!verb || !verb.sourceId) { // no edges in storage — stale count, harmless; don't re-probe on every read. this._graphAdjacencyVerified = true return 'live' } // Resolve the known edge's source through THIS brain's id-mapper. An unmapped source means // the sample is not one of this brain's own edges — e.g. a shared on-disk store reused // across instances surfaces a foreign verb whose UUID this brain's resident mapper never // interned. We cannot prove a cold-unloaded adjacency from such a sample, so treat it as // INCONCLUSIVE: mark verified and return 'live' rather than rebuilding/throwing. (The honest // cold-load signal for native providers is isReady(), checked above; the JS baseline keeps // its mapper resident, so its OWN edges always resolve — the targeted 7.x failure mode, // "mapper loaded but adjacency empty", still resolves the source and is detected below.) const sourceInt = this.graphEntityInt(verb.sourceId) if (sourceInt === undefined) { this._graphAdjacencyVerified = true return 'live' } // Ask the adjacency for ONE neighbor of the (mapped) known-edge source. const probeKnownSource = async (): Promise => (await this.graphIndex.getNeighbors(sourceInt, { limit: 1 })).length > 0 if (await probeKnownSource()) { this._graphAdjacencyVerified = true return 'live' // adjacency is live — the common case } // INCONSISTENT: the index reports edges but a KNOWN-mapped persisted edge's source has none → // the adjacency did not load on open. Hydrate the mapper and rebuild from storage. if (!this.config.silent) { console.warn( `[Brainy] Graph adjacency reports ${claimed} relationship(s) but a persisted edge ` + `resolves to none — the persisted adjacency did not load on open. Rebuilding from storage…` ) } await this.hydrateIdMapperForGraphRebuild() await this.graphIndex.rebuild() if (await probeKnownSource()) { this._graphAdjacencyVerified = true return 'rebuilt' } throw new GraphIndexNotReadyError( `Graph adjacency index reports ${claimed} relationship(s) but returns no edges even ` + `after a rebuild — the persisted adjacency could not be loaded. find({ connected }), ` + `neighbors() and related() cannot be served reliably for this brain.` ) } catch (err) { if (err instanceof GraphIndexNotReadyError) throw err // A transient probe/rebuild failure must not break the actual query NOR be // masked as "no data". Allow a re-check on the next graph read and fall through. this._graphAdjacencyVerified = false if (!this.config.silent) { console.warn(`[Brainy] Graph adjacency consistency check skipped (transient): ${err}`) } return 'live' } finally { this._graphAdjacencyVerifying = false } } /** * @description The metadata field-index counterpart of {@link verifyGraphAdjacencyLive}. * On a cold open a native metadata provider can report data yet not serve its * `where` postings, so `find({ where })` silently returns `[]` — the exact * failure a downstream deployment reported (cold reads blanking filtered pages * after every restart). This one-shot guard, run on the first FILTERED `find()`, * closes that: it takes a KNOWN persisted entity + one of its plain field values * and asks the index to resolve it. If the index returns the known id the field * postings are live (the common case, and the ONLY cost on a warm brain — one * O(1) probe). If it does not, the postings did not load: brainy rebuilds the * index from the canonical records and re-probes; if it STILL cannot serve the * known value it throws a loud {@link MetadataIndexNotReadyError} rather than * let a silent `[]` stand. Inconclusive cases (empty store, no plain field to * probe, a shared store surfacing a foreign entity) are treated as live — never * a false rebuild. A migrating provider is skipped (it owns its locked rebuild). * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. */ private async verifyMetadataLive(): Promise<'live' | 'rebuilt'> { if (this._metadataVerified) return 'live' // Migration LOCK (#18): a migrating provider owns its in-place rebuild — do // not race it. Defensive; the data-plane lock already gates callers upstream. if (this.providerIsMigrating(this.metadataIndex)) return 'live' // Re-entrancy: rebuild() can trigger reads that call back into this guard. if (this._metadataVerifying) return 'live' this._metadataVerifying = true try { // A KNOWN persisted entity + one plain field to probe. Sample a few so a // system-only entity (e.g. the VFS root) doesn't make every open inconclusive. const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) let probe: { field: string; value: string | number | boolean; id: string } | null = null for (const noun of sample.items ?? []) { probe = this.pickMetadataProbe(noun as { id?: string; metadata?: Record }) if (probe) break } if (!probe) { // Empty store, or nothing with a plain user field to probe — inconclusive. this._metadataVerified = true return 'live' } const p = probe const probeServes = async (): Promise => { try { const ids = await this.metadataIndex.getIdsForFilter({ [p.field]: p.value }) return ids.includes(p.id) } catch { // FIELD_NOT_INDEXED for a field a persisted entity actually holds is // itself the cold/broken signal — treat as not-serving (→ rebuild). return false } } if (await probeServes()) { this._metadataVerified = true return 'live' // field postings are live — the common case } if (!this.config.silent) { console.warn( `[Brainy] Metadata field index returns no match for a known persisted value of ` + `'${p.field}' — the field postings did not load on open. Rebuilding from storage…` ) } await this.metadataIndex.rebuild() if (await probeServes()) { this._metadataVerified = true return 'rebuilt' } throw new MetadataIndexNotReadyError( `Metadata field index cannot serve a known persisted value of '${p.field}' even after ` + `a rebuild — find({ where }) and other filtered reads cannot be served reliably for ` + `this brain (a silent empty result would misrepresent existing data).` ) } catch (err) { if (err instanceof MetadataIndexNotReadyError) throw err // A transient probe/rebuild failure must not break the query NOR mask as // "no data". Allow a re-check on the next filtered read and fall through. this._metadataVerified = false if (!this.config.silent) { console.warn(`[Brainy] Metadata consistency check skipped (transient): ${err}`) } return 'live' } finally { this._metadataVerifying = false } } /** * @description Choose one plain (scalar, user-written) field from an entity's * metadata to probe the field index with — skipping internal / system fields * (`__words__` text hash, VFS markers, reserved `visibility`/`subtype`/`service`, * the `noun` type alias, timestamps) that use different index paths, and any * non-scalar value. Returns `null` when the entity has no probeable field. */ private pickMetadataProbe( noun: { id?: string; metadata?: Record } ): { field: string; value: string | number | boolean; id: string } | null { const id = noun?.id const metadata = noun?.metadata if (!id || !metadata || typeof metadata !== 'object') return null const skip = new Set([ 'noun', 'type', 'subtype', 'service', 'visibility', 'id', 'vector', 'isVFSEntity', 'vfsType', 'vfsPath', 'vfsName', 'createdAt', 'updatedAt' ]) for (const [field, value] of Object.entries(metadata)) { if (field.startsWith('_') || skip.has(field)) continue if (value === null || value === undefined) continue const t = typeof value if (t === 'string' || t === 'number' || t === 'boolean') { return { field, value: value as string | number | boolean, id } } } return null } /** * @description The vector-index counterpart of {@link verifyGraphAdjacencyLive} * / {@link verifyMetadataLive}. On a cold open a native vector provider can * report a non-zero `size()` (its persisted COUNT loaded) yet not have loaded * its serving structure (the mmap/DiskANN graph) — so a pure semantic * `find({ query })` silently returns `[]`. A pure semantic query has * `hasFilterCriteria === false`, so the metadata guard never fires; this guard * closes that gap. Run one-shot on the first vector/proximity search: * - **Preferred (honest signal):** the provider exposes `isReady()`. `false` * → rebuild from storage, re-check; if still `false`, throw * {@link VectorIndexNotReadyError} rather than serving `[]`. * - **Fallback (no `isReady()`):** a KNOWN persisted vector (sampled + * hydrated) is searched against the index; if it does not self-match, the * serving structure did not load — rebuild + re-probe, else throw. * Inconclusive cases (empty store, no probeable vector, `size()===0` — where * the JS baseline's cold load is `ensureIndexesLoaded`'s job) are treated as * live: never a false rebuild. A migrating provider is skipped (it owns its * locked rebuild). * @returns `'live'` when the index serves, `'rebuilt'` when a rebuild restored it. */ private async verifyVectorLive(): Promise<'live' | 'rebuilt'> { if (this._vectorVerified) return 'live' // Migration LOCK (#18): a migrating provider owns its in-place rebuild. if (this.providerIsMigrating(this.index)) return 'live' // Re-entrancy: rebuild() can trigger reads that call back into this guard. if (this._vectorVerifying) return 'live' this._vectorVerifying = true try { // ── Strategy 1: honest isReady() signal (native provider) ────────────── const readiness = assessIndexReadiness(this.index) if (readiness !== 'unknown') { if (readiness === 'ready') { this._vectorVerified = true return 'live' } // Not ready: the serving structure did not load on open. Rebuild. if (!this.config.silent) { console.warn( `[Brainy] Vector index reports not-ready (isReady() === false) — the persisted ` + `vector index did not load on open. Rebuilding from storage…` ) } await this.index.rebuild() if (assessIndexReadiness(this.index) === 'ready') { this._vectorVerified = true return 'rebuilt' } throw new VectorIndexNotReadyError( `Vector index reports not-ready even after a rebuild — semantic find({ query }) and ` + `proximity search cannot be served reliably for this brain (a silent empty result ` + `would misrepresent existing data).` ) } // ── Strategy 2: known-vector probe (providers without isReady()) ─────── const claimed = this.index.size() if (!claimed || claimed <= 0) return 'live' // JS cold path is ensureIndexesLoaded's job const probe = await this.pickVectorProbe() if (!probe) { // Empty store, or nothing with a probeable vector — inconclusive. this._vectorVerified = true return 'live' } const p = probe const probeServes = async (): Promise => { // The failure mode we guard is the SILENT EMPTY result: a cold index that // loaded its COUNT but not its serving structure returns `[]` for a // known-present vector, while a warm index returns at least one hit. We // check for a NON-EMPTY result, NOT an exact self-match — HNSW is // approximate and `get()` may return a re-hydrated/normalized vector, so // demanding the exact self as top-1 would false-positive on a perfectly // healthy index (and wrongly rebuild → throw). const hits = await this.index.search(p.vector, 1) return hits.length > 0 } void p.id // probe keyed on the vector; id retained for diagnostics only if (await probeServes()) { this._vectorVerified = true return 'live' // serving structure is live — the common case } if (!this.config.silent) { console.warn( `[Brainy] Vector index reports ${claimed} vector(s) but a known persisted vector ` + `returns no results — the serving structure did not load on open. Rebuilding…` ) } await this.index.rebuild() if (await probeServes()) { this._vectorVerified = true return 'rebuilt' } throw new VectorIndexNotReadyError( `Vector index reports ${claimed} vector(s) but a known persisted vector returns no ` + `results even after a rebuild — semantic find({ query }) cannot be served reliably ` + `for this brain (a silent empty result would misrepresent existing data).` ) } catch (err) { if (err instanceof VectorIndexNotReadyError) throw err // A transient probe/rebuild failure must not break the query NOR mask as // "no data". Allow a re-check on the next vector read and fall through. this._vectorVerified = false if (!this.config.silent) { console.warn(`[Brainy] Vector consistency check skipped (transient): ${err}`) } return 'live' } finally { this._vectorVerifying = false } } /** * @description Sample a KNOWN persisted noun and hydrate its vector, to probe * the vector index with. `get()` omits vectors by default, so this passes * `{ includeVectors: true }`. Samples a few (a system-only / vectorless entity * must not make every open inconclusive). Returns `null` when nothing has a * probeable vector. */ private async pickVectorProbe(): Promise<{ id: string; vector: number[] } | null> { const sample = await this.storage.getNouns({ pagination: { limit: 5, offset: 0 } }) for (const noun of sample.items ?? []) { const id = (noun as { id?: string }).id if (!id) continue const full = await this.get(id, { includeVectors: true }) const vector = (full as { vector?: number[] } | null)?.vector if (Array.isArray(vector) && vector.length > 0) { return { id, vector } } } return null } // ------------------------------------------------------------------------- /** * Create a relationship (verb) between two entities * * Relationships connect entities with typed edges. Duplicate relationships * (same from, to, and type) are detected and return the existing ID. * * **Data vs Metadata (on relationships):** * - `data`: Opaque content stored on the relationship (e.g., a description or * context for the edge). Overrides the auto-computed vector for this verb. * - `metadata`: Structured queryable fields on the edge (e.g., role, startDate). * * @param params - Parameters for creating the relationship * @param params.from - Source entity ID (required) * @param params.to - Target entity ID (required) * @param params.type - VerbType classification (required) * @param params.weight - Connection strength 0-1 (default: 1.0) * @param params.data - Content for the relationship (optional, overrides auto-computed vector) * @param params.metadata - Structured queryable fields on the edge * @param params.bidirectional - Create reverse edge too (default: false) * @param params.service - Multi-tenancy service name * @param params.confidence - Relationship certainty 0-1 * @param params.evidence - Why this relationship exists * @returns Promise that resolves to the relationship ID. **Contract (8.0):** * relationship ids are always UUIDs (36-char 8-4-4-4-12 hex). Brainy * generates every verb id itself (`relate()` takes no custom id), so the * UUID shape is a guarantee, not a coincidence — graph-index providers may * key their verb-int interning on the raw UUID bytes for lossless * reverse lookup. * * @example * // Basic relationship creation * const userId = await brainy.add({ * data: { name: 'John', role: 'developer' }, * type: NounType.Person * }) * const projectId = await brainy.add({ * data: { name: 'AI Assistant', status: 'active' }, * type: NounType.Thing * }) * * const relationId = await brainy.relate({ * from: userId, * to: projectId, * type: VerbType.WorksOn * }) * * @example * // Bidirectional relationships * const friendshipId = await brainy.relate({ * from: 'user-1', * to: 'user-2', * type: VerbType.Knows, * bidirectional: true // Creates both directions automatically * }) * * @example * // Weighted relationships for importance/strength * const collaborationId = await brainy.relate({ * from: 'team-lead', * to: 'project-alpha', * type: VerbType.LeadsOn, * weight: 0.9, // High importance/strength * metadata: { * startDate: '2024-01-15', * responsibility: 'technical leadership', * hoursPerWeek: 40 * } * }) * * @example * // Typed relationships with custom metadata * interface CollaborationMeta { * role: string * startDate: string * skillLevel: number * } * * const brainy = new Brainy({ storage: 'filesystem' }) * const relationId = await brainy.relate({ * from: 'developer-123', * to: 'project-456', * type: VerbType.WorksOn, * weight: 0.85, * metadata: { * role: 'frontend developer', * startDate: '2024-03-01', * skillLevel: 8 * } * }) * * @example * // Creating complex relationship networks * const entities = [] * // Create entities * for (let i = 0; i < 5; i++) { * const id = await brainy.add({ * data: { name: `Entity ${i}`, value: i * 10 }, * type: NounType.Thing * }) * entities.push(id) * } * * // Create hierarchical relationships * for (let i = 0; i < entities.length - 1; i++) { * await brainy.relate({ * from: entities[i], * to: entities[i + 1], * type: VerbType.DependsOn, * weight: (i + 1) / entities.length * }) * } * * @example * // Error handling for invalid relationships * try { * await brainy.relate({ * from: 'nonexistent-entity', * to: 'another-entity', * type: VerbType.RelatedTo * }) * } catch (error) { * if (error instanceof EntityNotFoundError) { * console.log(`Entity does not exist: ${error.id}`) * // Handle missing entities... * } * } */ async relate(params: RelateParams): Promise { this.assertWritable('relate') await this.ensureInitialized() // Zero-config validation (static import for performance) validateRelateParams(params) // Id normalization (8.0): resolve BOTH endpoints so a caller may relate by // natural key on either side. Each maps to the same canonical UUID add() // stored; real UUIDs pass through. (The relationship's own id is an // engine-minted UUID — relation ids are never caller-supplied here.) params = { ...params, from: resolveEntityId(params.from), to: resolveEntityId(params.to) } // Reserved fields arriving via the metadata bag are normalized to their // canonical top-level params before enforcement — mirror of add()'s lift. params = this.remapReservedRelateMetadata(params) // Subtype pairing enforcement (Layer 3 — 7.30.0). Per-type rules registered // via brain.requireSubtype() compose with the brain-wide strict-mode flag. // Metadata is passed so infrastructure edges (VFS containment) can bypass // the missing-subtype check via the `isVFSEntity` / `isVFS` marker. this.enforceSubtypeOnRelate('relate', params.type, params.subtype, params.metadata) // Verify entities exist const fromEntity = await this.get(params.from) const toEntity = await this.get(params.to) if (!fromEntity) { throw new EntityNotFoundError(params.from, `Source entity ${params.from} not found`) } if (!toEntity) { throw new EntityNotFoundError(params.to, `Target entity ${params.to} not found`) } // CRITICAL FIX: Check for duplicate relationships // This prevents infinite loops where same relationship is created repeatedly // Bug #1 showed incrementing verb counts (7→8→9...) indicating duplicates // OPTIMIZATION: Use GraphAdjacencyIndex for O(log n) lookup instead of O(n) storage scan // 8.0 BigInt boundary: unmapped source → no existing relations to check. const dupSourceInt = this.graphEntityInt(params.from) const verbIds = dupSourceInt === undefined ? [] : await this.resolveVerbIntsToIds(await this.graphIndex.getVerbIdsBySource(dupSourceInt)) // Batch-load verbs for 5x faster duplicate checking on GCS // GCS: 5 verbs = 1×50ms vs 5×50ms = 250ms (5x faster) if (verbIds.length > 0) { const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds) for (const [verbId, verb] of verbsMap.entries()) { if (verb.targetId === params.to && verb.verb === params.type) { // Relationship already exists - return existing ID instead of creating duplicate return verb.id } } } // No duplicate found - proceed with creation // Generate ID const id = uuidv4() // Compute relationship vector (average of entities) const relationVector = fromEntity.vector.map( (v, i) => (v + toEntity.vector[i]) / 2 ) // Prepare verb metadata // User metadata spread FIRST, then system fields ALWAYS win (prevents collision) // One timestamp for both createdAt and updatedAt so a never-updated edge reports a // stable updatedAt (=== createdAt) instead of a fresh Date.now() fabricated per read. const relateTs = Date.now() const verbMetadata = { ...(params.metadata || {}), verb: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), // visibility: stored only when not 'public' (absent === public, keeps records lean) ...(params.visibility !== undefined && params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.service !== undefined && { service: params.service }), createdAt: relateTs, updatedAt: relateTs, ...(params.data !== undefined && { data: params.data }) } // Save to storage (vector and metadata separately) const verb: GraphVerb = { id, vector: relationVector, sourceId: params.from, targetId: params.to, verb: params.type, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.service !== undefined && { service: params.service }), metadata: params.metadata, data: params.data, createdAt: Date.now() } // 8.0 BigInt boundary: resolve endpoint ints ONCE (getOrAssign — both // entities were existence-checked above) and mirror them onto the verb. const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) // Reverse-edge id reserved up front (when bidirectional) so the Model-B // generation's touched-verb set covers both edges this write creates. const reverseId = params.bidirectional ? uuidv4() : undefined // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image of each new edge = absent). await this.persistSingleOp({ verbs: reverseId ? [id, reverseId] : [id] }, async (tx) => { // Operation 1: Save verb vector data tx.addOperation( new SaveVerbOperation(this.storage, { id, vector: relationVector, connections: new Map(), verb: params.type, sourceId: params.from, targetId: params.to }) ) // Operation 2: Save verb metadata tx.addOperation( new SaveVerbMetadataOperation(this.storage, id, verbMetadata) ) // Operation 3: Add to graph index for O(1) lookups tx.addOperation( new AddToGraphIndexOperation( this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, id) ) ) // Create bidirectional if requested if (params.bidirectional && reverseId) { const reverseVerb: GraphVerb = { ...verb, id: reverseId, sourceId: params.to, targetId: params.from, // Endpoints swap, so the derived ints swap with them. sourceInt: targetInt, targetInt: sourceInt } // Operation 4: Save reverse verb vector data tx.addOperation( new SaveVerbOperation(this.storage, { id: reverseId, vector: relationVector, connections: new Map(), verb: params.type, sourceId: params.to, targetId: params.from }) ) // Operation 5: Save reverse verb metadata tx.addOperation( new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata) ) // Operation 6: Add reverse relationship to graph index tx.addOperation( new AddToGraphIndexOperation( this.graphIndex, reverseVerb, { sourceInt: targetInt, targetInt: sourceInt }, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, reverseId) ) ) } }, undefined, this._changeFeed.hasListeners ? [ { kind: 'relation', op: 'relate', id, relation: { id, from: params.from, to: params.to, type: String(params.type), ...(params.metadata && { metadata: params.metadata as Record }) } }, ...(reverseId ? [ { kind: 'relation', op: 'relate', id: reverseId, relation: { id: reverseId, from: params.to, to: params.from, type: String(params.type), ...(params.metadata && { metadata: params.metadata as Record }) } } as PendingChangeEvent ] : []) ] : undefined) return id } /** * Delete a relationship (verb) by its ID * * Removes the relationship from the GraphAdjacencyIndex and deletes * the verb metadata from storage. Executed atomically. * * @param id - UUID of the relationship to delete * * @example * ```typescript * const relId = await brain.relate({ * from: personId, to: projectId, type: VerbType.WorksOn * }) * await brain.unrelate(relId) // Relationship removed * ``` */ async unrelate(id: string): Promise { this.assertWritable('unrelate') await this.ensureInitialized() // Get verb data before deletion for rollback const verb = await this.storage.getVerb(id) // 8.0 BigInt boundary: resolve endpoint ints before the transaction so // a rollback can re-add through the BigInt addVerb contract. const endpointInts = verb ? this.resolveVerbEndpointInts(verb) : undefined // Execute atomically with transaction system, generation-stamped as one // immutable Model-B generation (before-image = the relationship's state). await this.persistSingleOp({ verbs: [id] }, async (tx) => { // Operation 1: Remove from graph index if (verb && endpointInts) { tx.addOperation( new RemoveFromGraphIndexOperation( this.graphIndex, verb, endpointInts, this.graphWriteGeneration ) ) } // Operation 2: Delete verb metadata (which also deletes vector) tx.addOperation( new DeleteVerbMetadataOperation(this.storage, id) ) }, undefined, this._changeFeed.hasListeners && verb ? [ { kind: 'relation', op: 'unrelate', id, relation: { id, from: verb.sourceId, to: verb.targetId, type: String(verb.verb), ...(verb.metadata && { metadata: verb.metadata as Record }) } } ] : undefined) } /** * Update an existing relationship. * * Mirror of `update()` for relationships. Supports changing the verb type, the * sub-classification (`subtype`), weight/confidence, the opaque `data` payload, and * structured metadata (merge by default; set `merge: false` to replace). * * If `type` changes, the relationship is re-indexed in the graph adjacency * (`RemoveFromGraphIndex` + `AddToGraphIndex`) so traversal by verb type stays * consistent. The relationship ID is preserved across the type change. * * @param params - Update parameters (`id` required; at least one field to change) * @throws RelationNotFoundError if the relationship doesn't exist * * @example Change subtype * await brain.updateRelation({ id: relId, subtype: 'dotted-line' }) * * @example Merge metadata * await brain.updateRelation({ id: relId, metadata: { startDate: '2026-Q3' } }) * * @example Replace metadata entirely * await brain.updateRelation({ id: relId, metadata: { only: 'this' }, merge: false }) */ async updateRelation(params: UpdateRelationParams): Promise { this.assertWritable('updateRelation') await this.ensureInitialized() validateUpdateRelationParams(params) // Reserved fields arriving via the metadata patch are remapped to their // canonical top-level params — mirror of update()'s normalization. params = this.remapReservedUpdateRelationMetadata(params) const existing = await this.storage.getVerb(params.id) if (!existing) { throw new RelationNotFoundError(params.id) } // Legacy stored shapes carried the verb type under `type` instead of the // canonical `verb` field — read both, canonical first. const existingRec: HNSWVerbWithMetadata & { type?: VerbType } = existing const newVerbType = params.type ?? existingRec.verb ?? existingRec.type // Subtype pairing enforcement on update (7.30.0). The effective verb type after // the update may have changed; we check against the new type and the resulting // subtype value (explicit param or preserved existing). if (params.subtype !== undefined || params.type !== undefined) { const effectiveSubtype = params.subtype !== undefined ? params.subtype : existingRec.subtype const effectiveMetadata = params.metadata ?? existingRec.metadata this.enforceSubtypeOnRelate('updateRelation', newVerbType, effectiveSubtype, effectiveMetadata) } const typeChanged = params.type !== undefined && params.type !== (existingRec.verb ?? existingRec.type) // Merge metadata (mirror of update() for nouns) const newMetadata = params.merge !== false ? { ...(existingRec.metadata || {}), ...(params.metadata || {}) } : params.metadata || existingRec.metadata // Build updated stored metadata. System fields ALWAYS win — same shape as relate(). const updatedMetadata = { ...newMetadata, verb: newVerbType, ...(params.subtype !== undefined ? { subtype: params.subtype } : existingRec.subtype !== undefined && { subtype: existingRec.subtype }), // Visibility: new value if provided, else preserve existing; stored only when the // effective value is not 'public' (a change to 'public' drops the field). ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { visibility: params.visibility ?? existingRec.visibility }), weight: params.weight ?? existingRec.weight ?? 1.0, ...(params.confidence !== undefined ? { confidence: params.confidence } : existingRec.confidence !== undefined && { confidence: existingRec.confidence }), // service/createdBy are fixed at relate() time — always carried forward // (omitting them here silently erased them on every updateRelation()). ...(existingRec.service !== undefined && { service: existingRec.service }), ...(existingRec.createdBy !== undefined && { createdBy: existingRec.createdBy }), createdAt: existingRec.createdAt, updatedAt: Date.now(), ...(params.data !== undefined ? { data: params.data } : existingRec.data !== undefined && { data: existingRec.data }) } // Build the verb view used by the graph index — top-level fields mirror relate()'s. const verbForIndex: GraphVerb = { id: params.id, vector: existingRec.vector, sourceId: existingRec.sourceId, targetId: existingRec.targetId, verb: newVerbType, type: newVerbType, ...(params.subtype !== undefined ? { subtype: params.subtype } : existingRec.subtype !== undefined && { subtype: existingRec.subtype }), ...(((params.visibility ?? existingRec.visibility) ?? 'public') !== 'public' && { visibility: params.visibility ?? existingRec.visibility }), weight: updatedMetadata.weight, metadata: newMetadata, data: updatedMetadata.data, createdAt: existingRec.createdAt } // 8.0 BigInt boundary: endpoints are unchanged across a type swap, so one // resolution serves both the remove (rollback re-add) and the re-add. const reindexInts = typeChanged ? this.resolveVerbEndpointInts(verbForIndex) : undefined await this.persistSingleOp({ verbs: [params.id] }, async (tx) => { tx.addOperation( new UpdateVerbMetadataOperation(this.storage, params.id, updatedMetadata) ) // If the verb type changed, re-index in graph adjacency so traversal-by-type // stays consistent. The id is preserved across the swap. if (typeChanged && reindexInts) { tx.addOperation( new RemoveFromGraphIndexOperation( this.graphIndex, existing, reindexInts, this.graphWriteGeneration ) ) tx.addOperation( new AddToGraphIndexOperation( this.graphIndex, verbForIndex, reindexInts, this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, params.id) ) ) } }, undefined, this._changeFeed.hasListeners ? [ { kind: 'relation', op: 'updateRelation', id: params.id, relation: { id: params.id, from: verbForIndex.sourceId, to: verbForIndex.targetId, type: String(verbForIndex.verb ?? verbForIndex.type), ...(verbForIndex.metadata && { metadata: verbForIndex.metadata as Record }) } } ] : undefined) } /** * Get relationships between entities * * Supports multiple query patterns: * - No parameters: Returns all relationships (paginated, default limit: 100) * - String ID: Returns relationships from that entity (shorthand for { from: id }) * - Parameters object: Fine-grained filtering and pagination * * @param paramsOrId - Optional string ID or parameters object * @returns Promise resolving to array of relationships * * Named `related` to match the `Db.related()` surface — the same call works * on a live brain and on a pinned `Db` view. * * @example * ```typescript * // Get all relationships (first 100) * const all = await brain.related() * * // Get relationships from specific entity (shorthand syntax) * const fromEntity = await brain.related(entityId) * * // Get relationships with filters * const filtered = await brain.related({ * type: VerbType.FriendOf, * limit: 50 * }) * * // Pagination * const page2 = await brain.related({ offset: 100, limit: 100 }) * ``` * */ async related( paramsOrId?: string | RelatedParams ): Promise[]> { // Graph read: relationship traversal consults only the graph adjacency // family, so it waits on a graph migration but not on vector/metadata. await this.ensureInitialized({ needs: ['graph'] }) await this.verifyGraphAdjacencyLive() // Handle string ID shorthand: related(id) -> related({ from: id }) const rawParams = typeof paramsOrId === 'string' ? { from: paramsOrId } : (paramsOrId || {}) // Id normalization (8.0): resolve the anchor id(s) so a caller may traverse // by natural key. Each maps to the canonical UUID add() stored; real UUIDs // pass through. Mutates a copy so the caller's params object is untouched. const params: RelatedParams = { ...rawParams, ...(rawParams.from !== undefined && { from: resolveEntityId(rawParams.from) }), ...(rawParams.to !== undefined && { to: resolveEntityId(rawParams.to) }), ...(rawParams.node !== undefined && { node: resolveEntityId(rawParams.node) }) } const limit = params.limit || 100 const offset = params.offset || 0 // `node` is the both-direction shorthand; it cannot also constrain one direction. if (params.node !== undefined && (params.from !== undefined || params.to !== undefined)) { throw new Error( "related(): 'node' is mutually exclusive with 'from'/'to' — pass 'node' for both-direction incident edges, or 'from'/'to' for one direction." ) } // Production safety: warn for large unfiltered queries if (!params.from && !params.to && !params.node && !params.type && limit > 10000) { console.warn( `[Brainy] related(): Fetching ${limit} relationships without filters. ` + `Consider adding 'from', 'to', 'node', or 'type' filter for better performance.` ) } // Shared filter (type / subtype / service / visibility). Endpoint ids // (`sourceId` / `targetId`) are added per-direction below. const filter: any = {} if (params.type) { filter.verbType = Array.isArray(params.type) ? params.type : [params.type] } if (params.subtype !== undefined) { filter.subtype = Array.isArray(params.subtype) ? params.subtype : [params.subtype] } if (params.service) { filter.service = params.service } // Visibility (8.0): exclude hidden tiers by default. Applied on the O(degree) // candidate set in the graph-index fast path (see baseStorage.applyVerbMetadataFilters). const excludedTiers = this.excludedVisibilityTiers(params) if (excludedTiers) { filter.excludeVisibility = excludedTiers } // VFS relationships are no longer filtered // VFS is part of the knowledge graph - users can filter explicitly if needed // Both-direction incident-edge query: union the node's out-edges (it is the // source) and in-edges (it is the target) — each an O(degree) indexed lookup — // then dedupe a self-loop that appears in both. Sorted by id for a stable page, // sliced from the merged set. Fetching the node's full incidence is the intended // O(degree) cost; for a very-high-degree node, deep `offset` paging is best-effort // (use the native edgesForNode / a graph cursor for exhaustive paging). if (params.node !== undefined) { const endLimit = offset + limit const [outEdges, inEdges] = await Promise.all([ this.storage.getVerbs({ pagination: { limit: endLimit }, filter: { ...filter, sourceId: params.node } }), this.storage.getVerbs({ pagination: { limit: endLimit }, filter: { ...filter, targetId: params.node } }) ]) const byId = new Map() for (const verb of outEdges.items) byId.set(verb.id, verb) for (const verb of inEdges.items) byId.set(verb.id, verb) const merged = [...byId.values()].sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0 ) return this.verbsToRelations(merged.slice(offset, offset + limit)) } // Single-direction query (from / to). if (params.from) { filter.sourceId = params.from } if (params.to) { filter.targetId = params.to } // Fetch from storage with pagination at storage layer (efficient!) const result = await this.storage.getVerbs({ pagination: { limit, offset, cursor: params.cursor }, filter: Object.keys(filter).length > 0 ? filter : undefined }) // Convert to Relation format return this.verbsToRelations(result.items) } // ============= GRAPH VIEWS (brain.graph.*) ============= /** * @description The `brain.graph` namespace — graph-shaped reads. `subgraph` * extracts the multi-hop neighborhood around seed entities; the surface grows * with the engine (streaming `export`, `rank`, `path`, `communities`). Routes * to a registered native {@link GraphAccelerationProvider} when present, else * serves the same results from Brainy's pure-TS adjacency. Reads are live * ("now"); historical graph views come from `db.asOf(g).graph.*` (a later slice). * @returns The {@link GraphApi} bound to this brain. * @example * const view = await brain.graph.subgraph(personId, { depth: 2 }) * // view.nodes = the 2-hop neighborhood; view.edges = the relations among them */ get graph(): GraphApi { return { subgraph: (selector: SubgraphSelector, options?: SubgraphOptions) => this.graphSubgraph(selector, options), export: (options?: GraphExportOptions) => this.graphExport(options), rank: (options?: GraphRankOptions) => this.graphRank(options), communities: (options?: GraphCommunitiesOptions) => this.graphCommunities(options), path: (from: string, to: string, options?: GraphPathOptions) => this.graphPath(from, to, options) } } /** Resolve the optional native graph-acceleration provider (feature-detected). */ private graphAccelerationProvider(): GraphAccelerationProvider | undefined { // Resolve + cache the provider INSTANCE once. Accept EITHER a ready instance OR // a `(storage) => provider` factory (the convention the graphIndex/metadataIndex/ // vector providers use) — a registered factory would otherwise fail the // isGraphAccelerationProvider duck-test and the native path would silently never engage. if (this._graphAccelProvider === undefined) { const raw = this.pluginRegistry.getProvider('graphAcceleration') let resolved: unknown = raw if (typeof raw === 'function' && !isGraphAccelerationProvider(raw)) { try { resolved = (raw as (storage: StorageAdapter) => unknown)(this.storage) } catch { resolved = undefined } } this._graphAccelProvider = isGraphAccelerationProvider(resolved) ? resolved : null } const provider = this._graphAccelProvider ?? undefined // Readiness gate — checked LIVE each call, never cached: the native engine sets // `isInitialized` false until its engine + cursor have loaded (the cold-start / // rebuild window). Until then, route to the pure-TS path rather than calling a // not-ready provider (which throws) — and re-engage the native path automatically // the moment `isInitialized` flips true. Gates EVERY `brain.graph.*` route // (subgraph/export/rank/communities/path + the query→expand fusion) at once. return provider && provider.isInitialized ? provider : undefined } /** * @description {@link GraphApi.subgraph} dispatch: native engine when present, * else the TS BFS fallback. The selector is an id set, a `find()` result, or a * query (query→expand fusion). Explicit ids are id-normalized so a caller may * seed by natural key. */ private async graphSubgraph( selector: SubgraphSelector, options?: SubgraphOptions ): Promise> { await this.ensureInitialized() const depth = Math.max(0, options?.depth ?? 1) const direction = options?.direction ?? 'both' const accel = this.graphAccelerationProvider() // Query selector (a FindParams object — not a string / array) → query→expand. if (selector && typeof selector === 'object' && !Array.isArray(selector)) { return this.graphSubgraphFromQuery(accel, selector, depth, direction, options) } // Id set: a string, an array of ids, or a `find()` result (Result[]). The // FindParams (query) case returned above, so the remaining union is the id forms. const seedIds = this.selectorToSeedIds(selector as string | string[] | Result[]) if (seedIds.length === 0) return { nodes: [], edges: [], truncated: false } return accel ? this.graphSubgraphNative(accel, this.seedIdsToInts(seedIds), depth, direction, options) : this.graphSubgraphFallback(seedIds, depth, direction, options) } /** * @description Query→expand fusion (graph #61): run the query, then expand the * subgraph from every match. When the native engine is present AND the query is a * pure metadata filter (no vector/text/proximity criteria) AND the metadata * provider exposes the opaque-set producer, the matched universe is forwarded to * `traverse` as an {@link OpaqueIdSet} with NO id materialization in TS — the * O(1)-crossing path. Otherwise the query is materialized via `find()` and its * result ids seed the traversal (native or TS fallback). */ private async graphSubgraphFromQuery( accel: GraphAccelerationProvider | undefined, selector: FindParams, depth: number, direction: 'in' | 'out' | 'both', options?: SubgraphOptions ): Promise> { // Native opaque fast path: a metadata-only universe never leaves the engine. if (accel && !this.hasVectorOrTextCriteria(selector)) { const filter = this.buildMetadataFilter(selector) const mip = this.metadataIndex as unknown as MetadataIndexProvider if (filter && typeof mip.getIdSetForFilter === 'function') { const universe = await mip.getIdSetForFilter(filter) return this.graphSubgraphNative(accel, universe, depth, direction, options) } } // General path: materialize the matched ids, then expand from them. The find // default limit (10) is too small to seed a neighborhood, so seed from ALL // matches up to a bounded cap unless the caller pinned an explicit limit. const matches = await this.find({ ...selector, limit: selector.limit ?? Brainy.QUERY_SEED_CAP }) const seedIds = matches.map((m) => m.id) if (seedIds.length === 0) return { nodes: [], edges: [], truncated: false } return accel ? this.graphSubgraphNative(accel, this.seedIdsToInts(seedIds), depth, direction, options) : this.graphSubgraphFallback(seedIds, depth, direction, options) } /** True when a query needs vector/text/proximity search (not a pure metadata filter). */ private hasVectorOrTextCriteria(params: FindParams): boolean { return Boolean( (params.query && params.query.trim() !== '') || params.vector || params.near ) } /** Normalize an id-set selector (id / id[] / `find()` result) to canonical seed ids. */ private selectorToSeedIds(selector: string | string[] | Result[]): string[] { const arr = Array.isArray(selector) ? selector : [selector] return arr.map((s) => resolveEntityId(typeof s === 'string' ? s : s.id)) } /** Resolve seed ids to graph entity ints, dropping any that were never mapped. */ private seedIdsToInts(seedIds: string[]): bigint[] { const ints: bigint[] = [] for (const id of seedIds) { const int = this.graphEntityInt(id) if (int !== undefined) ints.push(int) } return ints } /** * @description Pure-TS subgraph BFS — expands each frontier through the * O(degree) `related()` adjacency (visibility / type / subtype filters applied * there), dedupes edges, tracks hop depth, and honors `maxNodes` / `maxEdges` * caps. Correct at small/medium scale; the native provider is the at-scale path. */ private async graphSubgraphFallback( seedIds: string[], depth: number, direction: 'in' | 'out' | 'both', options?: SubgraphOptions ): Promise> { const maxNodes = options?.maxNodes const maxEdges = options?.maxEdges const nodeDepth = new Map() for (const id of seedIds) nodeDepth.set(id, 0) const edgesById = new Map>() let truncated = false let frontier = [...seedIds] for (let d = 1; d <= depth && frontier.length > 0 && !truncated; d++) { const next: string[] = [] for (const nodeId of frontier) { const incident = await this.incidentEdges(nodeId, direction, options) for (const edge of incident) { if (!edgesById.has(edge.id)) { if (maxEdges !== undefined && edgesById.size >= maxEdges) { truncated = true break } edgesById.set(edge.id, edge) } const neighbor = edge.from === nodeId ? edge.to : edge.from if (!nodeDepth.has(neighbor)) { if (maxNodes !== undefined && nodeDepth.size >= maxNodes) { truncated = true continue } nodeDepth.set(neighbor, d) next.push(neighbor) } } if (truncated) break } frontier = next } return this.buildGraphView( nodeDepth, [...edgesById.values()], options?.hydrateNodes !== false, truncated ) } /** One frontier hop's incident edges in the requested direction (filters applied by `related()`). */ private incidentEdges( nodeId: string, direction: 'in' | 'out' | 'both', options?: SubgraphOptions ): Promise[]> { const shared: RelatedParams = { type: options?.type, subtype: options?.subtype, includeInternal: options?.includeInternal, includeSystem: options?.includeSystem, limit: options?.maxEdges ?? 100000 } if (direction === 'out') return this.related({ ...shared, from: nodeId }) if (direction === 'in') return this.related({ ...shared, to: nodeId }) return this.related({ ...shared, node: nodeId }) } /** * @description Native subgraph: resolve seeds to ints, call the provider's * `traverse`, then hydrate the columnar `Subgraph` (node ints → ids paired with * their depth, edge verb ints → `Relation`s) into a {@link GraphView}. The TS * fallback produces the same view, so Brainy CI exercises this shape via the * fallback; the native path itself is cross-layer-tested against the provider. */ private async graphSubgraphNative( accel: GraphAccelerationProvider, seeds: bigint[] | OpaqueIdSet, depth: number, direction: 'in' | 'out' | 'both', options?: SubgraphOptions ): Promise> { // Resolved int seeds may be empty (no mapped entities); an OpaqueIdSet (Buffer) // is passed straight through — the provider owns its membership. if (Array.isArray(seeds) && seeds.length === 0) return { nodes: [], edges: [], truncated: false } const verbTypes = options?.type ? (Array.isArray(options.type) ? options.type : [options.type]).map((t) => TypeUtils.getVerbIndex(t) ) : undefined const excluded = this.excludedVisibilityTiers(options ?? {}) const traverseOptions: TraverseOptions = { depth, direction, ...(verbTypes && { verbTypes }), ...(options?.subtype !== undefined && { subtypes: Array.isArray(options.subtype) ? options.subtype : [options.subtype] }), ...(excluded && { excludeVisibility: excluded as unknown as string[] }), ...(options?.maxNodes !== undefined && { maxNodes: options.maxNodes }), ...(options?.maxEdges !== undefined && { maxEdges: options.maxEdges }), includeEdges: true, includeDepth: true } const sub = await accel.traverse(seeds, traverseOptions) return this.hydrateNativeSubgraph(sub, options?.hydrateNodes !== false) } /** * @description Hydrate a columnar native `Subgraph` into a {@link GraphView}: * edge verb-ints → `Relation`s (via `verbIntsToIds` + `getVerbsBatchCached`), * and node ints paired with their depth (position-preserving — `entityIntsToUuids` * drops deleted ints, which would desync the depth column). Shared by the native * `subgraph` and `export` paths. * @param sub - The provider's columnar subgraph / cursor chunk. * @param hydrateNodes - Whether to batch-load each node's `type` / `subtype`. * @returns The hydrated {@link GraphView}. */ private async hydrateNativeSubgraph(sub: Subgraph, hydrateNodes: boolean): Promise> { const verbIds = (await this.graphIndex.verbIntsToIds(Array.from(sub.edgeVerbInts))).filter( (id): id is string => id !== null ) const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds) const edges = this.verbsToRelations([...verbsMap.values()]) const idMapper = this.metadataIndex.getIdMapper() const nodeDepth = new Map() for (let i = 0; i < sub.nodes.length; i++) { const uuid = idMapper.getUuid(Number(sub.nodes[i])) if (uuid !== undefined) nodeDepth.set(uuid, sub.nodeDepth ? sub.nodeDepth[i] : 0) } return this.buildGraphView(nodeDepth, edges, hydrateNodes, sub.truncated ?? false) } /** * @description Assemble a {@link GraphView} from discovered node depths + edges, * optionally hydrating each node's `type` / `subtype` in one batch read * (`hydrateNodes`, default `true`). */ private async buildGraphView( nodeDepth: Map, edges: Relation[], hydrateNodes: boolean, truncated: boolean ): Promise> { const ids = [...nodeDepth.keys()] let entities: Map> | undefined if (hydrateNodes && ids.length > 0) { entities = await this.batchGet(ids) } const nodes: GraphNode[] = ids.map((id) => { const entity = entities?.get(id) return { id, ...(entity && { type: entity.type, ...(entity.subtype !== undefined && { subtype: entity.subtype }) }), depth: nodeDepth.get(id) } }) return { nodes, edges, truncated } } /** * @description {@link GraphApi.export} dispatch: native snapshot-consistent * graph cursor when present, else the TS cursor walk over nouns + verbs. Both * are O(N+E) single passes (cursor pagination — no re-scan). */ private async *graphExport(options?: GraphExportOptions): AsyncGenerator> { await this.ensureInitialized() const accel = this.graphAccelerationProvider() if (accel) { yield* this.graphExportNative(accel, options) } else { yield* this.graphExportFallback(options) } } /** * @description TS export: stream all nouns (node chunks) then all verbs (edge * chunks) via cursor pagination — O(N+E), no re-scan. Node refs carry * `type`/`subtype` straight from the noun records (no extra read). Hidden tiers * are excluded by default. */ private async *graphExportFallback(options?: GraphExportOptions): AsyncGenerator> { const chunkSize = options?.chunkSize ?? 1000 const excludedTiers = this.excludedVisibilityTiers(options ?? {}) const excludedSet = excludedTiers ? new Set(excludedTiers) : null if (options?.includeNodes !== false) { let cursor: string | undefined let offset = 0 for (;;) { const page = await this.storage.getNouns({ pagination: cursor ? { limit: chunkSize, cursor } : { limit: chunkSize, offset } }) const nodes: GraphNode[] = [] for (const noun of page.items) { if (excludedSet && noun.visibility && excludedSet.has(noun.visibility)) continue nodes.push({ id: noun.id, type: noun.type, ...(noun.subtype !== undefined && { subtype: noun.subtype }) }) } if (nodes.length > 0) yield { nodes, edges: [], truncated: false } if (!page.hasMore || page.items.length === 0) break if (page.nextCursor) cursor = page.nextCursor else offset += page.items.length } } if (options?.includeEdges !== false) { const filter = excludedTiers ? { excludeVisibility: excludedTiers as unknown as string[] } : undefined let cursor: string | undefined let offset = 0 for (;;) { const page = await this.storage.getVerbs({ pagination: cursor ? { limit: chunkSize, cursor } : { limit: chunkSize, offset }, filter }) if (page.items.length > 0) { yield { nodes: [], edges: this.verbsToRelations(page.items), truncated: false } } if (!page.hasMore || page.items.length === 0) break if (page.nextCursor) cursor = page.nextCursor else offset += page.items.length } } } /** * @description Native export: open a snapshot-consistent graph cursor (pins a * generation — no dup/skip under concurrent writes), pull light chunks, and * hydrate each into a {@link GraphView}. The cursor is always closed (even on * early break / error). Cross-layer tested against the native provider; the TS * fallback exercises the same chunk shape in CI. */ private async *graphExportNative( accel: GraphAccelerationProvider, options?: GraphExportOptions ): AsyncGenerator> { const excludedTiers = this.excludedVisibilityTiers(options ?? {}) const handle = await accel.graphCursorOpen({ direction: 'both', ...(excludedTiers && { excludeVisibility: excludedTiers as unknown as string[] }), projection: 'light' }) try { for (;;) { const chunk = await accel.graphCursorNext(handle, options?.chunkSize ?? 1000) const view = await this.hydrateNativeSubgraph(chunk.subgraph, true) if (view.nodes.length > 0 || view.edges.length > 0) yield view if (chunk.done) break } } finally { await accel.graphCursorClose(handle) } } // ============= GRAPH ANALYTICS (rank / communities / path) ============= /** * @description Load the whole visible graph into a dense integer adjacency for * the rank / communities fallbacks: a node-id list, an id→index map, and * `outAdj[i]` = the dense indices of node `i`'s out-edge targets (multiplicity * preserved). Streamed via {@link graphExport} (cursor pagination — O(N+E), no * re-scan), so hidden tiers are excluded the same way every other read excludes * them. Isolated nodes (no edges) are retained so they form singleton groups. */ private async loadAnalyticsGraph(opts: { includeInternal?: boolean includeSystem?: boolean }): Promise<{ ids: string[]; outAdj: number[][] }> { const ids: string[] = [] const index = new Map() const outAdj: number[][] = [] const idxOf = (id: string): number => { let i = index.get(id) if (i === undefined) { i = ids.length ids.push(id) index.set(id, i) outAdj.push([]) } return i } for await (const chunk of this.graphExport({ includeInternal: opts.includeInternal, includeSystem: opts.includeSystem })) { for (const node of chunk.nodes) idxOf(node.id) for (const edge of chunk.edges) { const s = idxOf(edge.from) const t = idxOf(edge.to) outAdj[s].push(t) } } return { ids, outAdj } } /** * @description {@link GraphApi.rank} dispatch: native ranking when a provider is * present, else the TS PageRank fallback. */ private async graphRank(options?: GraphRankOptions): Promise { await this.ensureInitialized() const accel = this.graphAccelerationProvider() return accel ? this.graphRankNative(accel, options) : this.graphRankFallback(options) } /** TS PageRank over the dense visible adjacency; sorted descending, `topK`-capped. */ private async graphRankFallback(options?: GraphRankOptions): Promise { const { ids, outAdj } = await this.loadAnalyticsGraph(options ?? {}) if (ids.length === 0) return [] const scores = pageRank(outAdj) const entries: GraphRankEntry[] = ids.map((id, i) => ({ id, score: scores[i] })) entries.sort((a, b) => b.score - a.score) return options?.topK !== undefined ? entries.slice(0, options.topK) : entries } /** Native ranking → hydrate node-ints to ids (descending order preserved). */ private async graphRankNative( accel: GraphAccelerationProvider, options?: GraphRankOptions ): Promise { const excluded = this.excludedVisibilityTiers(options ?? {}) const opts: RankOptions = { ...(options?.topK !== undefined && { topK: options.topK }), ...(excluded && { excludeVisibility: excluded as unknown as string[] }) } const result = await accel.rank(opts) const idMapper = this.metadataIndex.getIdMapper() const entries: GraphRankEntry[] = [] for (let i = 0; i < result.nodeInts.length; i++) { const uuid = idMapper.getUuid(Number(result.nodeInts[i])) if (uuid !== undefined) entries.push({ id: uuid, score: result.scores[i] }) } return options?.topK !== undefined ? entries.slice(0, options.topK) : entries } /** * @description {@link GraphApi.communities} dispatch: native grouping when a * provider is present, else the TS connected-components fallback. */ private async graphCommunities( options?: GraphCommunitiesOptions ): Promise { await this.ensureInitialized() const accel = this.graphAccelerationProvider() return accel ? this.graphCommunitiesNative(accel, options) : this.graphCommunitiesFallback(options) } /** * @description TS grouping: weakly-connected components by default (union-find * over the undirected projection), or strongly-connected components when * `directed: true` (Tarjan). Largest group first. */ private async graphCommunitiesFallback( options?: GraphCommunitiesOptions ): Promise { const { ids, outAdj } = await this.loadAnalyticsGraph(options ?? {}) if (ids.length === 0) return { groups: [], count: 0 } const labels = options?.directed ? stronglyConnectedComponents(outAdj) : connectedComponents(outAdj) return this.groupByLabel(ids, labels) } /** Collapse parallel `(id, label)` arrays into id-groups, largest first. */ private groupByLabel(ids: string[], labels: number[]): GraphCommunitiesResult { const byLabel = new Map() for (let i = 0; i < ids.length; i++) { const label = labels[i] const existing = byLabel.get(label) if (existing) existing.push(ids[i]) else byLabel.set(label, [ids[i]]) } const groups = [...byLabel.values()].sort((a, b) => b.length - a.length) return { groups, count: groups.length } } /** Native grouping → bucket node-ints by community label, hydrate to ids. */ private async graphCommunitiesNative( accel: GraphAccelerationProvider, options?: GraphCommunitiesOptions ): Promise { const excluded = this.excludedVisibilityTiers(options ?? {}) const opts: CommunitiesOptions = { ...(options?.directed !== undefined && { directed: options.directed }), ...(excluded && { excludeVisibility: excluded as unknown as string[] }) } const result = await accel.communities(opts) const idMapper = this.metadataIndex.getIdMapper() const byCommunity = new Map() for (let i = 0; i < result.nodeInts.length; i++) { const uuid = idMapper.getUuid(Number(result.nodeInts[i])) if (uuid === undefined) continue const community = result.communityIds[i] const existing = byCommunity.get(community) if (existing) existing.push(uuid) else byCommunity.set(community, [uuid]) } const groups = [...byCommunity.values()].sort((a, b) => b.length - a.length) return { groups, count: groups.length } } /** * @description {@link GraphApi.path} dispatch: native pathfinding when a provider * is present, else the TS BFS (hops) / Dijkstra (weight) fallback. Endpoints are * id-normalized so callers may pass natural keys. */ private async graphPath( from: string, to: string, options?: GraphPathOptions ): Promise { await this.ensureInitialized() const fromId = resolveEntityId(from) const toId = resolveEntityId(to) const accel = this.graphAccelerationProvider() return accel ? this.graphPathNative(accel, fromId, toId, options) : this.graphPathFallback(fromId, toId, options) } /** * @description On-demand TS pathfinding — expands frontiers through the O(degree) * `related()` adjacency (visibility / type filters applied there) rather than * loading the whole graph, so a short path terminates early. BFS for fewest hops * (default); Dijkstra (min-heap) for least summed edge weight (`by: 'weight'` — * each edge costs its stored `weight`, the 0–1 connection strength, default 1.0). */ private async graphPathFallback( fromId: string, toId: string, options?: GraphPathOptions ): Promise { if (fromId === toId) return { nodes: [fromId], relationships: [], cost: 0 } const direction = options?.direction ?? 'both' const maxDepth = options?.maxDepth ?? Number.POSITIVE_INFINITY const subOptions: SubgraphOptions = { ...(options?.type !== undefined && { type: options.type }), ...(options?.includeInternal !== undefined && { includeInternal: options.includeInternal }), ...(options?.includeSystem !== undefined && { includeSystem: options.includeSystem }) } const pred = new Map() if (options?.by === 'weight') { const dist = new Map([[fromId, 0]]) const hops = new Map([[fromId, 0]]) const settled = new Set() const heap = new MinHeap() heap.push(fromId, 0) while (heap.size > 0) { const node = heap.pop() as string if (settled.has(node)) continue settled.add(node) if (node === toId) break const depth = hops.get(node) as number if (depth >= maxDepth) continue const baseCost = dist.get(node) as number const edges = await this.incidentEdges(node, direction, subOptions) for (const edge of edges) { const neighbor = edge.from === node ? edge.to : edge.from if (settled.has(neighbor)) continue // Cost = the edge's stored weight (0–1 connection strength, default 1.0). // Non-negative by construction, so Dijkstra stays valid. const weight = edge.weight ?? 1 const candidate = baseCost + weight if (!dist.has(neighbor) || candidate < (dist.get(neighbor) as number)) { dist.set(neighbor, candidate) hops.set(neighbor, depth + 1) pred.set(neighbor, { prev: node, edgeId: edge.id }) heap.push(neighbor, candidate) } } } if (!settled.has(toId)) return null return this.reconstructPath(fromId, toId, pred, dist.get(toId) as number) } // Fewest hops — breadth-first, each edge cost 1. const visited = new Set([fromId]) const queue: Array<{ id: string; depth: number }> = [{ id: fromId, depth: 0 }] let head = 0 while (head < queue.length) { const { id, depth } = queue[head++] if (depth >= maxDepth) continue const edges = await this.incidentEdges(id, direction, subOptions) for (const edge of edges) { const neighbor = edge.from === id ? edge.to : edge.from if (visited.has(neighbor)) continue visited.add(neighbor) pred.set(neighbor, { prev: id, edgeId: edge.id }) if (neighbor === toId) return this.reconstructPath(fromId, toId, pred, depth + 1) queue.push({ id: neighbor, depth: depth + 1 }) } } return null } /** Walk predecessor links from `toId` back to `fromId` into a forward route. */ private reconstructPath( fromId: string, toId: string, pred: Map, cost: number ): GraphPathResult { const nodes: string[] = [] const relationships: string[] = [] let cursor = toId while (cursor !== fromId) { nodes.push(cursor) const step = pred.get(cursor) as { prev: string; edgeId: string } relationships.push(step.edgeId) cursor = step.prev } nodes.push(fromId) nodes.reverse() relationships.reverse() return { nodes, relationships, cost } } /** Native pathfinding → resolve endpoints to ints, hydrate the returned route. */ private async graphPathNative( accel: GraphAccelerationProvider, fromId: string, toId: string, options?: GraphPathOptions ): Promise { const fromInt = this.graphEntityInt(fromId) const toInt = this.graphEntityInt(toId) if (fromInt === undefined || toInt === undefined) return null const verbTypes = options?.type ? (Array.isArray(options.type) ? options.type : [options.type]).map((t) => TypeUtils.getVerbIndex(t) ) : undefined const excluded = this.excludedVisibilityTiers(options ?? {}) const pathOptions: PathOptions = { ...(options?.direction !== undefined && { direction: options.direction }), ...(options?.by !== undefined && { by: options.by }), ...(verbTypes && { verbTypes }), ...(excluded && { excludeVisibility: excluded as unknown as string[] }), ...(options?.maxDepth !== undefined && { maxDepth: options.maxDepth }) } const result = await accel.path(fromInt, toInt, pathOptions) if (!result) return null const nodes = this.entityIntsToUuids(Array.from(result.nodeInts)) const relationships = ( await this.graphIndex.verbIntsToIds(Array.from(result.edgeVerbInts)) ).filter((id): id is string => id !== null) return { nodes, relationships, cost: result.cost } } // ============= SEARCH & DISCOVERY ============= /** * Unified find method - supports natural language and structured queries * Implements Triple Intelligence with parallel search optimization * * Combines three search dimensions in one query: * - **Vector (semantic):** `query` string is embedded and matched via HNSW similarity * - **Metadata:** `where` filters query the MetadataIndex (exact/range/set operators) * - **Graph:** `connected` traverses relationships via GraphAdjacencyIndex * * `data` is searchable via semantic/hybrid vector search (the `query` parameter). * `metadata` is searchable via structured `where` filters. * `type` in FindParams is an alias for filtering by `where.noun` (entity type). * * @param query - Natural language string or structured FindParams object * @returns Promise that resolves to array of search results with scores * * **Result Structure:** * Each result includes flattened entity fields for convenient access: * - `metadata`, `type`, `data` - Direct access (flattened from entity) * - `confidence`, `weight` - Entity confidence/importance (if set) * - `entity` - Full Entity object (backward compatible) * - `score` - Search relevance score (0-1) * * @example * // Natural language queries (most common) * const results = await brainy.find('users who work on AI projects') * const docs = await brainy.find('documents about machine learning') * const code = await brainy.find('JavaScript functions for data processing') * * @example * // Structured queries with filtering * const results = await brainy.find({ * query: 'artificial intelligence', * type: NounType.Document, * limit: 5, * where: { * status: 'published', * author: 'expert' * } * }) * * // Access flattened fields directly * for (const result of results) { * console.log(`Score: ${result.score}`) * console.log(`Type: ${result.type}`) // Flattened! * console.log(`Metadata:`, result.metadata) // Flattened! * console.log(`Confidence: ${result.confidence ?? 'N/A'}`) // Flattened! * console.log(`Weight: ${result.weight ?? 'N/A'}`) // Flattened! * } * * // Backward compatible: Nested access still works * console.log(result.entity.data) // Also works * * @example * // Metadata-only filtering (no vector search) * const activeUsers = await brainy.find({ * type: NounType.Person, * where: { * status: 'active', * department: 'engineering' * }, * service: 'user-management' * }) * * @example * // Vector similarity search with custom vectors * const queryVector = await brainy.embed('machine learning algorithms') * const similar = await brainy.find({ * vector: queryVector, * limit: 10, * type: [NounType.Document, NounType.Thing] * }) * * @example * // Proximity search (find entities similar to existing ones) * const relatedContent = await brainy.find({ * near: 'document-123', // Find entities similar to this one * limit: 8, * where: { * published: true * } * }) * * @example * // Pagination for large result sets * const firstPage = await brainy.find({ * query: 'research papers', * limit: 20, * offset: 0 * }) * * const secondPage = await brainy.find({ * query: 'research papers', * limit: 20, * offset: 20 * }) * * @example * // Complex search with multiple criteria * const results = await brainy.find({ * query: 'machine learning models', * type: [NounType.Thing, NounType.Document], * where: { * accuracy: { $gte: 0.9 }, // Metadata filtering * framework: { $in: ['tensorflow', 'pytorch'] } * }, * service: 'ml-pipeline', * limit: 15 * }) * * @example * // Empty query returns all entities (paginated) * const allEntities = await brainy.find({ * limit: 50, * offset: 0 * }) * * @example * // Performance-optimized search patterns * // Fast metadata-only search (no vector computation) * const fastResults = await brainy.find({ * type: NounType.Person, * where: { active: true }, * limit: 100 * }) * * // Combined vector + metadata for precision * const preciseResults = await brainy.find({ * query: 'senior developers', * where: { * experience: { $gte: 5 }, * skills: { $includes: 'javascript' } * }, * limit: 10 * }) * * @example * // Error handling and result processing * try { * const results = await brainy.find('complex query here') * * if (results.length === 0) { * console.log('No results found') * return * } * * // Filter by confidence threshold * const highConfidence = results.filter(r => r.score > 0.7) * * // Sort by score (already sorted by default) * const topResults = results.slice(0, 5) * * return topResults.map(r => ({ * id: r.id, * content: r.entity.data, * confidence: r.score, * metadata: r.entity.metadata * })) * } catch (error) { * console.error('Search failed:', error) * return [] * } * * @example * // VFS Filtering: Exclude VFS entities by default * // Knowledge graph queries stay clean - no VFS files in results * const knowledge = await brainy.find({ query: 'AI concepts' }) * // Returns only knowledge entities, VFS files excluded * * @example * // VFS entities included by default * const everything = await brainy.find({ * query: 'documentation' * }) * // Returns both knowledge entities AND VFS files * * @example * // Search only VFS files * const files = await brainy.find({ * where: { vfsType: 'file', extension: '.md' } * }) * * @example * // Exclude VFS entities (if needed) * const concepts = await brainy.find({ * query: 'machine learning', * excludeVFS: true // Exclude VFS files * }) */ // ============= AGGREGATION ============= /** * Define a named aggregate for incremental computation. * * Aggregate definitions persist across restarts. Once defined, all matching * entities (existing and future) contribute to the aggregate automatically. * * @param def - Aggregate definition (name, source filter, groupBy, metrics) * * @example * ```typescript * brain.defineAggregate({ * name: 'monthly_spending', * source: { type: NounType.Event, where: { domain: 'financial' } }, * groupBy: ['category', { field: 'date', window: 'month' }], * metrics: { * total: { op: 'sum', field: 'amount' }, * count: { op: 'count' }, * average: { op: 'avg', field: 'amount' } * } * }) * ``` */ defineAggregate(def: AggregateDefinition): void { this.ensureAggregationIndex() this._aggregationIndex!.defineAggregate(def) } /** * Register a field for cardinality + per-NounType breakdown stats. * * Layer 2 of the subtype-and-facets primitive: a lightweight wrapper over the * aggregation engine that auto-defines an internal `__fieldCounts__` * aggregate so consumers can query value frequencies without writing the * aggregate definition themselves. Backfill-on-define (shipped 7.23.0) means * existing entities are scanned on the first query, not at registration time. * * Use this for facets that don't warrant top-level promotion (e.g. `status`, * `source`, `role`). For sub-classification within a NounType, prefer the * top-level `subtype` field — it takes the standard-field fast path and has * its own statistics rollup. * * @param name - Field name to track. Resolves via the standard-fields-first / * metadata-fallback path, so both `'subtype'` (top-level) and `'status'` * (metadata) work the same way. * @param options - `perType` adds `noun` to the groupBy so counts are split * by NounType; `values` registers a whitelist that rejects writes containing * off-vocabulary values (validated in `add()`/`update()`). * * @example Track status without per-type breakdown * brain.trackField('status') * const counts = await brain.counts.byField('status') * // → { todo: 12, doing: 3, done: 47 } * * @example Track status with per-NounType breakdown * brain.trackField('status', { perType: true }) * const taskCounts = await brain.counts.byField('status', { type: NounType.Task }) * // → { todo: 8, doing: 2, done: 30 } * * @example Strict vocabulary * brain.trackField('priority', { values: ['low', 'medium', 'high'] }) * // brain.add({ ..., metadata: { priority: 'urgent' } }) throws */ trackField( name: string, options: { perType?: boolean; values?: string[] } = {} ): void { if (!name || typeof name !== 'string') { throw new Error('trackField: name must be a non-empty string') } const perType = options.perType === true const valuesSet = options.values && options.values.length > 0 ? new Set(options.values) : undefined this._trackedFields.set(name, { perType, values: valuesSet }) // Auto-define the backing aggregate. groupBy uses 'noun' (storage field name // for type) when perType is on, so the column-store key matches the persisted // shape and resolveEntityField doesn't need to rewrite the dimension. this.ensureAggregationIndex() const aggregateName = this.fieldCountsAggregateName(name) if (!this._aggregationIndex!.hasAggregate(aggregateName)) { this._aggregationIndex!.defineAggregate({ name: aggregateName, source: {}, groupBy: perType ? [name, 'noun'] : [name], metrics: { count: { op: 'count' } } }) } } /** * Internal aggregate name for a tracked field. Centralized so `trackField()` * and `counts.byField()` agree on the convention. */ private fieldCountsAggregateName(name: string): string { return `__fieldCounts__${name}` } /** * Register subtype enforcement for a specific NounType or VerbType. * * Two complementary mechanisms shipped together in 7.30.0: * * 1. **Per-type enforcement** (this method): mark a specific type as requiring * a subtype, optionally with a fixed vocabulary. Writes targeting that * type without a matching subtype throw at the boundary. * * 2. **Brain-wide strict mode**: enable via `new Brainy({ requireSubtype: true })`. * Every public write path checks the strict-mode rule. See * `BrainyConfig.requireSubtype`. * * Both compose: a per-type registration always applies regardless of the * brain-wide flag. * * @param type - The `NounType` or `VerbType` to register * @param options.values - Optional vocabulary whitelist (rejects off-vocab values) * @param options.required - When `true`, subtype is required on `add()`/`relate()`/`update()`/`updateRelation()` for this type (default: `true`) * * @example Lock down Person sub-classification * brain.requireSubtype(NounType.Person, { * values: ['employee', 'customer', 'vendor'], * required: true * }) * * @example Lock down management relationships * brain.requireSubtype(VerbType.ReportsTo, { * values: ['direct', 'dotted-line'], * required: true * }) */ requireSubtype( type: NounType | VerbType, options: { values?: string[]; required?: boolean } = {} ): void { if (type === undefined || type === null) { throw new Error('requireSubtype: type must be a valid NounType or VerbType') } const required = options.required !== false const valuesSet = options.values && options.values.length > 0 ? new Set(options.values) : undefined this._requiredSubtypes.set(String(type), { required, values: valuesSet }) } /** * Resolve the per-type subtype rule for a given NounType or VerbType. * Returns `null` when no rule is registered for the type. Used by the * write-path enforcement hooks (`enforceSubtypeOnAdd`, `enforceSubtypeOnRelate`). */ private getSubtypeRule(type: NounType | VerbType | undefined): { required: boolean; values?: Set } | null { if (type === undefined || type === null) return null return this._requiredSubtypes.get(String(type)) ?? null } /** * Check whether brain-wide strict mode is on for a given type. Brain-wide * `requireSubtype: true` enforces on every type; the `{ except: [...] }` * form allows the listed types through. Used by the write-path enforcement * hooks for both nouns and verbs. */ private brainWideStrictRequiresSubtype(type: NounType | VerbType | undefined): boolean { const flag = this.config.requireSubtype if (!flag) return false if (flag === true) return true // { except: [...] } form — strict except for listed types if (typeof flag === 'object' && Array.isArray(flag.except)) { if (type === undefined || type === null) return true return !flag.except.includes(type) } return false } /** * Whether a write should bypass subtype enforcement because it represents * internal Brainy infrastructure rather than user data. Currently triggers on: * * - `metadata.isVFSEntity === true` — Virtual File System root + directories * + file entities. These are platform plumbing and follow their own * subtype conventions (`'vfs-root'`, `'vfs-directory'`, `'vfs-file'`). * - `metadata.isVFS === true` — same intent; older marker. * * If user code sets these flags, they opt out of enforcement and accept the * responsibility. Documented as such on `AddParams.metadata` JSDoc. */ private isInfrastructureWrite(metadata: unknown): boolean { if (!metadata || typeof metadata !== 'object') return false const m = metadata as Record return m.isVFSEntity === true || m.isVFS === true } /** * Enforce subtype rules for a noun write (`add` / `update`). * * Walks the per-type registration AND the brain-wide strict-mode flag, and * throws with a descriptive message on missing or off-vocabulary values. * Called from `add()` / `addMany()` / `update()` before the storage write. * * Skips infrastructure writes (see `isInfrastructureWrite`) so Brainy's own * VFS root + directories + file entities don't get rejected when strict mode * is on. Vocabulary rules still apply to user-supplied subtypes — the bypass * only covers the missing-subtype case for internal plumbing. * * @param op - 'add' or 'update' (for error messages) * @param type - The NounType being written * @param subtype - The subtype value (or undefined) * @param metadata - The metadata bag (checked for infrastructure markers) */ private enforceSubtypeOnAdd( op: 'add' | 'update', type: NounType | undefined, subtype: string | undefined, metadata?: unknown ): void { const rule = this.getSubtypeRule(type) const strict = this.brainWideStrictRequiresSubtype(type) const isInfra = this.isInfrastructureWrite(metadata) if (!rule && !strict) return if (subtype === undefined || subtype === null || subtype === '') { if ((rule?.required || strict) && !isInfra) { throw new Error( this.formatSubtypeError({ op, kind: 'noun', typeName: String(type), issue: subtype === undefined ? 'undefined' : 'empty', rule, strict }) ) } return } if (rule?.values && !rule.values.has(subtype)) { throw new Error( this.formatSubtypeError({ op, kind: 'noun', typeName: String(type), issue: 'off-vocabulary', offValue: subtype, rule, strict }) ) } } /** * Enforce subtype rules for a verb write (`relate` / `updateRelation`). * Mirror of `enforceSubtypeOnAdd` for relationships. Skips infrastructure * edges (VFS containment, etc.) — same `isVFSEntity` / `isVFS` markers. */ private enforceSubtypeOnRelate( op: 'relate' | 'updateRelation', verb: VerbType | undefined, subtype: string | undefined, metadata?: unknown ): void { const rule = this.getSubtypeRule(verb) const strict = this.brainWideStrictRequiresSubtype(verb) const isInfra = this.isInfrastructureWrite(metadata) if (!rule && !strict) return if (subtype === undefined || subtype === null || subtype === '') { if ((rule?.required || strict) && !isInfra) { throw new Error( this.formatSubtypeError({ op, kind: 'verb', typeName: String(verb), issue: subtype === undefined ? 'undefined' : 'empty', rule, strict }) ) } return } if (rule?.values && !rule.values.has(subtype)) { throw new Error( this.formatSubtypeError({ op, kind: 'verb', typeName: String(verb), issue: 'off-vocabulary', offValue: subtype, rule, strict }) ) } } /** * Build the user-facing enforcement-error message. * * Three goals: * * 1. Diagnose: name the operation, the type, and what went wrong (missing, * empty, or off-vocabulary). * 2. Locate: include the first non-Brainy frame from the current stack so * the caller knows which line of THEIR code triggered the rejection — * eliminates the "grep your repo for `brain.add`" debugging step. * 3. Teach: include the canonical migration recipe URL so the next consumer * learns the SDK-vocabulary pattern from the error, not a postmortem. * * Added 7.30.1. */ private formatSubtypeError(opts: { op: string kind: 'noun' | 'verb' typeName: string issue: 'undefined' | 'empty' | 'off-vocabulary' offValue?: string rule: { required: boolean; values?: Set } | null strict: boolean }): string { const typeLabel = opts.kind === 'noun' ? 'NounType' : 'VerbType' const callSite = findCallerLocation() const vocab = opts.rule?.values ? Array.from(opts.rule.values).join(', ') : null let head: string if (opts.issue === 'off-vocabulary') { head = `${opts.op}(): ${typeLabel}.${opts.typeName} subtype '${opts.offValue}' is not in registered vocabulary [${vocab}].` } else { head = `${opts.op}(): ${typeLabel}.${opts.typeName} requires subtype but got ${opts.issue}.` } const callerLine = callSite ? ` at ${callSite}` : null let guidance: string if (vocab) { // The consumer (or a platform layer like the SDK) registered a specific // vocabulary. Tell them what to pass. guidance = ` Pass one of: ${vocab}.` } else if (opts.strict) { // Brain-wide strict mode is on without per-type values. Caller picks the // subtype string but it must be non-empty. guidance = ' Brain-wide strict mode (`requireSubtype: true`) is on — pass a non-empty `subtype` on every write, or add this type to `requireSubtype.except`.' } else { guidance = ' Register vocabulary via `brain.requireSubtype()` or pass a `subtype` value.' } const docLink = ' Migration recipe: https://soulcraft.com/docs/guides/subtypes-and-facets#strict-mode' return [head, callerLine, guidance, docLink].filter(Boolean).join('\n') } // findCallerLocation is now exported from src/utils/callerLocation.ts so // both the subtype enforcement errors here and the query-limit warnings in // paramValidation.ts can share it without re-importing brainy.ts. /** * Validate a metadata bag (or top-level field assignment) against any registered * value whitelists. Called from `add()`/`update()` after the standard zero-config * validation. Throws on the first off-vocabulary value to fail fast. * * Tracked fields with no `values` whitelist are skipped — registration alone * does not imply validation. */ private enforceTrackedFieldValues( bag: Record | undefined, bagLabel: 'metadata' | 'top-level' ): void { if (!bag || this._trackedFields.size === 0) return for (const [field, def] of this._trackedFields.entries()) { if (!def.values) continue if (!(field in bag)) continue const value = bag[field] if (value === undefined || value === null) continue const asString = typeof value === 'string' ? value : String(value) if (!def.values.has(asString)) { throw new Error( `trackField('${field}') rejected ${bagLabel} value '${asString}': not in registered vocabulary [${Array.from(def.values).join(', ')}]` ) } } } /** * Remove a named aggregate and clean up its state. * * @param name - Name of the aggregate to remove */ removeAggregate(name: string): void { if (this._aggregationIndex) { this._aggregationIndex.removeAggregate(name) } } /** * Query a named aggregate, returning the documented `AggregateResult[]` shape * (`{ groupKey, metrics, count }`) directly — the report-friendly view. * * `find({ aggregate })` returns the same data wrapped as search `Result` rows (with * `score`/`type`/`entity`) for uniformity with the rest of `find()`; this method is the * first-class analytics path for dashboards and reports. * * @param name - Aggregate name (must be defined via `defineAggregate`) * @param params - Optional `where` (group-key filter), `having` (metric filter), `orderBy`, `order`, `limit`, `offset` * @returns Computed group rows * * @example * const rows = await brain.queryAggregate('sales_by_category', { orderBy: 'revenue', order: 'desc', limit: 10 }) * // [{ groupKey: { category: 'food' }, metrics: { revenue: 17.5, count: 2 }, count: 2 }, ...] */ async queryAggregate( name: string, params?: Omit ): Promise { await this.ensureInitialized() this.ensureAggregationIndex() // Persisted definitions load asynchronously — wait for them before deciding // the aggregate doesn't exist (an app that relies on persisted definitions // without re-defining at boot would otherwise race a spurious throw here). await this._aggregationIndex!.ready() if (!this._aggregationIndex!.hasAggregate(name)) { throw new Error(`Aggregate '${name}' is not defined. Call defineAggregate() first.`) } await this.backfillAggregateIfNeeded(name) return this._aggregationIndex!.queryAggregate({ name, ...params }) } /** * Lazily create the AggregationIndex on first use. * Checks for a native 'aggregation' provider from plugins. */ private ensureAggregationIndex(): void { if (this._aggregationIndex) return const nativeProvider = this.pluginRegistry.getProvider('aggregation') this._aggregationIndex = new AggregationIndex(this.storage, nativeProvider) // Note: init() is async but definitions can be registered synchronously. // State loading happens lazily on first query. this._aggregationIndex.init().catch((err) => { // Non-fatal — definitions still work and state backfills on first query — // but a failed state load means aggregates read empty/stale until then, so // surface it loudly rather than swallow it. if (!this.config.silent) { prodLog.warn( `[Brainy] Aggregation index state failed to load at init ` + `(${(err as Error).message}). Aggregates may read empty until a ` + `backfill-on-query repopulates them.` ) } }) } /** * The visibility tiers a read should EXCLUDE, given the caller's opt-ins. * Default (no opts) hides both `'internal'` and `'system'`; `includeInternal` * unhides internal; `includeSystem` unhides system. Returns `null` when nothing * is excluded (both opts set) so callers can skip the filter entirely. * * @param params - The read params carrying `includeInternal` / `includeSystem`. * @returns The set of excluded tier strings, or `null` for "exclude nothing". */ private excludedVisibilityTiers( params: { includeInternal?: boolean; includeSystem?: boolean } ): EntityVisibility[] | null { const excluded: EntityVisibility[] = [] if (!params.includeInternal) excluded.push('internal') if (!params.includeSystem) excluded.push('system') return excluded.length > 0 ? excluded : null } /** * Resolve the set of entity/relationship ids to hide for this read, by asking * the metadata index for everything tagged with an excluded visibility tier. * Public entities carry NO stored `visibility` (absent === public), so they are * never in this set — exactly the entities we want to keep. Returns an empty set * when nothing is excluded. The result is used as a HARD candidate filter * (subtracted from candidate ids BEFORE pagination), so `limit`/`offset` stay correct. * * @param params - The read params carrying the visibility opt-ins. * @returns A set of ids to omit from results. */ private async resolveHiddenIds( params: { includeInternal?: boolean; includeSystem?: boolean } ): Promise> { const excluded = this.excludedVisibilityTiers(params) if (!excluded) return new Set() const ids = await this.metadataIndex.getIdsForFilter({ visibility: excluded.length === 1 ? excluded[0] : { oneOf: excluded } }) return new Set(ids) } /** * @description The unified Triple-Intelligence query. Composes vector similarity * (`query` / `vector` / `near`), metadata filtering (`type` / `subtype` / `where` / * `service`), and graph traversal (`connected`) into one ranked, paginated result * set — pass a bare string for a quick semantic search, or a {@link FindParams} * object to combine dimensions. Internal/system entities are hidden by default * (opt in with `includeInternal` / `includeSystem`); `orderBy`/`order` and * `limit`/`offset` apply across the composed result. Every returned row is * re-validated against its own predicate, so a stale or cross-bucket index entry * can never surface an entity that does not actually match. * @param query - A semantic search string, or a {@link FindParams} object combining * any of `query`, `vector`, `near`, `type`, `subtype`, `where`, `connected`, * `orderBy`/`order`, `limit`, `offset`. * @returns The matching {@link Result}s — flattened (`id`, `score`, `type`, * `metadata`, `data`, …) and ranked; an empty array when nothing matches. * @example * // Semantic + metadata + graph, composed in one call: * const rows = await brain.find({ * query: 'climate policy', * where: { year: { gte: 2020 } }, * connected: { from: orgId, via: VerbType.Authored }, * limit: 10 * }) */ async find(query: string | FindParams): Promise[]> { // Init only here — the migration gate is applied family-scoped below, once // the query params reveal which index families this read actually consults // (a string query may parse to a where / connected / vector shape). The lazy // loader and cold-read probes below already defer to a migrating provider. await this.ensureInitialized({ needs: [] }) // Ensure indexes are loaded (lazy loading when disableAutoRebuild: true) // This is a production-safe, concurrency-controlled lazy load await this.ensureIndexesLoaded() // One-shot cold-open self-heal: an O(1) probe of the metadata index (when the // provider offers one) repairs an already-poisoned index on first read — the // metadata counterpart of the graph cold-load guard. No-op for the JS index. await this.ensureMetadataConsistencyProbed() // Loudly flag a degraded derived index (failed init rebuild, or an // adopt-forward degraded commit) so a partial result is never mistaken for // authoritative. The streaming search() generator delegates to find(), so it // is covered here. this.warnIfReadsDegraded('find') // Parse natural language queries let params: FindParams = typeof query === 'string' ? await this.parseNaturalQuery(query) : query // Id normalization (8.0): resolve the graph-traversal anchor id(s) so a // caller may constrain by natural key. Each maps to the canonical UUID // add() stored; real UUIDs pass through. Done once here so every downstream // consumer of params.connected sees canonical ids. if (params.connected && (params.connected.from !== undefined || params.connected.to !== undefined)) { params = { ...params, connected: { ...params.connected, ...(params.connected.from !== undefined && { from: resolveEntityId(params.connected.from) }), ...(params.connected.to !== undefined && { to: resolveEntityId(params.connected.to) }) } } } // Zero-config validation (static import for performance) validateFindParams(params) // Family-scoped migration gate: wait only on the index families THIS query // consults, so a filter or graph query served from a healthy family never // blocks on an unrelated family's one-time 7.x → 8.0 migration. Placed once // params are final (after natural-language parse + connected-id resolution) // and before the aggregate path, which carries no gate of its own. await this.awaitMigrationLock(this.queryIndexFamilies(params)) // Aggregate query path — early return when params.aggregate is set if (params.aggregate) { return this.findAggregate(params) } // Visibility (8.0): resolve the ids to hide for this read ONCE. Default hides // internal + system; opt back in with includeInternal / includeSystem. Applied as a // hard candidate filter at every candidate-gathering point below so limit/offset stay // correct. Empty set when both opts are set (nothing excluded). const hiddenIds = await this.resolveHiddenIds(params) const isHidden = (id: string): boolean => hiddenIds.size > 0 && hiddenIds.has(id) const startTime = Date.now() let result = await (async () => { let results: Result[] = [] // Distinguish between search criteria (need vector search) and filter criteria (metadata only) // Treat empty string query as no query const hasVectorSearchCriteria = (params.query && params.query.trim() !== '') || params.vector || params.near const hasFilterCriteria = params.where || params.type || params.subtype || params.service const hasGraphCriteria = params.connected // Validate the raw where clause up front: an unrecognized operator (a typo // like `notIn`, or any non-operator key on a field's object value) throws a // typed BrainyError('INVALID_QUERY') instead of silently matching nothing. // Done before the index/vector/graph paths so it fires even on an empty // result set, and before brainy injects its own internal filter fields. if (params.where) { validateWhereFilter(params.where) } // Metadata cold-read guard: before trusting a filter result, verify the // field index actually serves a known persisted value (one-shot per brain). // A cold native index that has not loaded its `where` postings self-heals // here (rebuild) or throws MetadataIndexNotReadyError — never a silent []. // The graph counterpart (verifyGraphAdjacencyLive) covers `connected`. if (hasFilterCriteria) { await this.verifyMetadataLive() } // Handle metadata-only queries (no vector search needed) if (!hasVectorSearchCriteria && !hasGraphCriteria && hasFilterCriteria) { // Build filter for metadata index let filter: any = {} if (params.where) { Object.assign(filter, params.where) // Alias: where.type → where.noun (storage field name for entity type) if ('type' in filter && !('noun' in filter)) { filter.noun = filter.type delete filter.type } } if (params.service) filter.service = params.service // Subtype (top-level standard field — fast path, not metadata fallback). // Must be assigned BEFORE the type-array expansion below so the spread // into each anyOf branch carries it through. if (params.subtype !== undefined) { filter.subtype = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype } if (params.type) { const types = Array.isArray(params.type) ? params.type : [params.type] if (types.length === 1) { filter.noun = types[0] } else { filter = { anyOf: types.map(type => ({ noun: type, ...filter })) } } } // ExcludeVFS helper - ONLY exclude VFS infrastructure entities // Applied AFTER type filter to avoid execution order bugs // Excludes entities where: // - vfsType is 'file' or 'directory' (VFS files/folders) // - isVFSEntity is true (explicitly marked as VFS) // Includes extracted entities (person/concept/etc) even if they have vfsPath metadata if (params.excludeVFS === true) { // VFS infrastructure entities ALWAYS have vfsType set // Extracted entities do NOT have vfsType (undefined) filter.vfsType = { exists: false } // Extra safety: exclude entities explicitly marked as VFS filter.isVFSEntity = { ne: true } } // Apply sorting if requested, otherwise just filter let filteredIds: string[] if (params.orderBy) { // Get sorted IDs using production-scale sorted filtering. Bound the sort to // the page (offset+limit) plus the hidden over-fetch — so a broad filter + // orderBy returning 20 rows doesn't materialize every matching sorted id. const sortTopK = (params.offset || 0) + (params.limit || 10) + hiddenIds.size filteredIds = await this.metadataIndex.getSortedIdsForFilter( filter, params.orderBy, params.order || 'asc', sortTopK ) } else { // Just filter without sorting. Pass a page bound so a native provider can // early-stop and return only the page-prefix (killing the O(N) FFI marshal // at billion scale). Over-fetch by the hidden count, then re-window below; // offset stays 0 because the visibility filter + slice happen here. The JS // index ignores the bound and returns all matches (behaviour unchanged). const pageEnd = (params.offset || 0) + (params.limit || 10) + hiddenIds.size filteredIds = await this.metadataIndex.getIdsForFilter(filter, { limit: pageEnd, offset: 0 }) } // Visibility hard filter — drop hidden ids BEFORE pagination so limit is exact. if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) // Paginate BEFORE loading entities (production-scale!) const limit = params.limit || 10 const offset = params.offset || 0 const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { results.push(this.createResult(id, 1.0, entity)) } } return results } // Handle completely empty query - return all results paginated if (!hasVectorSearchCriteria && !hasFilterCriteria && !hasGraphCriteria) { const limit = params.limit || 20 const offset = params.offset || 0 // Unfiltered sort: column store handles this at O(K log S) scale. // No per-entity storage reads, no bucketing precision loss. if (params.orderBy) { // Over-fetch by the hidden count so dropping hidden ids still fills the page. const k = limit + offset + hiddenIds.size const sortedIntIds = await this.metadataIndex.columnStore.sortTopK( params.orderBy, params.order || 'asc', k ) // Convert int IDs (BigInt at the provider boundary) to UUIDs and // paginate. Number() narrowing is lossless under the u32 guard. const idMapper = this.metadataIndex.getIdMapper() let allUuids = sortedIntIds .map(intId => idMapper.getUuid(Number(intId))) .filter((uuid): uuid is string => uuid !== undefined) // Visibility hard filter — drop hidden ids BEFORE pagination. if (hiddenIds.size > 0) allUuids = allUuids.filter((id) => !hiddenIds.has(id)) const pageIds = allUuids.slice(offset, offset + limit) const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { results.push(this.createResult(id, 1.0, entity)) } } return results } // ExcludeVFS helper - exclude VFS infrastructure entities // VFS files/folders have vfsType set, extracted entities do NOT let filter: any = {} if (params.excludeVFS === true) { filter.vfsType = { exists: false } filter.isVFSEntity = { ne: true } } // Use metadata index when there's an actual filter (e.g. excludeVFS). The empty // filter returns nothing from getIdsForFilter, so the unfiltered case below uses // getNouns instead (it returns all nouns, including their visibility). if (Object.keys(filter).length > 0) { let filteredIds = await this.metadataIndex.getIdsForFilter(filter) // Visibility hard filter — drop hidden ids BEFORE pagination. if (hiddenIds.size > 0) filteredIds = filteredIds.filter((id) => !hiddenIds.has(id)) const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for 10x faster cloud storage performance const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { results.push(this.createResult(id, 1.0, entity)) } } } else { // No metadata filter, use direct storage query. Over-fetch by the hidden count // so the visibility filter below still fills the requested page (limit-exact). const storageResults = await this.storage.getNouns({ pagination: { limit: limit + offset + hiddenIds.size, offset: 0 } }) // Visibility hard filter on the materialized nouns (each carries `visibility`). const visible = hiddenIds.size > 0 ? storageResults.items.filter((noun) => noun && !hiddenIds.has(noun.id)) : storageResults.items for (let i = offset; i < Math.min(offset + limit, visible.length); i++) { const noun = visible[i] if (noun) { const entity = await this.convertNounToEntity(noun) results.push(this.createResult(noun.id, 1.0, entity)) } } } return results } // Metadata-first optimization: pre-resolve filter IDs before vector search. // This enables HNSW to search only within matching candidates instead of // doing expensive post-filtering. Native HNSW uses searchWithCandidates (Rust // bitmap), JS HNSW converts to a Set-based filter function. let preResolvedMetadataIds: string[] | null = null let preResolvedFilter: any = null // 8.0 #46 (CTX-PUSHDOWN-ALLOWEDIDS): the matched universe as an opaque roaring // Buffer, forwarded straight into the vector beam walk with NO id materialization // when the active metadata provider can produce one (native/cor). Absent on the // JS path — there the materialized `candidateIds` restricts the walk instead. let preResolvedAllowedIds: OpaqueIdSet | undefined if (params.where || params.type || params.subtype || params.service || params.excludeVFS) { preResolvedFilter = this.buildMetadataFilter(params) preResolvedMetadataIds = await this.metadataIndex.getIdsForFilter(preResolvedFilter) // Visibility hard filter — restrict the HNSW candidate set to non-hidden ids. if (hiddenIds.size > 0) { preResolvedMetadataIds = preResolvedMetadataIds.filter((id) => !hiddenIds.has(id)) } // Short-circuit: if metadata filter matches nothing, skip expensive vector search if (preResolvedMetadataIds.length === 0) { return [] } // 8.0 #46: when the active metadata provider exposes the opaque-set producer // (native/cor — the JS index does not), forward the matched universe to the // vector beam walk as an OpaqueIdSet (zero id materialization). The opaque set // is the RAW filter universe (the set is opaque, so the visibility exclusion // cannot be AND-ed into it in TS) — hidden tiers are instead dropped by the // post-search visibility hard filter below, and the JS path's materialized // `candidateIds` stays visibility-precise. Both forms are passed; the native // provider consumes the opaque one, the JS index the string one. const mip = this.metadataIndex as unknown as MetadataIndexProvider if (typeof mip.getIdSetForFilter === 'function') { preResolvedAllowedIds = await mip.getIdSetForFilter(preResolvedFilter) } } // Zero-Config Hybrid Search // Determine search mode: auto (default) combines text + semantic for query searches const searchMode = params.searchMode || 'auto' const limit = params.limit || 10 // Handle text-only query (user explicitly wants text search) if (searchMode === 'text' && params.query && params.query.trim() !== '') { results = await this.executeTextSearch(params.query, limit * 2) } // Handle semantic-only query (user explicitly wants vector search) else if ((searchMode === 'semantic' || searchMode === 'vector') && (params.query || params.vector)) { results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) } // Handle explicit hybrid or auto mode with query else if ((searchMode === 'auto' || searchMode === 'hybrid') && params.query && params.query.trim() !== '' && !params.vector) { // Zero-config hybrid: combine text + semantic search with RRF fusion const [textResults, semanticResults] = await Promise.all([ this.executeTextSearch(params.query, limit * 2), this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) ]) // Use user-specified alpha or auto-detect based on query length const alpha = params.hybridAlpha ?? this.autoAlpha(params.query) // Tokenize query for match visibility const queryWords = this.metadataIndex.tokenize(params.query) // RRF fusion combines both result sets with match visibility results = await this.rrfFusion(textResults, semanticResults, alpha, queryWords) } // Handle direct vector search (no query text) - no hybrid needed else if (params.vector && !params.query) { results = await this.executeVectorSearch(params, preResolvedMetadataIds ?? undefined, preResolvedAllowedIds) } // Handle proximity search else if (params.near) { results = await this.executeProximitySearch(params) } // Execute parallel searches for additional criteria (proximity search in addition to query) if (params.near && params.query) { const proximityResults = await this.executeProximitySearch(params) results.push(...proximityResults) } // Remove duplicate results from parallel searches if (results.length > 0) { const uniqueResults = new Map>() for (const result of results) { const existing = uniqueResults.get(result.id) if (!existing || result.score > existing.score) { uniqueResults.set(result.id, result) } } results = Array.from(uniqueResults.values()) } // Visibility hard filter for vector/text/proximity candidates (the pre-resolved // path already excluded hidden ids; this covers searches with no metadata filter). // Applied BEFORE the final sort+slice so limit/offset remain exact. if (hiddenIds.size > 0 && results.length > 0) { results = results.filter((r) => !isHidden(r.id)) } // Apply metadata filtering using pre-resolved IDs (metadata-first optimization). // When vector search was performed, HNSW already filtered by candidateIds — // this block handles pagination, entity loading, and metadata-only queries. if (preResolvedMetadataIds && preResolvedFilter) { const filteredIds = preResolvedMetadataIds if (results.length > 0) { // Filter results by pre-resolved metadata IDs. // With metadata-first HNSW, most results already match — this is a safety net // for text search results and proximity results that weren't pre-filtered. const filteredIdSet = new Set(filteredIds) results = results.filter((r) => filteredIdSet.has(r.id)) // Apply early pagination for vector + metadata queries const limit = params.limit || 10 const offset = params.offset || 0 // If we have enough filtered results, sort and paginate early. // Rank by score (top offset+limit), then drop the offset — identical ordering // to a full `sort((a, b) => b.score - a.score)` + slice, but the native // `sort:topK` provider can compute only the page instead of the full sort. if (results.length >= offset + limit) { const k = offset + limit const order = rankIndicesByScore(results.map(r => r.score), k, true) results = reorderByIndices(results, order).slice(offset, k) // Batch-load entities only for the paginated results (10x faster on GCS) const idsToLoad = results.filter(r => !r.entity).map(r => r.id) if (idsToLoad.length > 0) { const entitiesMap = await this.batchGet(idsToLoad) for (const result of results) { if (!result.entity) { const entity = entitiesMap.get(result.id) if (entity) { result.entity = entity } } } } // Early return if no other processing needed if (!params.connected && !params.fusion) { return results } } } else { // OPTIMIZED: Apply pagination to filtered IDs BEFORE loading entities const limit = params.limit || 10 const offset = params.offset || 0 const pageIds = filteredIds.slice(offset, offset + limit) // Batch-load entities for current page - O(page_size) instead of O(total_results) // GCS: 10 entities = 1×50ms vs 10×50ms = 500ms (10x faster) const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { results.push(this.createResult(id, 1.0, entity)) } } // Early return for metadata-only queries with pagination applied if (!params.query && !params.connected) { // Apply sorting if requested for metadata-only queries if (params.orderBy) { // Page-bounded sort (offset+limit + hidden over-fetch) so we don't // materialize every matching sorted id to return one page. const limit = params.limit || 10 const offset = params.offset || 0 const sortedIds = await this.metadataIndex.getSortedIdsForFilter( preResolvedFilter, params.orderBy, params.order || 'asc', offset + limit + hiddenIds.size ) // Paginate sorted IDs BEFORE loading entities (production-scale!) const pageIds = sortedIds.slice(offset, offset + limit) // Batch-load entities for paginated results (10x faster on GCS) const sortedResults: Result[] = [] const entitiesMap = await this.batchGet(pageIds) for (const id of pageIds) { const entity = entitiesMap.get(id) if (entity) { sortedResults.push(this.createResult(id, 1.0, entity)) } } return sortedResults } return results } } } // Graph search component with O(1) traversal if (params.connected) { results = await this.executeGraphSearch(params, results) } // Apply fusion scoring if requested if (params.fusion && results.length > 0) { results = this.applyFusionScoring(results, params.fusion) } // OPTIMIZED: Sort first, then apply efficient pagination // Support custom orderBy for vector + metadata queries if (params.orderBy && results.length > 0) { // For vector + metadata queries, sort by specified field instead of score // Load sort field values for all results (small set, already filtered) const resultsWithValues = await Promise.all(results.map(async (r) => ({ result: r, value: await this.metadataIndex.getFieldValueForEntity(r.id, params.orderBy!) }))) // Sort by field value resultsWithValues.sort((a, b) => { // Handle null/undefined if (a.value == null && b.value == null) return 0 if (a.value == null) return (params.order || 'asc') === 'asc' ? 1 : -1 if (b.value == null) return (params.order || 'asc') === 'asc' ? -1 : 1 // Compare values if (a.value === b.value) return 0 const comparison = a.value < b.value ? -1 : 1 return (params.order || 'asc') === 'asc' ? comparison : -comparison }) results = resultsWithValues.map(({ result }) => result) } else { // Default: sort by relevance score. Rank to the page (offset+limit) via the // swappable sort:topK seam; the slice below drops the offset. Ordering is // identical to a full `sort((a, b) => b.score - a.score)`. const k = (params.offset || 0) + limit const order = rankIndicesByScore(results.map(r => r.score), k, true) results = reorderByIndices(results, order) } const finalOffset = params.offset || 0 // Efficient pagination - only slice what we need (limit already defined above) return results.slice(finalOffset, finalOffset + limit) })() // Index-integrity guard — applied ONCE here so every find() path (metadata, // vector, text, proximity, graph) is covered uniformly. The indexes are // acceleration structures; the loaded entity is ground truth. Re-validate // each result against the query predicate so a stale or cross-bucket index // entry — e.g. an id left in a field-value posting by a delete or an // `update({ field: undefined })` — can never surface an entity that does not // actually match `type`/`subtype`/`where`/`service`/`excludeVFS`. This is // the live-path mirror of the historical path's per-candidate // `entityMatchesFind` (db.ts). On a healthy index it is a no-op; on a // corrupted one it drops the bad row instead of returning a phantom. if (result.length > 0) { result = result.filter((r) => { if (r.entity == null) return false try { return entityMatchesFind(r.entity as unknown as Entity, params as unknown as FindParams) } catch { // The JS matcher doesn't implement a where-operator the provider already // matched on (e.g. a future native-only operator). The guard cannot // DISPROVE a match the provider made, so keep the row rather than // hard-failing a correct provider result. (The db.ts historical path // keeps its throw, which legitimately reroutes to materialization.) return true } }) } // includeVectors — opt-in vector hydration. Default (false) keeps the perf // contract: every result path above builds entities via the metadata-only // fast path, so `entity.vector` is the empty stub. When requested, fetch the // stored vectors for the already-paginated page in ONE batch and attach them, // mirroring get({ includeVectors: true }). Done once here so every find() // path (metadata-only, vector/text/proximity, graph) honors it uniformly. if (params.includeVectors && result.length > 0) { const nounsMap = await this.storage.getNounBatch(result.map((r) => r.id)) for (const r of result) { const noun = nounsMap.get(r.id) if (noun?.vector && r.entity) { r.entity.vector = noun.vector } } } // Record performance for auto-tuning const duration = Date.now() - startTime recordQueryPerformance(duration, result.length) return result } /** * Find similar entities using vector similarity * * @param params - Parameters specifying the target for similarity search * @param params.to - Entity ID, Entity object, or Vector to find similar to (required) * @param params.limit - Maximum results (default: 10) * @param params.threshold - Minimum similarity (0-1) * @param params.type - Filter by NounType(s) * @param params.where - Metadata filters * @returns Promise that resolves to array of Result objects with similarity scores (same structure as find()) * * **Returns:** * Same Result structure as find() with flattened fields for convenient access * * @example * // Find entities similar to a specific entity by ID * const similarDocs = await brainy.similar({ * to: 'document-123', * limit: 10 * }) * * // Access flattened fields * for (const result of similarDocs) { * console.log(`Similarity: ${result.score}`) * console.log(`Type: ${result.type}`) // Flattened! * console.log(`Metadata:`, result.metadata) // Flattened! * console.log(`Confidence: ${result.confidence ?? 'N/A'}`) // Flattened! * } * * @example * // Find similar entities with type filtering * const similarUsers = await brainy.similar({ * to: 'user-456', * type: NounType.Person, * limit: 5, * where: { * active: true, * department: 'engineering' * } * }) * * @example * // Find similar using a custom vector * const customVector = await brainy.embed('artificial intelligence research') * const similar = await brainy.similar({ * to: customVector, * limit: 8, * type: [NounType.Document, NounType.Thing] * }) * * @example * // Find similar using an entity object * const sourceEntity = await brainy.get('research-paper-789') * if (sourceEntity) { * const relatedPapers = await brainy.similar({ * to: sourceEntity, * limit: 12, * where: { * published: true, * category: 'machine-learning' * } * }) * } * * @example * // Content recommendation system * async function getRecommendations(userId: string) { * // Get user's recent interactions * const user = await brainy.get(userId) * if (!user) return [] * * // Find similar content * const recommendations = await brainy.similar({ * to: userId, * type: NounType.Document, * limit: 20, * where: { * published: true, * language: 'en' * } * }) * * // Filter out already seen content * return recommendations.filter(rec => * !user.metadata.viewedItems?.includes(rec.id) * ) * } * * @example * // Duplicate detection system * async function findPotentialDuplicates(entityId: string) { * const duplicates = await brainy.similar({ * to: entityId, * limit: 10 * }) * * // High similarity might indicate duplicates * const highSimilarity = duplicates.filter(d => d.score > 0.95) * * if (highSimilarity.length > 0) { * console.log('Potential duplicates found:', highSimilarity.map(d => d.id)) * } * * return highSimilarity * } * * @example * // Error handling for missing entities * try { * const similar = await brainy.similar({ * to: 'nonexistent-entity', * limit: 5 * }) * } catch (error) { * if (error instanceof EntityNotFoundError) { * console.log(`Source entity does not exist: ${error.id}`) * // Handle missing source entity * } * } */ async similar(params: SimilarParams): Promise[]> { await this.ensureInitialized() // Get target vector let targetVector: Vector if (typeof params.to === 'string') { // Need vector for similarity, so use includeVectors: true const entity = await this.get(params.to, { includeVectors: true }) if (!entity) { throw new EntityNotFoundError(params.to) } targetVector = entity.vector } else if (Array.isArray(params.to)) { targetVector = params.to as Vector } else { // Entity object passed - check if vectors are loaded const entityVector = (params.to as Entity).vector if (!entityVector || entityVector.length === 0) { throw new Error( 'Entity passed to brain.similar() has no vector embeddings loaded. ' + 'Please retrieve the entity with { includeVectors: true } or pass the entity ID instead.\n\n' + 'Example: brain.similar({ to: entityId }) OR brain.similar({ to: await brain.get(entityId, { includeVectors: true }) })' ) } targetVector = entityVector } // Use find with vector const results = await this.find({ vector: targetVector, limit: params.limit, type: params.type, where: params.where, service: params.service, excludeVFS: params.excludeVFS // Pass through VFS filtering }) // A min-similarity `threshold` is applied as a post-filter on result.score // — the canonical way to impose a minimum score on plain semantic results // (top-level vector search does not honor it; see the find({ near }) guidance). // Previously `params.threshold` was silently dropped. return params.threshold != null ? results.filter((r) => r.score >= params.threshold!) : results } // ============= BATCH OPERATIONS ============= /** * Add multiple entities in a single batch operation * * Uses batch embedding (embedBatch) to pre-compute all vectors before any * storage write, keeping model inference out of the per-item write path. * (On the default WASM engine, batch throughput is comparable to sequential * embed() calls — measured ~160 ms/text either way; a native embedding * provider may batch faster.) Automatically adapts batch size and * parallelism to the storage adapter (e.g., smaller batches for cloud * storage). * * @param params - Batch add parameters * @param params.items - Array of AddParams (same shape as brain.add()) * @param params.parallel - Process in parallel (default: true, auto-adapts per storage) * @param params.chunkSize - Batch size per storage round-trip (default: auto from storage) * @param params.onProgress - Callback: (completed, total) => void * @param params.continueOnError - If true, skip failed items instead of aborting * @returns BatchResult with successful (string[] of IDs), failed, total, duration * * @example * ```typescript * const result = await brain.addMany({ * items: [ * { data: 'First entity', type: NounType.Document, metadata: { priority: 1 } }, * { data: 'Second entity', type: NounType.Concept } * ], * onProgress: (done, total) => console.log(`${done}/${total}`) * }) * console.log(`Added ${result.successful.length}, failed ${result.failed.length}`) * ``` */ async addMany(params: AddManyParams): Promise> { this.assertWritable('addMany') await this.ensureInitialized() // Pre-validate every item against per-type + brain-wide subtype rules BEFORE // any storage write — atomic-fail semantics: a missing-subtype anywhere in // the batch fails the whole call, no partial writes. (7.30.0) for (let i = 0; i < params.items.length; i++) { const item = params.items[i] try { this.enforceSubtypeOnAdd('add', item.type, item.subtype, item.metadata) } catch (err) { const msg = err instanceof Error ? err.message : String(err) throw new Error(`addMany(): item[${i}] failed subtype enforcement: ${msg}`) } } // Get optimal batch configuration from storage adapter // This automatically adapts to storage characteristics: // - GCS: 50 batch size, 100ms delay, sequential // - S3/R2: 100 batch size, 50ms delay, parallel // - Memory: 1000 batch size, 0ms delay, parallel const storageConfig = this.storage.getBatchConfig() // Use storage preferences (allow explicit user override) const batchSize = params.chunkSize ?? storageConfig.maxBatchSize const parallel = params.parallel ?? storageConfig.supportsParallelWrites const delayMs = storageConfig.batchDelayMs const result: BatchResult = { successful: [], failed: [], total: params.items.length, duration: 0 } const startTime = Date.now() let lastBatchTime = Date.now() // OPTIMIZATION: Pre-compute vectors using batch embedding // Items that already have vectors are skipped // This changes N individual WASM calls → 1 batched WASM call (5-10x faster) const itemsNeedingEmbedding: { index: number; text: string }[] = [] for (let i = 0; i < params.items.length; i++) { const item = params.items[i] if (!item.vector && item.data !== undefined && item.data !== null) { // Convert data to string for embedding const text = typeof item.data === 'string' ? item.data : JSON.stringify(item.data) itemsNeedingEmbedding.push({ index: i, text }) } } // Batch embed all texts that need vectors if (itemsNeedingEmbedding.length > 0) { const texts = itemsNeedingEmbedding.map(item => item.text) const vectors = await this.embedBatch(texts) // Attach pre-computed vectors to items for (let i = 0; i < itemsNeedingEmbedding.length; i++) { const { index } = itemsNeedingEmbedding[i] // Mutate the item to include the pre-computed vector // This way add() will skip embedding (vector already provided) params.items[index].vector = vectors[i] } } // OPTIMIZATION: Defer HNSW persistence during batch insert. // Without this, each add() triggers ~16-20 neighbor saveVectorIndexData calls // (each a read→gzip→atomic-write cycle). For 450 items that's ~8,100 // individual storage writes. Deferred mode collects dirty node IDs and // flushes once at the end — deduplicating repeated neighbor updates. const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks const prevPersistMode = typeof index.getPersistMode === 'function' ? index.getPersistMode() : null const canDefer = prevPersistMode === 'immediate' && typeof index.setPersistMode === 'function' && typeof index.flush === 'function' if (canDefer) { // canDefer verified setPersistMode is a function just above. index.setPersistMode!('deferred') } try { // Process in batches for (let i = 0; i < params.items.length; i += batchSize) { const chunk = params.items.slice(i, i + batchSize) const promises = chunk.map(async (item) => { try { // ifAbsent (7.31.0) / upsert — propagate each batch-level flag to every // item, but let a per-item flag take precedence so callers can override // individual rows. The two flags stay independent: per-item upsert handling // (create-or-merge) and per-item ifAbsent handling (create-or-skip) both // fire inside the delegated add() call, which also enforces their mutual // exclusion when a single resolved item carries both. const resolvedItem = (params.ifAbsent && item.ifAbsent === undefined) || (params.upsert && item.upsert === undefined) ? { ...item, ...(params.ifAbsent && item.ifAbsent === undefined && { ifAbsent: true }), ...(params.upsert && item.upsert === undefined && { upsert: true }) } : item const id = await this.add(resolvedItem) result.successful.push(id) } catch (error) { result.failed.push({ item, error: (error as Error).message }) if (!params.continueOnError) { throw error } } }) // Parallel vs Sequential based on storage preference if (parallel) { await Promise.allSettled(promises) } else { // Sequential processing for rate-limited storage for (const promise of promises) { await promise } } // Progress callback if (params.onProgress) { params.onProgress( result.successful.length + result.failed.length, result.total ) } // Adaptive delay between batches if (i + batchSize < params.items.length && delayMs > 0) { const batchDuration = Date.now() - lastBatchTime // If batch was too fast, add delay to respect rate limits if (batchDuration < delayMs) { await new Promise(resolve => setTimeout(resolve, delayMs - batchDuration) ) } lastBatchTime = Date.now() } } } finally { // Restore persist mode and flush all dirty nodes in one pass. // canDefer implies prevPersistMode === 'immediate', so the null re-check // never fires at runtime — it narrows the type for the restore call. if (canDefer && prevPersistMode !== null) { await index.flush() // canDefer verified setPersistMode is a function before the try block. index.setPersistMode!(prevPersistMode) } } result.duration = Date.now() - startTime return result } /** * Remove multiple entities */ async removeMany(params: RemoveManyParams): Promise> { this.assertWritable('removeMany') await this.ensureInitialized() // Determine what to delete let idsToDelete: string[] = [] if (params.ids) { // Id normalization (8.0): resolve each supplied id to the canonical UUID // add() stored, so callers may delete by natural key. The find()-derived // path below already yields canonical ids. idsToDelete = params.ids.map((id) => resolveEntityId(id)) } else if (params.type || params.where) { // Find entities to delete const entities = await this.find({ type: params.type, where: params.where, limit: params.limit || 1000 }) idsToDelete = entities.map((e) => e.id) } const result: BatchResult = { successful: [], failed: [], total: idsToDelete.length, duration: 0 } const startTime = Date.now() // Batch deletes into chunks for 10x faster performance with proper error handling. // Single transaction per chunk = atomic within chunk, graceful failure across chunks. // Chunk size is storage-adaptive (matching addMany/relateMany): the adapter's // maxBatchSize (memory: 1000, S3/R2: 100, GCS: 50) unless the caller overrides it. const storageConfig = this.storage.getBatchConfig() const chunkSize = params.chunkSize ?? storageConfig.maxBatchSize for (let i = 0; i < idsToDelete.length; i += chunkSize) { const chunk = idsToDelete.slice(i, i + chunkSize) // Track IDs queued during builder phase separately from confirmed deletions. // result.successful must only be updated AFTER the transaction commits — pushing // inside the builder runs before transaction.execute(), so a rollback would leave // successfully-queued IDs incorrectly listed as deleted. const chunkQueued: string[] = [] const chunkBuilderFailed: Array<{ item: string; error: string }> = [] // Model-B pre-pass: collect the chunk's touched-id set (entities + their // cascade-deleted relationships) so the generation captures before-images // for all of them. Tolerant by design — the builder below still does the // authoritative per-id load + error handling, and an over-included id whose // delete is skipped simply gets a before-image equal to its live state (a // harmless no-op history entry). const touchedNouns: string[] = [...chunk] const touchedVerbs: string[] = [] for (const id of chunk) { try { const srcVerbs = await this.storage.getVerbsBySource(id) const tgtVerbs = await this.storage.getVerbsByTarget(id) for (const v of [...srcVerbs, ...tgtVerbs]) touchedVerbs.push(v.id) } catch { // Ignore — the builder's per-id try/catch is authoritative. } } try { // Change feed: the BUILDER populates this (per id actually staged), so // an id whose builder failed never emits — the array is read only // after the chunk's commit succeeds. Verb events dedupe within the // chunk (two related chunk-members see the same cascade verb twice). const chunkEvents: PendingChangeEvent[] | undefined = this._changeFeed .hasListeners ? [] : undefined const seenVerbEvents = new Set() // Process chunk as ONE atomic Model-B generation (entities + cascade verbs). await this.persistSingleOp({ nouns: touchedNouns, verbs: touchedVerbs }, async (tx) => { for (const id of chunk) { try { // Load entity data const metadata = await this.storage.getNounMetadata(id) const noun = await this.storage.getNoun(id) const verbs = await this.storage.getVerbsBySource(id) const targetVerbs = await this.storage.getVerbsByTarget(id) const allVerbs = [...verbs, ...targetVerbs] // Add delete operations to transaction if (noun) { tx.addOperation( new RemoveFromHNSWOperation(this.index, id, noun.vector) ) } if (metadata) { tx.addOperation( new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata) ) } // Pre-read metadata rides along: the count decrement must not // depend on re-reading the record being removed (see remove()). tx.addOperation( new DeleteNounMetadataOperation(this.storage, id, metadata) ) for (const verb of allVerbs) { const { sourceInt, targetInt } = this.resolveVerbEndpointInts(verb) tx.addOperation( new RemoveFromGraphIndexOperation(this.graphIndex, verb, { sourceInt, targetInt }, this.graphWriteGeneration) ) tx.addOperation( new DeleteVerbMetadataOperation(this.storage, verb.id) ) } if (chunkEvents) { chunkEvents.push({ kind: 'entity', op: 'remove', id, ...(metadata && { entity: this.entityViewFromRawRecord( id, metadata as Record ) }) }) for (const verb of allVerbs) { if (seenVerbEvents.has(verb.id)) continue seenVerbEvents.add(verb.id) chunkEvents.push({ kind: 'relation', op: 'unrelate', id: verb.id, relation: { id: verb.id, from: verb.sourceId, to: verb.targetId, type: String(verb.verb) } }) } } chunkQueued.push(id) } catch (error) { chunkBuilderFailed.push({ item: id, error: (error as Error).message }) if (!params.continueOnError) { throw error } } } }, undefined, chunkEvents) // Transaction committed — queued IDs were actually deleted result.successful.push(...chunkQueued) result.failed.push(...chunkBuilderFailed) } catch (error) { // Transaction failed/rolled back — queued IDs were NOT deleted result.failed.push(...chunkBuilderFailed) for (const id of chunkQueued) { result.failed.push({ item: id, error: (error as Error).message }) } // Stop processing if continueOnError is false if (!params.continueOnError) { break } } if (params.onProgress) { params.onProgress( result.successful.length + result.failed.length, result.total ) } } result.duration = Date.now() - startTime return result } /** * Update multiple entities with batch processing */ async updateMany(params: UpdateManyParams): Promise> { await this.ensureInitialized() const result: BatchResult = { successful: [], failed: [], total: params.items.length, duration: 0 } const startTime = Date.now() const chunkSize = params.chunkSize || 100 // Process in chunks for (let i = 0; i < params.items.length; i += chunkSize) { const chunk = params.items.slice(i, i + chunkSize) const promises = chunk.map(async (item, chunkIndex) => { try { await this.update(item) result.successful.push(item.id) } catch (error) { result.failed.push({ item, error: (error as Error).message }) if (!params.continueOnError) { throw error } } }) if (params.parallel !== false) { await Promise.allSettled(promises) } else { for (const promise of promises) { await promise } } // Report progress if (params.onProgress) { params.onProgress( result.successful.length + result.failed.length, result.total ) } } result.duration = Date.now() - startTime return result } /** * Create multiple relationships in a single batch operation * * Automatically adapts batch size and parallelism to the storage adapter. * Duplicate relationships are detected and deduplicated per relate(). * * @param params - Batch relate parameters * @param params.items - Array of RelateParams (same shape as brain.relate()) * @param params.parallel - Process in parallel (default: true, auto-adapts per storage) * @param params.chunkSize - Batch size per storage round-trip (default: auto from storage) * @param params.onProgress - Callback: (completed, total) => void * @param params.continueOnError - If true, skip failed items instead of aborting * @returns Array of relationship IDs (string[]) * * @example * ```typescript * const ids = await brain.relateMany({ * items: [ * { from: id1, to: id2, type: VerbType.RelatedTo }, * { from: id1, to: id3, type: VerbType.Contains, data: 'section content' } * ] * }) * ``` */ async relateMany(params: RelateManyParams): Promise { this.assertWritable('relateMany') await this.ensureInitialized() // Pre-validate every item against per-type + brain-wide subtype rules BEFORE // any storage write — atomic-fail semantics: a missing-subtype anywhere in // the batch fails the whole call, no partial writes. (7.30.0) for (let i = 0; i < params.items.length; i++) { const item = params.items[i] try { this.enforceSubtypeOnRelate('relate', item.type, item.subtype, item.metadata) } catch (err) { const msg = err instanceof Error ? err.message : String(err) throw new Error(`relateMany(): item[${i}] failed subtype enforcement: ${msg}`) } } // Get optimal batch configuration from storage adapter // Automatically adapts to storage characteristics const storageConfig = this.storage.getBatchConfig() // Use storage preferences (allow explicit user override) const batchSize = params.chunkSize ?? storageConfig.maxBatchSize const parallel = params.parallel ?? storageConfig.supportsParallelWrites const delayMs = storageConfig.batchDelayMs const result: BatchResult = { successful: [], failed: [], total: params.items.length, duration: 0 } const startTime = Date.now() let lastBatchTime = Date.now() for (let i = 0; i < params.items.length; i += batchSize) { const chunk = params.items.slice(i, i + batchSize) if (parallel) { // Parallel processing const promises = chunk.map(async (item) => { try { const relationId = await this.relate(item) result.successful.push(relationId) } catch (error: any) { result.failed.push({ item, error: error.message || 'Unknown error' }) if (!params.continueOnError) { throw error } } }) await Promise.allSettled(promises) } else { // Sequential processing for (const item of chunk) { try { const relationId = await this.relate(item) result.successful.push(relationId) } catch (error: any) { result.failed.push({ item, error: error.message || 'Unknown error' }) if (!params.continueOnError) { throw error } } } } // Progress callback if (params.onProgress) { params.onProgress( result.successful.length + result.failed.length, result.total ) } // Adaptive delay if (i + batchSize < params.items.length && delayMs > 0) { const batchDuration = Date.now() - lastBatchTime if (batchDuration < delayMs) { await new Promise(resolve => setTimeout(resolve, delayMs - batchDuration) ) } lastBatchTime = Date.now() } } result.duration = Date.now() - startTime return result.successful } /** * @description Clear ALL data from the database — entities, relationships, * blobs, and every derived index. The instance remains fully usable * afterwards: the vector, metadata, and graph indexes are re-resolved * exactly as on `init()`, and when the VFS was active a fresh VFS root * entity is re-created (so a thresholdless vector search against a cleared * store can still surface that one root entity). * @example * await brain.clear() */ async clear(): Promise { await this.ensureInitialized() // Clear storage await this.storage.clear() // Invalidate GraphAdjacencyIndex to prevent stale in-memory data // The index has LSMTree data and verbIdSet pointing to deleted entities. // Without this, relate()'s duplicate check uses stale data, potentially // allowing duplicate relationships or missing valid duplicates. if (typeof this.storage.invalidateGraphIndex === 'function') { this.storage.invalidateGraphIndex() } // Reset index if ('clear' in this.index && typeof this.index.clear === 'function') { await this.index.clear() } else { // Recreate index using plugin factory when available this.index = this.createIndex() } // Recreate metadata index to clear cached data — mirror init()'s // resolution: provider factory first, then the JS implementation (with a // plugin-provided native entityIdMapper when registered). const clearMetadataFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('metadataIndex') if (clearMetadataFactory) { this.metadataIndex = clearMetadataFactory(this.storage) } else { const entityIdMapperFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('entityIdMapper') this.metadataIndex = new MetadataIndexManager(this.storage, {}, { entityIdMapper: entityIdMapperFactory ? entityIdMapperFactory(this.storage) : undefined, }) } await this.metadataIndex.init() // Re-resolve the graph index the same way init() does (provider factory, // else the storage layer's lazy singleton) and re-wire the shared // UUID ↔ int resolver. Without this, every graph-touching call after // clear() — relate(), getNeighbors(), stats() — would hit an undefined // index. const clearGraphFactory = this.pluginRegistry.getProvider<(storage: StorageAdapter) => any>('graphIndex') if (clearGraphFactory) { this.graphIndex = clearGraphFactory(this.storage) this.storage.setGraphIndex(this.graphIndex) } else { this.graphIndex = await this.storage.getGraphIndex() } this.wireGraphIdResolver() // Reset dimensions this.dimensions = undefined // Clear any cached sub-APIs this._nlp = undefined this._tripleIntelligence = undefined // Re-initialize the content-addressed blob store after storage.clear() // (clear() drops the BlobStorage instance). The VFS stores all file // content through it, so this must happen BEFORE VFS reinitialization. if (typeof this.storage.initializeBlobStorage === 'function') { await this.storage.initializeBlobStorage() } // Reset VFS state - root entity was deleted by storage.clear() // Bug: VFS instance remained in memory pointing to deleted root entity if (this._vfs) { // Clear PathResolver caches (including UnifiedCache VFS entries). // Boundary: peeks at VirtualFileSystem's private `pathResolver` member — // VFS exposes no public cache-invalidation surface, and the resolver is // about to be discarded with the VFS instance recreated below. const vfsInternals = this._vfs as unknown as { pathResolver?: { invalidateAllCaches?: () => void } } if (vfsInternals.pathResolver?.invalidateAllCaches) { vfsInternals.pathResolver.invalidateAllCaches() } // Recreate and reinitialize VFS so it's ready for use this._vfs = new VirtualFileSystem(this) await this._vfs.init() // _vfsInitialized remains true since we just initialized } else { // VFS was never used, reset flag for clean state this._vfsInitialized = false } // Change feed: clear() wipes the store wholesale (raw, not per-record) — // one store-level event tells subscribers everything they held is gone. this._changeFeed.emit([ { kind: 'store', op: 'clear', timestamp: Date.now() } ]) } // ─── Migration API ─────────────────────────────────────────────── /** * Run pending data migrations, or preview what would change. * * @param options - Pass `{ dryRun: true }` to preview without writing; * pass `{ backupTo: path }` to persist a pre-migration snapshot. * @returns Migration result (or preview if dryRun) * * Safety: with `backupTo`, a hard-link snapshot of the current generation * is persisted BEFORE any transform runs (the Db API's `persist()`), and * `brain.restore(path, { confirm: true })` brings it back wholesale. * Transforms are also idempotent (they return `null` when already * applied), so re-running migrations is safe. * * @example * ```typescript * // Preview what would change * const preview = await brain.migrate({ dryRun: true }) * console.log(preview.affectedEntities) * * // Apply migrations with a restore point * const result = await brain.migrate({ backupTo: '/backups/pre-migration' }) * console.log(result.entitiesModified, result.backupPath) * ``` */ async migrate(options?: MigrateOptions): Promise { await this.ensureInitialized() const runner = this._pendingMigrationRunner || new MigrationRunner(this.storage) if (options?.dryRun) { return runner.preview() } return this.migrateInternal(runner, options) } /** * Check for pending migrations during init(). * Runs inline for small datasets if autoMigrate is enabled, * otherwise logs a warning. */ private async checkMigrations(): Promise { const runner = new MigrationRunner(this.storage) if (!(await runner.hasPendingMigrations())) { return } const count = await runner.pendingCount() if (this.config.autoMigrate) { // Quick entity count check to decide inline vs deferred const probe = await this.storage.getNouns({ pagination: { limit: 1 } }) const totalEstimate = probe.totalCount ?? (probe.hasMore ? 10001 : probe.items.length) if (totalEstimate < 10000) { // Small dataset — migrate inline during init await this.migrateInternal(runner) } else { // Large dataset — defer to explicit brain.migrate() call this._pendingMigrationRunner = runner if (!this.config.silent) { console.log(`[brainy] ${count} pending migration(s) detected. Call brain.migrate() to apply (dataset too large for inline migration).`) } } } else { if (!this.config.silent) { console.log(`[brainy] ${count} pending migration(s) available. Set autoMigrate: true or call brain.migrate() to apply.`) } } } /** * Internal: run pending migrations and rebuild the metadata index when * anything changed. With `backupTo`, a hard-link snapshot of the current * generation is persisted before any transform runs — the Db-API * replacement for the pre-8.0 automatic backup branches. Transforms are * idempotent (return `null` when already applied), so re-runs are safe. */ private async migrateInternal(runner: MigrationRunner, options?: MigrateOptions): Promise { let backupPath: string | null = null if (options?.backupTo) { const pin = this.now() try { await pin.persist(options.backupTo) backupPath = options.backupTo } finally { await pin.release() } } const runResult = await runner.run({ onProgress: options?.onProgress, maxErrors: options?.maxErrors }) // Rebuild MetadataIndex if any entities were modified if (runResult.entitiesModified > 0) { await this.metadataIndex.rebuild() } // Clear deferred runner this._pendingMigrationRunner = undefined return { backupPath, ...runResult } } // ============= 8.0 DB API — GENERATIONAL MVCC ============= // // Datomic-style immutable database values over the generational record // layer (src/db/). `now()` pins the current generation in O(1); // `transact()` commits a declarative batch atomically as exactly one // generation; `asOf()` opens past generations, timestamps, or persisted // snapshots; `compactHistory()` reclaims unpinned history. The `Db` value // type lives in src/db/db.ts; this region is the brain-side host: // pin/release plumbing (storage + versioned index providers), record // materialization, the transact planner, and snapshot/restore. /** * @description The store's current generation — a monotonic u64 watermark * bumped once per committed `transact()` batch and once per * single-operation write batch (`add`/`update`/`remove`/`relate`/…). * Persisted in `_system/generation.json`; never reused, including across * restarts and `restore()`. * * @returns The current generation number. * @throws Error when called before `init()` completes. * @example * const before = brain.generation() * await brain.transact([{ op: 'add', type: NounType.Document, data: 'x', subtype: 'note' }]) * brain.generation() // before + 1 */ generation(): number { this.assertGenerationStoreReady('generation') return this.generationStore.generation() } /** * @description The data-layer + derived-index format this brain instance runs * as — the SYNCHRONOUS half of the 7.x → 8.0 version handshake. Returns the * compiled {@link CURRENT_DATA_FORMAT} / {@link EXPECTED_INDEX_EPOCH} * constants (the single source of truth shared with the native provider, in * `src/storage/brainFormat.ts`): after `init()` the brain has reconciled any * on-disk drift and IS running the current format, so this reports what it * runs as, not necessarily the (possibly older) marker it opened. * * This is a pure in-memory constant read — no `await`, no storage I/O — by * design: a native metadata/index provider calls it synchronously at its own * `init()` (which runs during this brain's provider-construction phase, after * the marker has been loaded) to confirm "running data-format X, index epoch * N" before binding its native readers. * * The on-disk marker (`_system/brain-format.json`) is reconciled separately: * on open Brainy compares the on-disk `indexEpoch` against * {@link EXPECTED_INDEX_EPOCH}; a mismatch or an absent marker rebuilds the * derived indexes and re-stamps the marker (see `rebuildIndexesIfNeeded`). * * @returns `{ dataFormat, indexEpoch }` for the running build. * @example * const { dataFormat, indexEpoch } = brain.formatInfo() * // dataFormat === '8.0', indexEpoch === 1 */ formatInfo(): BrainFormat { return { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } } /** * @description Stamp `_system/brain-format.json` with this build's * `{ dataFormat, indexEpoch }` ({@link CURRENT_DATA_FORMAT} / * {@link EXPECTED_INDEX_EPOCH}). A native provider calls this once its * background index migration (the no-freeze path, where brainy DEFERRED the * rebuild because the provider's `isMigrating()` was true) has * verified-and-swapped all derived indexes — brainy authors `dataFormat`, the * provider never does. The marker is the single switch that declares the * on-disk derived indexes current for this build's epoch, so it is advanced * only after the indexes it certifies are in place. Unconditional and * idempotent (writes the same two compiled constants every call); a no-op for * read-only brains, which never author markers. * @returns Resolves once the marker is durable on disk. * @example * // cor, after its build-new→verify→swap completes: * await brain.stampBrainFormat() */ async stampBrainFormat(): Promise { if (this.isReadOnly) return await writeBrainFormat(this.storage) this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } this._indexEpochStale = false // The upgrade has verified + stamped — drop the pre-upgrade backup (no-op if // none was taken, e.g. a non-migrating stamp of a fresh brain's initial marker). await this.removeMigrationBackupSafe() } /** * @description Open a sequential scan over the generation FACT LOG — the * append-only record of every committed generation as an AFTER-IMAGE fact * (what each touched entity/relationship became, or a body-less tombstone * for a removal). The scan is the streaming substrate for index heals and * incremental replays: one sequential read in commit order replaces a * per-entity directory walk. The handle carries heal-narration telemetry * (`headGeneration` / `segmentCount` / `approxFactCount` up front; ordered * batches each stamped with their generation range, byte size, and segment; * a `summary()` cross-check after iteration). A detected gap or damaged * segment ABORTS the scan loudly — never a silent skip. * * Returns `null` when this store hosts no fact log: the storage adapter * lacks the binary append primitives, or the brain predates the fact log * (its history began before dual-write — facts exist only from the first * write after upgrade; callers fall back to the canonical enumeration walk). * * @param options - `fromGeneration`/`toGeneration` bound the scan (inclusive * both ends); `kinds` filters ops to `'noun'`/`'verb'`; `batchSize` caps * facts per yielded batch (default 256). * @returns The scan handle, or `null` when no fact log exists. * @example * const scan = brain.scanFacts({ fromGeneration: 1 }) * if (scan) { * for await (const batch of scan.batches()) { * for (const fact of batch.facts) { * // fact.ops: [{ kind, id, record | null (tombstone) }, ...] * } * } * } */ scanFacts(options?: { fromGeneration?: number toGeneration?: number kinds?: Array<'noun' | 'verb'> batchSize?: number }): FactScanHandle | null { const factLog = this.generationStore?.getFactLog() return factLog ? factLog.scanFacts(options) : null } /** * @description The immutable, sealed fact-log segment files covering * `fromGeneration` — the zero-copy handoff for consumers that map segment * files directly instead of streaming {@link scanFacts} batches. The * append-mutable TAIL segment is deliberately excluded (read it via * `scanFacts`). Paths are storage-root-relative. Empty when no fact log * exists or nothing is sealed yet. */ factSegmentPaths(options?: { fromGeneration?: number }): string[] { return this.generationStore?.getFactLog()?.segmentPaths(options) ?? [] } /** * @description Read the reified transaction log — one entry per committed * generation, carrying the committed generation, the commit timestamp, and * the `meta` the transaction was submitted with. Entries are returned newest * first. Under Model-B EVERY write is its own generation, so single-operation * writes (`add`/`update`/`remove`/`relate`/…) appear here too (without `meta` * — transaction metadata is a `transact()`-only concept). `compactHistory()` * never rewrites the log — entries may reference generations whose * record-sets were already reclaimed. * * @param options - `from`/`to` (generation number or `Date`) bound the window * to commits in `[from, to]` INCLUSIVE on both ends — a log window names * commits, in deliberate contrast to `since()`'s EXCLUSIVE lower bound. * `limit` caps the result and is applied LAST (after windowing), newest * first. All omitted = the whole log. `from`/`to` resolve through the same * reachability gate as `asOf()`. * @returns Committed transaction entries, newest first. * @throws RangeError when the resolved `from` generation is after `to`. * @example * await brain.transact([{ op: 'update', id, metadata: { v: 2 } }], { meta: { author: 'sync-job' } }) * const [latest] = await brain.transactionLog({ limit: 1 }) * latest.meta // { author: 'sync-job' } * // Commits in a window, newest first: * const window = await brain.transactionLog({ from: 10, to: 20 }) */ async transactionLog(options?: { from?: number | Date to?: number | Date limit?: number }): Promise { await this.ensureInitialized() let entries = await this.generationStore.txLog() // oldest first if (options?.from !== undefined || options?.to !== undefined) { const fromGen = options?.from !== undefined ? (await this.resolveAsOfGeneration(options.from)).generation : 0 const toGen = options?.to !== undefined ? (await this.resolveAsOfGeneration(options.to)).generation : this.generationStore.generation() if (fromGen > toGen) { throw new RangeError( `transactionLog(): resolved from-generation ${fromGen} is after to-generation ${toGen}` ) } // Inclusive on BOTH ends (a log window names the commits it spans). entries = entries.filter((e) => e.generation >= fromGen && e.generation <= toGen) } entries.reverse() // newest first return options?.limit !== undefined ? entries.slice(0, options.limit) : entries } /** * @description Pin the CURRENT generation and return an immutable `Db` * view of it — O(1), no I/O. The returned view keeps serving the FULL * query surface at exactly this state no matter what commits afterwards: * `get()` and metadata-level `find()`/`related()` resolve from immutable * generation records at no extra cost, while index-accelerated queries * (semantic/vector search, graph traversal, cursors, aggregation) are * served by an at-generation index materialization built lazily on first * use — O(n at G) time and memory once per `Db`, freed on `release()` * (a native `VersionedIndexProvider` serves the same reads from retained * segments without rebuild). * * Call `db.release()` when done — pins gate `compactHistory()`. A * `FinalizationRegistry` backstop releases leaked pins at GC time, but * explicit release is what makes compaction deterministic. * * @returns A `Db` pinned at the current generation. * @throws Error when called before `init()` completes (pinning is * synchronous, so it cannot await initialization). * @example * const db = brain.now() * await brain.transact([{ op: 'update', id, metadata: { v: 2 } }]) * await db.get(id) // still sees v: 1 * await brain.get(id) // sees v: 2 * await db.release() */ now(): Db { this.assertGenerationStoreReady('now') return this.createPinnedDb({ generation: this.generationStore.generation(), timestamp: Date.now() }) } /** * @description Serialize part or all of the brain into a portable `PortableGraph` * document — sugar for `brain.now().export(selector, options)` (the current * generation). For a past generation use `(await brain.asOf(g)).export(...)`; * for a speculative state use `brain.now().with(ops).export(...)`. * * Restore a `PortableGraph` with {@link Brainy.import} (it dispatches a backup to the * graph round-trip; a file/buffer to ingestion). Distinct from `db.persist()` * (native whole-brain snapshot with generation history). * * @param selector - WHAT to export (omit for the whole brain). See {@link ExportSelector}. * @param options - HOW to export (vectors / VFS bytes / edge policy). See {@link ExportOptions}. * @returns A versioned, portable `PortableGraph` document. * @example * const graph = await brain.export({ ids }, { includeVectors: true }) */ async export(selector: ExportSelector = {}, options: ExportOptions = {}): Promise { return this.now().export(selector, options) } /** * @description Execute a declarative operation batch atomically: either * every operation applies and the store advances exactly ONE generation, * or none apply and the store is byte-identical to its pre-transaction * state. The semantics of each operation mirror the corresponding * single-operation method (`add`/`update`/`remove`/`relate`/`unrelate`), * including validation, subtype enforcement, per-entity `ifRev` CAS, * relationship deduplication, and delete cascades. Operations may * reference ids created by earlier operations of the same batch. * * Durability protocol (see `src/db/generationStore.ts`): before-images of * every touched id are staged and fsynced, the batch executes through the * TransactionManager, and the atomic rename of `_system/manifest.json` is * the commit point — a crash anywhere before the rename is rolled back to * the exact pre-transaction bytes on the next open. * * Isolation: concurrent `transact()` calls commit serially * (snapshot-isolated batches). For strict lost-update protection across a * read-modify-write cycle, pass `ifAtGeneration` (whole-store CAS) or use * per-entity `ifRev` on update operations. * * @param ops - Declarative operations: `{ op: 'add', ... }`, * `{ op: 'update', ... }`, `{ op: 'remove', id }`, * `{ op: 'relate', ... }`, `{ op: 'unrelate', id }`. * @param options - `meta` (reified transaction metadata, recorded in * `_system/tx-log.jsonl`) and `ifAtGeneration` (whole-store CAS). * @returns A `Db` pinned at the freshly committed generation, carrying a * `receipt` with the resolved id per input operation. * @throws GenerationConflictError when `ifAtGeneration` does not match the * store's current generation. * @throws RevisionConflictError when an update operation's `ifRev` does * not match the entity's persisted revision (whole batch rejected). * @example * const db = await brain.transact([ * { op: 'add', id: aId, type: NounType.Person, subtype: 'employee', data: 'Ada' }, * { op: 'add', id: bId, type: NounType.Project, subtype: 'milestone', data: 'Apollo' }, * { op: 'relate', from: aId, to: bId, type: VerbType.ParticipatesIn, subtype: 'assignment' } * ], { meta: { author: 'import-job-42' } }) * db.receipt.ids // [aId, bId, relationshipId] */ async transact(ops: TxOperation[], options?: TransactOptions): Promise> { this.assertWritable('transact') await this.ensureInitialized() if (!Array.isArray(ops) || ops.length === 0) { throw new Error('transact(): provide a non-empty array of transaction operations') } // Commit any buffered single-op generations FIRST so this transaction's // generation lands strictly after them and the committed-generation // sequence stays gap-free (single-ops reserve generations eagerly). await this.generationStore.flushPendingSingleOps() // Early CAS check — avoids expensive planning (embedding) on a stale // expectation. The generation store re-checks authoritatively under the // commit mutex; this one is purely a fast-fail. if ( options?.ifAtGeneration !== undefined && options.ifAtGeneration !== this.generationStore.generation() ) { throw new GenerationConflictError( options.ifAtGeneration, this.generationStore.generation() ) } const plan = await this.planTransact(ops) // Authoritative per-op ifRev CAS, run by the generation store UNDER the // commit mutex against the just-read before-images — atomic with the // apply (the planning-time checks above are fast-fails that can // interleave with concurrent writers). Any conflict rejects the WHOLE // batch before anything is staged or applied. Sequencing rules: // - runningRev tracks multiple updates to the SAME entity in one batch // (each op CASes against — and bumps — its predecessor's result). // - An entity the batch itself creates (null before-image, forward-ref // add+update) keeps its plan-sequenced stamps: no concurrent writer // can have moved a rev that did not exist, so plan-time was exact. const casPrecommit = (before: CommitBeforeImages): void => { const runningRev = new Map() for (const cas of plan.casUpdates) { // An id this batch adds owns its revision baseline (the add restamps // `_rev: 1` even on overwrite) — plan-time sequencing is exact, no // concurrent writer can move a baseline the batch itself sets. if (plan.createdNouns.has(cas.id)) { continue } const beforeMeta = before.nouns.get(cas.id)?.metadata as | { _rev?: unknown } | null | undefined if (!beforeMeta && !runningRev.has(cas.id)) { // A pre-existing entity a concurrent remove() deleted after // planning: a CAS caller gets the truth; a plain update keeps its // last-writer-wins re-create semantics (plan-stamped rev). if (typeof cas.ifRev === 'number') { throw new EntityNotFoundError(cas.id) } continue } const base = runningRev.get(cas.id) ?? (typeof beforeMeta?._rev === 'number' ? beforeMeta._rev : 1) if (typeof cas.ifRev === 'number' && cas.ifRev !== base) { throw new RevisionConflictError(cas.id, cas.ifRev, base) } const next = base + 1 cas.updatedMetadata._rev = next runningRev.set(cas.id, next) } } let generation: number let timestamp: number try { ;({ generation, timestamp } = await this.generationStore.commitTransaction({ touched: { nouns: plan.touchedNouns, verbs: plan.touchedVerbs }, meta: options?.meta, ifAtGeneration: options?.ifAtGeneration, precommit: casPrecommit, execute: async () => { await this.transactionManager.executeTransaction( async (tx) => { for (const operation of plan.operations) { tx.addOperation(operation) } }, // Budget scales with the batch (or the caller's explicit // timeoutMs): a flat 30s cap silently limited honest bulk work // to ~15 ops on network disks (~2s/op measured in the field). { timeout: transactTimeoutBudget(plan.operations.length, options?.timeoutMs) } ) } })) } catch (err) { // A batch whose rollback failed and left canonical records unreconciled // quarantines writes until repairIndex() reconciles and lifts it. if (err instanceof StoreInconsistentError) this.storeInconsistency = err throw err } // Aggregation-index maintenance: derived data, applied after the commit // point — exactly where the single-operation methods apply it. for (const hook of plan.postCommit) { hook() } // Change feed: the batch's events share its single committed generation. // A rejected batch throws at commitTransaction and never reaches here. this.emitCommitted(plan.changeEvents, undefined, generation, timestamp) const receipt: TransactReceipt = { generation, timestamp, ids: plan.ids } return this.createPinnedDb({ generation, timestamp, receipt }) } /** * @description Open an immutable `Db` view of PAST state: * * - **`number`** — a generation: pins it on this store; reads resolve * through the generational record layer. Under Model-B every write is its * own generation, so single-operation writes are individually addressable * too (a pin always freezes against later single-op writes). * - **`Date`** — a wall-clock instant: resolved via the transaction log to * the newest generation committed at or before it, then pinned as above. * - **`string`** — a snapshot directory previously produced by * `db.persist(path)`: opened as a self-contained read-only store * (equivalent to {@link Brainy.load}) with the FULL query surface, * including vector search. * * Release the returned `Db` when done. * * @param target - Generation number, `Date`, or snapshot directory path. * @param options - `exclusive: true` pins the generation immediately BEFORE the * one `target` resolves to (strict-before): `asOf(g, { exclusive: true })` is * the state at `g − 1`; strict-before the first commit pins generation 0 (the * empty pre-commit state), never a `RangeError`. Ignored for the snapshot * (string) form. * @returns A `Db` pinned at the resolved state. * @throws GenerationCompactedError when the generation's records were * reclaimed by `compactHistory()`. * @throws RangeError for negative or future generations. * @example * const yesterday = await brain.asOf(new Date(Date.now() - 86_400_000)) * const atGen42 = await brain.asOf(42) * const beforeGen42 = await brain.asOf(42, { exclusive: true }) // state at gen 41 * const fromSnapshot = await brain.asOf('/backups/2026-06-01') */ async asOf( target: number | Date | string, options?: { exclusive?: boolean } ): Promise> { await this.ensureInitialized() if (typeof target === 'string') { const stat = await fs.promises.stat(target).catch(() => null) if (!stat || !stat.isDirectory()) { throw new Error( `asOf(): '${target}' is not a snapshot directory. Pass a generation number, ` + `a Date, or a directory created by db.persist(path).` ) } return Brainy.load(target) } const { generation, timestamp } = await this.resolveAsOfGeneration(target, options) return this.createPinnedDb({ generation, timestamp }) } /** * @description The shared `generation | Date` → `{ generation, timestamp }` * resolver used by every gen/Date-accepting temporal verb (`asOf`, `since`, * `diff`, `transactionLog`) so reachability, `RangeError`, and `Date` semantics * stay identical across all of them. The inclusive path is byte-for-byte the * pre-extraction `asOf` logic; `exclusive` pins the generation immediately * before the resolved one (clamped at 0, never a `RangeError`). All resolved * generations pass through `assertReachable` (so a compacted target throws * {@link GenerationCompactedError}); `history()` deliberately does NOT use this * resolver — it truncates below the horizon rather than throwing. * @param target - A generation number or a `Date`. * @param options - `exclusive: true` for strict-before (resolved − 1). */ private async resolveAsOfGeneration( target: number | Date, options?: { exclusive?: boolean } ): Promise<{ generation: number; timestamp: number }> { const exclusive = options?.exclusive === true let generation: number let timestamp: number if (target instanceof Date) { const resolved = await this.generationStore.resolveTimestamp(target.getTime()) if (exclusive) { generation = Math.max(0, resolved.generation - 1) this.generationStore.assertReachable(generation) timestamp = (await this.generationStore.commitTimestampAtOrBefore(generation)) ?? target.getTime() } else { this.generationStore.assertReachable(resolved.generation) generation = resolved.generation timestamp = resolved.entry?.timestamp ?? target.getTime() } } else { generation = exclusive ? Math.max(0, target - 1) : target this.generationStore.assertReachable(generation) timestamp = (await this.generationStore.commitTimestampAtOrBefore(generation)) ?? Date.now() } return { generation, timestamp } } /** * @description Compare two states of the brain and report what CHANGED between * them — entities/relationships `added`, `removed`, or `modified`, each split by * kind. Unlike a raw touched-id list, `diff` EARNS its name: the candidate set is * ONLY the ids a committed transaction touched in the interval, and each is then * resolved at BOTH endpoints and classified by existence and a stable value * comparison. An id touched within the interval but whose endpoint states are * identical (e.g. changed then reverted) appears in NONE of the three buckets. * * Orientation is `a → b`: `added` = exists at `b` not `a`; `removed` = exists at * `a` not `b`; `modified` = exists at both with a changed stored value * (key-order-insensitive). Endpoints may be passed in either order. * * @param a - The "before" state: a generation number, a `Date`, or a `Db`. * @param b - The "after" state: a generation number, a `Date`, or a `Db`. * @returns A {@link DiffResult} (each bucket's ids sorted). * @throws GenerationCompactedError when a number/`Date` endpoint is below the horizon. * @example * const before = brain.generation() * await brain.transact(ops) * const d = await brain.diff(before, brain.generation()) * d.added.nouns // ids created by the transaction * d.modified.nouns // ids whose stored value actually changed */ async diff(a: number | Date | Db, b: number | Date | Db): Promise { await this.ensureInitialized() const gA = await this.resolveDiffEndpoint(a) const gB = await this.resolveDiffEndpoint(b) const result: DiffResult = { added: { nouns: [], verbs: [] }, removed: { nouns: [], verbs: [] }, modified: { nouns: [], verbs: [] }, fromGeneration: gA, toGeneration: gB } // Candidate set: ONLY the ids touched in the interval (never all ids) — this // is what lets diff classify rather than dump raw changed-ids. const gLow = Math.min(gA, gB) const gHigh = Math.max(gA, gB) if (gLow === gHigh) return result // same state → no changes const changed = await this.generationStore.changedBetween(gLow, gHigh) for (const kind of ['noun', 'verb'] as const) { const ids = kind === 'noun' ? changed.nouns : changed.verbs for (const id of ids) { const stateA = await this.recordStateAt(kind, id, gA) const stateB = await this.recordStateAt(kind, id, gB) const bucket = (name: 'added' | 'removed' | 'modified') => kind === 'noun' ? result[name].nouns : result[name].verbs if (!stateA.exists && stateB.exists) bucket('added').push(id) else if (stateA.exists && !stateB.exists) bucket('removed').push(id) else if ( stateA.exists && stateB.exists && !stableDeepEqual(stateA.metadata, stateB.metadata) ) { bucket('modified').push(id) } // both absent, or both exist and identical → in NO bucket (earns the name) } } for (const name of ['added', 'removed', 'modified'] as const) { result[name].nouns.sort() result[name].verbs.sort() } return result } /** * @description Every distinct version of ONE entity or relationship over a * generation range, oldest → newest — the per-id companion to `asOf()`. Each * {@link HistoryVersion} ties to the trusted `asOf()` path: its `value` equals * `(await brain.asOf(version.generation)).get(id)`. Consecutive identical states * are collapsed, and a leading "did not exist yet" baseline is dropped, so the * first version is the id's first real state in the range and a `null` value * marks a removal. * * Range defaults to `[horizon, currentGeneration]`. Unlike `diff`/`since`, a * `from` below the compaction horizon is TRUNCATED to the horizon, not thrown — * history is best-effort over the records that survive. * * @param id - The entity or relationship id (kind auto-detected). * @param options - `from`/`to` (generation number or `Date`) bound the range. * @returns An {@link EntityHistory} (`versions` oldest first; empty if the id is * unknown over the range). * @throws Error if `id` resolves in BOTH the entity and relationship spaces. * @example * const h = await brain.history(invoiceId) * h.versions.map(v => v.value?.metadata?.status) // the status at each change */ async history( id: string, options?: { from?: number | Date; to?: number | Date } ): Promise> { await this.ensureInitialized() const store = this.generationStore const horizon = store.horizon() const current = store.generation() // history TRUNCATES below the horizon (never throws): resolve raw, then clamp. const rawFrom = options?.from !== undefined ? await this.resolveRawGeneration(options.from) : horizon const rawTo = options?.to !== undefined ? await this.resolveRawGeneration(options.to) : current const fromGen = Math.min(Math.max(rawFrom, horizon), current) const toGen = Math.min(Math.max(rawTo, 0), current) const kind = await this.detectIdKind(id, horizon, current) if (kind === null || fromGen > toGen) { return { id, kind: kind ?? 'noun', versions: [] } } // Snapshot generations: the range start (state as-of fromGen) plus every // committed change-point in (fromGen, toGen]. const touches = await store.generationsTouching(kind, id, fromGen, toGen) const snapshotGens = [fromGen, ...touches] const versions: HistoryVersion[] = [] let prev: { exists: boolean; metadata: any } | null = null for (const g of snapshotGens) { const state = await this.recordStateAt(kind, id, g) if ( prev !== null && prev.exists === state.exists && stableDeepEqual(prev.metadata, state.metadata) ) { continue // collapse consecutive identical states } prev = { exists: state.exists, metadata: state.metadata } const timestamp = (await store.commitTimestampAtOrBefore(g)) ?? 0 let value: Entity | Relation | null = null if (state.exists) { value = kind === 'noun' ? await this.entityFromGenerationRecord( id, { metadata: state.metadata, vector: state.vector }, false ) : this.relationFromGenerationRecord(id, { metadata: state.metadata, vector: state.vector }) } versions.push({ generation: g, timestamp, value }) } // Drop a leading "did not exist yet" baseline so history starts at the id's // first real state (internal/trailing nulls = real removals stay). while (versions.length > 0 && versions[0].value === null) versions.shift() return { id, kind, versions } } /** Resolve a `Db | generation | Date` diff endpoint to a concrete generation. */ private async resolveDiffEndpoint(endpoint: number | Date | Db): Promise { if (endpoint instanceof Db) { if (!endpoint.belongsToStore(this.generationStore)) { throw new Error('diff(): a Db endpoint must come from this same Brainy instance') } return endpoint.generation } return (await this.resolveAsOfGeneration(endpoint)).generation } /** * Resolve a `generation | Date` to a raw generation number WITHOUT the * reachability assertion — for `history()`, which truncates below the horizon * rather than throwing. */ private async resolveRawGeneration(target: number | Date): Promise { if (target instanceof Date) { return (await this.generationStore.resolveTimestamp(target.getTime())).generation } return target } /** * The stored state of `id` at pinned generation `gen`, normalizing `resolveAt`'s * three sources to `{ exists, metadata, vector }`. A `'current'` source means the * live storage state IS the state at `gen`, so it reads the live metadata. */ private async recordStateAt( kind: 'noun' | 'verb', id: string, gen: number ): Promise<{ exists: boolean; metadata: any; vector: any }> { const r = await this.generationStore.resolveAt(kind, id, gen) if (r.source === 'absent') return { exists: false, metadata: null, vector: null } if (r.source === 'record') return { exists: true, metadata: r.metadata, vector: r.vector } const metadata = kind === 'noun' ? await this.storage.getNounMetadata(id) : await this.storage.getVerbMetadata(id) return { exists: metadata !== null, metadata, vector: null } } /** * Auto-detect whether `id` is an entity (`'noun'`) or relationship (`'verb'`) by * its presence live and in the generation deltas over `(fromGen, toGen]`. Returns * `null` for an unknown id; throws on the (UUID-space) collision where the id * resolves in both spaces. */ private async detectIdKind( id: string, fromGen: number, toGen: number ): Promise<'noun' | 'verb' | null> { const liveNoun = (await this.storage.getNounMetadata(id)) !== null const liveVerb = (await this.storage.getVerbMetadata(id)) !== null const nounTouched = (await this.generationStore.generationsTouching('noun', id, fromGen, toGen)).length > 0 const verbTouched = (await this.generationStore.generationsTouching('verb', id, fromGen, toGen)).length > 0 const isNoun = liveNoun || nounTouched const isVerb = liveVerb || verbTouched if (isNoun && isVerb) { throw new Error( `history(): id '${id}' resolves in BOTH the entity and relationship spaces — ` + `ambiguous (should not happen with UUID ids).` ) } if (isNoun) return 'noun' if (isVerb) return 'verb' return null } /** * @description Reclaim the immutable generation record-sets that no * retention rule and no live pin protects. Records are what serve * historical reads (`asOf()`, pinned `Db` values); compaction trades that * history for disk space. Generations at or below the resulting horizon * become unreachable (`asOf()` throws `GenerationCompactedError`). * * Safety invariant: a record-set is never removed while any live `Db` pin * could need it — pinned reads stay correct across compaction, always. * Note that `transact()` returns a PINNED `Db`: release views you do not * keep (the GC backstop eventually releases leaked ones, but until then * compaction retains the history they protect). * * @param options - Explicit retention CAPS (`maxGenerations` / `maxAge` / * `maxBytes`). Reclaim the oldest unpinned generations while ANY supplied * cap is exceeded. Neither = reclaim everything unpinned. * @returns Count of reclaimed record-sets and the new horizon. * @example * await brain.compactHistory({ maxGenerations: 100, maxAge: 7 * 86_400_000 }) */ async compactHistory(options?: CompactHistoryOptions): Promise { this.assertWritable('compactHistory') await this.ensureInitialized() // Persist buffered single-op generations first so compaction reasons over a // complete on-disk history (the pending tier is never itself reclaimable). await this.generationStore.flushPendingSingleOps() return this.generationStore.compact(options) } /** * @description Drive the adaptive retention byte budget at runtime — the * settable input a machine-level coordinator (e.g. cor's `ResourceManager`, * which fair-shares one box-wide history budget across co-located instances) * pushes whenever the box's free-resource picture changes. Takes effect at the * next auto-compaction (flush/close). Only meaningful while `retention` is * adaptive (unset or `'adaptive'`); ignored under `'all'` or explicit caps. * @param bytes - The new per-brain history byte budget (non-negative). * @example brain.setRetentionBudget(2 * 1024 ** 3) // 2 GiB for this brain */ setRetentionBudget(bytes: number): void { if (!Number.isFinite(bytes) || bytes < 0) { throw new RangeError(`setRetentionBudget(): bytes must be a non-negative finite number (got ${bytes})`) } this._retentionBudgetBytes = bytes } /** * @description Resolve the effective `retention` policy from `config.retention`. * The single read site for the policy {@link autoCompactHistory} applies. * * - `'all'` → `{ mode: 'all' }` (never reclaim history). * - unset / `'adaptive'` → `{ mode: 'adaptive' }` (budget-driven; the budget * is `setRetentionBudget()`/`budgetBytes` when set, else a local probe). * - `{ … }` → `{ mode: 'explicit', maxGenerations?, maxAge?, maxBytes? }`. * * `autoCompact` defaults to `true` in every mode. * @returns The normalized retention policy. */ private resolveRetentionPolicy(): { mode: 'all' | 'adaptive' | 'explicit' maxGenerations?: number maxAge?: number maxBytes?: number budgetBytes?: number autoCompact: boolean } { const retention = this.config.retention if (retention === 'all') { return { mode: 'all', autoCompact: true } } if (retention === undefined || retention === 'adaptive') { return { mode: 'adaptive', autoCompact: true } } return { mode: 'explicit', maxGenerations: retention.maxGenerations, maxAge: retention.maxAge, maxBytes: retention.maxBytes, budgetBytes: retention.budgetBytes, autoCompact: retention.autoCompact ?? true } } /** * @description Compute the adaptive history byte budget for this brain: the * coordinator-driven value when present (`setRetentionBudget()` / * `config.retention.budgetBytes` — e.g. cor's fair-shared box budget), else a * conservative LOCAL probe of free resources. Standalone brainy must not be * cor-dependent, so the fallback uses `os.freemem()` (and is intentionally * generous — a single instance owning the box can keep a lot of history). * @returns The byte budget; `Infinity` only if no signal is available. */ private adaptiveHistoryBudgetBytes(driven?: number): number { if (driven !== undefined) return driven if (this._retentionBudgetBytes !== undefined) return this._retentionBudgetBytes // Local single-instance probe: allot a quarter of currently-free RAM to // history. Bounded, zero-config, and never reclaims pinned generations. try { const free = os.freemem() if (Number.isFinite(free) && free > 0) return Math.floor(free / 4) } catch { // os unavailable (exotic runtime) — fall through to unbounded. } return Infinity } /** * @description Run history compaction under the resolved `retention` policy * when `autoCompact` is on (the default). Invoked from `flush()` and * `close()` so generational record-sets cannot accumulate unbounded across a * long-lived writer's lifetime. * * - `'all'` → returns without reclaiming (index compaction for speed still * runs elsewhere; history is decoupled and kept). * - `'adaptive'` → reclaim oldest-unpinned history down to the byte budget. * - `'explicit'` → apply the supplied `maxGenerations`/`maxAge`/`maxBytes` caps. * * Read-only instances and an explicit `autoCompact: false` skip silently. * Pinned generations are never reclaimed ({@link GenerationStore.compact}). * Failures are logged and swallowed — compaction is housekeeping and must * never fail a flush or a clean shutdown. */ private async autoCompactHistory(): Promise { // Nothing to compact on a read-only instance or before init wired up the // generation store (close() can run defensively on an uninitialized brain). if (this.isReadOnly || !this.generationStore) { return } const policy = this.resolveRetentionPolicy() if (!policy.autoCompact || policy.mode === 'all') { return } try { if (policy.mode === 'adaptive') { const budget = this.adaptiveHistoryBudgetBytes(policy.budgetBytes) if (budget === Infinity) return // no pressure signal → keep everything this pass await this.generationStore.compact({ maxBytes: budget }) } else { await this.generationStore.compact({ maxGenerations: policy.maxGenerations, maxAge: policy.maxAge, maxBytes: policy.maxBytes }) } } catch (error) { console.warn( `Auto-compaction of generational history failed (non-fatal): ${ error instanceof Error ? error.message : String(error) }` ) } } /** * @description Replace this store's ENTIRE state from a snapshot directory * previously produced by `db.persist(path)`. Destructive: current * entities, relationships, indexes, and history records are all replaced * by the snapshot's. The generation counter is floored at its pre-restore * value, so generation numbers observed before the restore are never * reissued. * * Live `Db` values pinned before a restore are NOT remapped — their * snapshot-isolation guarantee does not survive a wholesale state * replacement. Release them first; a warning is logged when live pins * exist. * * NON-DESTRUCTIVE STAGING + SPARSE-AWARE: the snapshot is copied into a * staging area BEFORE any live data is touched (all-zero blocks stay * holes, so a sparse mmap store restores at its true allocated size — not * its apparent size). Any copy failure, ENOSPC included, removes only the * staging and throws; the live store is untouched. Only after the copy * succeeds does an atomic per-entry swap move it into place; a crash * mid-swap completes FORWARD on the next open. * * @param path - Snapshot directory to restore from. * @param options - Must be `{ confirm: true }` — an explicit acknowledgment * that current state is destroyed. * @throws Error when `confirm` is not `true`, or `path` is not a snapshot * directory. * @example * await brain.restore('/backups/2026-06-01', { confirm: true }) */ async restore(path: string, options: { confirm: boolean }): Promise { this.assertWritable('restore') await this.ensureInitialized() if (options?.confirm !== true) { throw new Error( `restore() replaces the store's entire current state with the snapshot at ` + `'${path}'. Pass { confirm: true } to proceed.` ) } const pinCount = this.generationStore.activePinCount() if (pinCount > 0) { prodLog.warn( `[Brainy] restore() with ${pinCount} live Db pin(s): pinned views do not ` + `survive a restore — release Db values before restoring.` ) } const floorGeneration = this.generationStore.generation() await this.storage.restoreFromDirectory(path) await this.generationStore.reopenAfterRestore(floorGeneration) // If the entity-id mapper is a NATIVE provider with a `rebuild()`, reload it // from the restored snapshot FIRST. The graph index resolves every verb // endpoint (sourceId/targetId → stable int) through this mapper, so a native // adjacency (keyed on those ints) needs the mapper to reflect the snapshot's // assignments BEFORE graphIndex.rebuild() — otherwise edges resolve to // stale/missing ints and are silently dropped (CTX-BR-RESTORE-REBUILD). The // JS mapper has no `rebuild()` and needs no reload: MetadataIndex.rebuild() // re-derives it from the restored entities via append-only getOrAssign, // consistently with the bitmaps it builds. await this.hydrateIdMapperForGraphRebuild() // Every in-memory index is now stale — rebuild all three from the restored // canonical records (each rebuild clears its own state first). The graph // rebuild resolves endpoints via the (now-reloaded, for native) mapper. await Promise.all([ this.metadataIndex.rebuild(), this.index.rebuild(), this.graphIndex.rebuild() ]) // Change feed: a restore is a wholesale raw-state replacement, not a // per-record commit — one store-level event tells subscribers to refetch. this._changeFeed.emit([ { kind: 'store', op: 'restore', timestamp: Date.now() } ]) } /** * @description Construct AND initialize a `Brainy` instance in one call — * `new Brainy(config)` + `await init()`. * * @param config - Standard `BrainyConfig`. * @returns The initialized instance. * @example * const brain = await Brainy.open({ storage: { type: 'filesystem', path: './data' } }) */ static async open(config?: BrainyConfig): Promise> { const brain = new Brainy(config) await brain.init() return brain } /** * @description Open a persisted snapshot directory (from `db.persist()`) * as a self-contained READ-ONLY store and return it as a `Db`. The * snapshot's indexes load from its own files, so the full query surface — * including vector search — works at the snapshot's generation. Releasing * the returned `Db` closes the underlying read-only instance. * * @param path - Snapshot directory produced by `db.persist(path)`. * @returns A `Db` over the snapshot (release it to free resources). * @example * const db = await Brainy.load('/backups/2026-06-01') * const hits = await db.find({ query: 'quarterly invoices' }) * await db.release() */ static async load(path: string): Promise> { const stat = await fs.promises.stat(path).catch(() => null) if (!stat || !stat.isDirectory()) { throw new Error( `Brainy.load(): '${path}' is not a snapshot directory ` + `(expected a directory created by db.persist(path))` ) } const brain = new Brainy({ storage: { type: 'filesystem', path }, mode: 'reader' }) await brain.init() return brain.createPinnedDb({ generation: brain.generationStore.generation(), timestamp: Date.now(), closeOnRelease: () => brain.close() }) } // --- Db host plumbing ------------------------------------------------------ /** * Guard for the synchronous Db entry points (`now()`, `generation()`): * they cannot await initialization, so they demand it up front. */ private assertGenerationStoreReady(method: string): void { if (!this.initialized || !this.generationStore) { throw new Error( `Cannot call ${method}() before init() completes. ` + `Await brain.init() (or brain.ready, or use Brainy.open()).` ) } } /** * Pin `generation` and construct the `Db` value over the shared host. The * `Db` constructor registers itself with the GC-backstop finalization * registry; explicit `db.release()` unregisters first. */ private createPinnedDb(init: { generation: number timestamp: number receipt?: TransactReceipt closeOnRelease?: () => Promise }): Db { this.pinGeneration(init.generation) return new Db({ host: this.dbHost, ...init }) } /** * The host surface every `Db` of this brain shares (lazily built once): * live read fast paths, generation-record materialization, snapshot * support, and pin lifecycle. See {@link DbHost} in src/db/db.ts. */ private get dbHost(): DbHost { if (!this._dbHost) { this._dbHost = { store: this.generationStore, storage: this.storage, get: (id, options) => this.get(id, options), find: (query) => this.find(query), related: (paramsOrId) => this.related(paramsOrId), resolveGeneration: (target, options) => this.resolveAsOfGeneration(target, options), entityFromRecord: (id, record, includeVectors) => this.entityFromGenerationRecord(id, record, includeVectors), relationFromRecord: (id, record) => this.relationFromGenerationRecord(id, record), persistPinned: (targetPath, generation) => this.persistPinnedGeneration(targetPath, generation), materializeAt: (generation) => this.materializeAtGeneration(generation), canServeVectorAtGeneration: (generation) => this.canServeVectorAtGeneration(generation), vectorSearchAtGeneration: (params, allowedIds, k, generation) => this.vectorSearchAtGeneration(params, allowedIds, k, generation), pinGeneration: (generation) => this.pinGeneration(generation), releaseGeneration: (generation) => this.releaseGeneration(generation), registerDbForFinalization: (db, generation, closeOnRelease) => { this.dbFinalizationRegistry.register(db, { generation, closeOnRelease }, db) }, unregisterDbFromFinalization: (db) => { this._dbFinalizationRegistry?.unregister(db) } } } return this._dbHost } /** Lazily build the GC backstop for leaked (never-released) `Db` pins. */ private get dbFinalizationRegistry(): FinalizationRegistry<{ generation: number closeOnRelease?: () => Promise }> { if (!this._dbFinalizationRegistry) { this._dbFinalizationRegistry = new FinalizationRegistry((held) => { this.releaseGeneration(held.generation) if (held.closeOnRelease) { void held.closeOnRelease().catch((err: Error) => { prodLog.warn(`[Brainy] Db finalization close failed: ${err.message}`) }) } }) } return this._dbFinalizationRegistry } /** * Registered index providers that implement the optional * {@link VersionedIndexProvider} capability (feature-detected on the three * index surfaces: vector, metadata, graph). Brainy's own JS indexes do not * implement it. */ private versionedIndexProviders(): VersionedIndexProvider[] { const candidates: unknown[] = [this.index, this.metadataIndex, this.graphIndex] return candidates.filter(isVersionedIndexProvider) } /** * Take one refcounted pin on `generation`: storage-record pin (gates * compaction) plus a `pin()` on every versioned index provider — the * explicit pin lifetime overrides any time-based snapshot retention the * provider has. Provider visibility (`isGenerationVisible`) is evaluated * at pin time per the locked read-routing rule. Without a versioned * provider, historical reads resolve from canonical generation records * and the at-generation index materialization (strictly correct, * provider-independent); a provider that retains the pinned generation * serves the same reads from its segments without rebuild. */ private pinGeneration(generation: number): void { this.generationStore.pin(generation) const big = BigInt(generation) for (const provider of this.versionedIndexProviders()) { const visible = provider.isGenerationVisible(big) provider.pin(big) if (!visible) { prodLog.debug( `[Brainy] generation ${generation} pinned but not provider-visible — ` + `historical reads at this pin resolve from canonical generation records` ) } } } /** Release one refcounted pin on `generation` (mirror of {@link pinGeneration}). */ private releaseGeneration(generation: number): void { this.generationStore.release(generation) const big = BigInt(generation) for (const provider of this.versionedIndexProviders()) { provider.release(big) } } /** * Materialize an `Entity` from a generation record's raw stored objects — * the historical-read counterpart of the live `get()` paths. * `record.metadata` is the exact stored metadata object; * `record.vector` is the stored HNSW noun object (whose `.vector` carries * the embedding) or `null` when the vector file was absent. */ private async entityFromGenerationRecord( id: string, record: { metadata: any; vector: any | null }, includeVectors: boolean ): Promise> { const entity = await this.convertMetadataToEntity(id, record.metadata) if (includeVectors && record.vector && Array.isArray(record.vector.vector)) { entity.vector = record.vector.vector } return entity } /** * Materialize a `Relation` from a generation record's raw stored objects. * Field split mirrors `storage.getVerb()` + `verbsToRelations()`: the * stored vector object carries the structural core (`sourceId`/`targetId`/ * `verb`), the stored metadata object carries typed fields plus the custom * metadata bag. Returns `null` when the metadata part is absent (the * relationship did not exist as a live edge). * @throws Error when metadata exists but the structural core is missing — * that state is unreachable through the API (relate() writes both parts * in one atomic batch) and indicates external tampering; throwing beats * silently dropping an edge from historical results. */ private relationFromGenerationRecord( id: string, record: { metadata: any; vector: any | null } ): Relation | null { if (record.metadata === null || record.metadata === undefined) { return null } const core = record.vector as { sourceId?: string; targetId?: string; verb?: string } | null if (!core || typeof core.sourceId !== 'string' || typeof core.targetId !== 'string') { throw new Error( `Generation record for relationship ${id} has metadata but no structural core ` + `(sourceId/targetId) — store corrupted or records modified outside Brainy` ) } // Canonical reserved/custom split — same single source of truth as the // live verb read paths (see src/types/reservedFields.ts). const { reserved, custom } = splitVerbMetadataRecord( record.metadata as Record ) return { id, from: core.sourceId, to: core.targetId, type: (reserved.verb ?? core.verb) as VerbType, ...(reserved.subtype !== undefined && { subtype: reserved.subtype as string }), ...(reserved.visibility !== undefined && { visibility: reserved.visibility as EntityVisibility }), weight: (reserved.weight as number) ?? 1.0, data: reserved.data, metadata: custom as T, service: reserved.service as string, createdAt: typeof reserved.createdAt === 'number' ? reserved.createdAt : Date.now(), ...(typeof reserved.updatedAt === 'number' && { updatedAt: reserved.updatedAt }), ...(typeof reserved.confidence === 'number' && { confidence: reserved.confidence }) } } /** * Snapshot the store at `generation` into `targetPath` — the brain-side * half of `db.persist()`. Indexes are flushed first (so the snapshot's * persisted index files match its records), then the snapshot is cut * under the generation store's commit lock: no transact commit, no * compaction, and no counter write can interleave with the hard-link * walk, and the counter is durably persisted into the snapshot. * * @throws GenerationConflictError when `generation` is no longer the * store's latest — a snapshot captures current bytes, so persist the * view before further writes (pin with `brain.now()`, persist, then * mutate). */ private async persistPinnedGeneration(targetPath: string, generation: number): Promise { await this.ensureInitialized() await this.flush() await this.generationStore.snapshotWith(async () => { if (generation !== this.generationStore.generation()) { throw new GenerationConflictError(generation, this.generationStore.generation()) } await this.storage.snapshotToDirectory(targetPath) }) } /** * Build the at-`generation` index materialization — the brain-side half of * the historical full-query surface (`Db.find`/`Db.related` at * past pinned generations; see `src/db/db.ts` and * `docs/ADR-001-generational-mvcc.md`): * * 1. Flush, then enumerate every live entity/relationship id plus every id * touched by transactions committed after `generation`. * 2. Resolve each id AT `generation` (live bytes when untouched since the * pin; immutable before-images otherwise) and copy the raw stored * objects into a fresh in-memory storage adapter. * 3. A final reconciliation pass under the store's commit mutex * re-resolves ids touched by transactions that committed DURING the * copy, so the materialized set is exactly the at-`generation` record * set. (Single-operation writes racing the copy follow the documented * history granularity — they remain visible through earlier pins.) * 4. Open a read-only `Brainy` over that storage: its init reconciliation * rebuilds the metadata and graph-adjacency indexes by scanning the * copied records, and the materializer then builds the * `JsHnswVectorIndex` by inserting every copied at-`generation` vector * (the persisted-HNSW restore path has nothing to restore — the * at-generation graph never existed on disk). The host's embedder is * shared so semantic queries embed through the already-loaded model, * and the host's aggregate definitions are re-registered so * `find({ aggregate })` computes at-generation values via * backfill-on-define. * * COST — document-grade contract: O(n at G) time and memory, ONCE per * `Db` (the `Db` caches the handle; `db.release()` frees it). This is the * open-core price of historical index queries. A native * `VersionedIndexProvider` serves the same reads from its retained * segments without any rebuild — when one is registered, it accelerates * these queries instead. */ private async materializeAtGeneration(generation: number): Promise> { await this.ensureInitialized() await this.flush() const snapshotStorage = new MemoryStorage() await snapshotStorage.init() // Copy one id's at-`generation` state into the snapshot (or ensure absence) // as a PURE LOOKUP against a precomputed `firstAfter` map (id → the first // generation after the pin that touched it). An id absent from the map was // untouched since the pin → its live storage state IS its at-`generation` // state; an id present resolves from that generation's immutable // before-image. Both `resolveManyAt` (builds the map) and // `readGenerationRecord` (reads the before-image) are MUTEX-FREE, so this is // safe to call inside the reconciliation pass below (which holds the commit // mutex via `snapshotWith`) — a per-id `resolveAt` there would re-enter the // mutex and DEADLOCK, and would regress the O(R) bulk copy to O(N·R). const copyAt = async ( kind: 'noun' | 'verb', id: string, firstAfter: Map ): Promise => { let record: { metadata: any; vector: any | null } | null = null const candidate = firstAfter.get(id) if (candidate === undefined) { // Untouched since the pin → live storage IS the at-`generation` state. const raw = kind === 'noun' ? await this.storage.readNounRaw(id) : await this.storage.readVerbRaw(id) if (raw.metadata !== null) record = raw } else { const before = await this.generationStore.readGenerationRecord(kind, candidate, id) if (before === null) { throw new Error( `Generation record missing: _generations/${candidate}/prev/${id}.json ` + `(store corrupted or records removed outside compactHistory())` ) } // A non-null metadata before-image is the at-`generation` live state; a // null-metadata before-image (the create sentinel, or a metadata-less // stored state) is absent at this generation. if (before.metadata !== null) record = { metadata: before.metadata, vector: before.vector } } // `record === null` — absent at this generation: writing nulls ensures the // id is absent in the snapshot, which also un-copies ids created by // transactions that commit mid-build. if (kind === 'noun') { await snapshotStorage.writeNounRaw(id, record ?? { metadata: null, vector: null }) } else { await snapshotStorage.writeVerbRaw(id, record ?? { metadata: null, vector: null }) } } // Bulk copy: live ids ∪ ids touched after the pinned generation (the // union covers entities deleted after the pin, which are no longer // listed live but resolve from before-images). let watermark = this.generationStore.committedGeneration() const changed = await this.generationStore.changedBetween(generation, watermark) const nounIds = new Set(changed.nouns) const verbIds = new Set(changed.verbs) for (const path of await this.storage.listRawObjects('entities/nouns')) { const id = entityIdFromCanonicalPath(path) if (id) nounIds.add(id) } for (const path of await this.storage.listRawObjects('entities/verbs')) { const id = entityIdFromCanonicalPath(path) if (id) verbIds.add(id) } // ONE ascending pass per kind resolves the whole universe's first-after // generations (O(R) getDelta reads, NOT O(N·R)), then each copy is a lookup. const nounFirstAfter = await this.generationStore.resolveManyAt('noun', nounIds, generation) const verbFirstAfter = await this.generationStore.resolveManyAt('verb', verbIds, generation) for (const id of nounIds) await copyAt('noun', id, nounFirstAfter) for (const id of verbIds) await copyAt('verb', id, verbFirstAfter) // Reconciliation pass under the commit mutex: nothing can commit during // this section, so afterwards the snapshot is exactly the at-`generation` // record set even when transactions committed while the bulk copy ran. // Reconciled ids join the enumeration sets so the vector-index build // below sees them too. `resolveManyAt`/`readGenerationRecord` are mutex-free, // so this section never re-enters the commit mutex (no deadlock). await this.generationStore.snapshotWith(async () => { const committedNow = this.generationStore.committedGeneration() if (committedNow === watermark) return const delta = await this.generationStore.changedBetween(watermark, committedNow) const reconNouns = new Set(delta.nouns) const reconVerbs = new Set(delta.verbs) const reconNounFirstAfter = await this.generationStore.resolveManyAt( 'noun', reconNouns, generation ) const reconVerbFirstAfter = await this.generationStore.resolveManyAt( 'verb', reconVerbs, generation ) for (const id of reconNouns) { nounIds.add(id) await copyAt('noun', id, reconNounFirstAfter) } for (const id of reconVerbs) { verbIds.add(id) await copyAt('verb', id, reconVerbFirstAfter) } watermark = committedNow }) // Open the ephemeral reader: init's index reconciliation rebuilds the // metadata and graph indexes by scanning the copied records. const reader = new Brainy({ storage: snapshotStorage, mode: 'reader', requireSubtype: false, silent: true }) await reader.init() // Share the host's embedder and dimensionality — semantic queries on the // materialization must never load a second embedding model. reader.embedder = this.embedder reader.dimensions = this.dimensions // Build the reader's vector index by INSERTING the copied at-generation // vectors. Unlike the metadata/graph indexes (which rebuild by scanning // records), `JsHnswVectorIndex.rebuild()` only restores a previously // persisted graph structure — and the at-generation HNSW graph never // existed on disk, so the materializer constructs it the honest way: // one insert per vector (the O(n log n at G) component of the documented // materialization cost). for (const id of nounIds) { const noun = await snapshotStorage.getNoun(id) if (noun && Array.isArray(noun.vector) && noun.vector.length > 0) { await reader.index.addItem({ id: noun.id, vector: noun.vector }) } } // Re-register the host's aggregate definitions; backfill-on-define // computes their at-generation values from the materialized record set // on first query. if (this._aggregationIndex) { for (const def of this._aggregationIndex.getDefinitions()) { reader.defineAggregate(def) } } let closed = false return { find: (query) => reader.find(query), related: (paramsOrId) => reader.related(paramsOrId), close: async () => { if (closed) return closed = true await reader.close() } } } /** * @description 8.0 #35 — true when the vector index is a * {@link VersionedIndexProvider} that advertises `isGenerationVisible(generation)`: * it can serve the at-`generation` vector leg from retained segments, so a * historical filtered semantic read needs no O(n@G) JS-HNSW rebuild. The built-in * `JsHnswVectorIndex` is not versioned (only ever "now"), and a native provider * that cannot honor `generation` MUST return `false` here (refuse, never fabricate * now-vectors-as-at-gen), so the caller falls back to materialization. */ private canServeVectorAtGeneration(generation: number): boolean { return ( isVersionedIndexProvider(this.index) && this.index.isGenerationVisible(BigInt(generation)) ) } /** * @description 8.0 #35 — run the vector kNN AS OF `generation`, restricted to * `allowedIds` (the at-gen metadata∩graph universe the `Db` resolved from the * record layer). The versioned provider serves the at-gen walk; Brainy composes * the metadata half. Only reached when {@link canServeVectorAtGeneration} is true. * @param params - The semantic (`query`) or explicit-`vector` find params. * @param allowedIds - The at-gen candidate universe (membership-correct at `generation`). * @param k - Over-fetch (page + headroom). * @param generation - The as-of generation (Brainy's u64 commit counter). * @returns `[id, distance]` pairs, ascending distance (descending relevance). */ private async vectorSearchAtGeneration( params: FindParams, allowedIds: ReadonlySet, k: number, generation: number ): Promise> { const vector = params.vector || (await this.embed(params.query!)) // #35 part-3: supply the at-gen candidate VECTORS (resolved from Brainy's // per-generation record before-images) so the provider reranks the historically // correct vectors with zero per-vector crossing. `atGenerationVectors.ids` is the // candidate set; `allowedIds` rides along (the provider may AND it). const atGenerationVectors = await this.buildAtGenerationVectors(allowedIds, generation) return this.index.search(vector, k, undefined, { allowedIds, generation: BigInt(generation), ...(atGenerationVectors && { atGenerationVectors }) }) } /** * @description 8.0 #35 part-3 — assemble the {@link AtGenerationVectors} columnar * payload for a filtered at-gen exact-rerank: resolve each universe id's vector AS * OF `generation` (the canonical record before-image, or live storage when the id * was untouched since the pin), intern the id via the shared mapper, and pack the * vectors flat row-major (`vectors[i*dim .. (i+1)*dim]` ↔ `ids[i]`). Ids absent at * the generation, vectorless, or of the wrong dimension are dropped (they cannot be * vector-ranked). Bounded by the filtered universe — far cheaper than the full-corpus * rebuild it replaces. Returns `undefined` when no dimension is known or nothing maps. * @param allowedIds - The at-gen filtered universe (candidate ids). * @param generation - The as-of generation. * @returns The columnar payload, or `undefined` when there is nothing to ship. */ private async buildAtGenerationVectors( allowedIds: ReadonlySet, generation: number ): Promise { const dim = this.dimensions if (!dim || allowedIds.size === 0) return undefined const idMapper = this.metadataIndex.getIdMapper() const ints: bigint[] = [] const rows: number[][] = [] for (const id of allowedIds) { const resolved = await this.generationStore.resolveAt('noun', id, generation) let vec: number[] | null = null if (resolved.source === 'record') { // The generation record carries the at-gen vector before-image. vec = resolved.vector } else if (resolved.source === 'current') { // Untouched since the pin → the live vector IS the at-gen vector. Use // getNoun (full hydration, lazy-load aware) — readNounRaw's canonical path // is empty under lazy-vector eviction. const noun = await this.storage.getNoun(id) vec = noun?.vector ?? null } // 'absent' / vectorless / wrong-dim → skip (not vector-rankable at this gen). if (Array.isArray(vec) && vec.length === dim) { ints.push(BigInt(idMapper.getInt(id) ?? idMapper.getOrAssign(id))) rows.push(vec) } } if (ints.length === 0) return undefined // Row-major pack: INVARIANT vectors.length === ids.length * dim. const vectors = new Float32Array(ints.length * dim) for (let i = 0; i < rows.length; i++) vectors.set(rows[i], i * dim) return { ids: BigInt64Array.from(ints), vectors, dim } } // --- Transact planner ------------------------------------------------------ /** * Plan a full `transact()` batch: validate and resolve every operation * against "current state + batch so far", producing the ordered * TransactionManager operations, the receipt ids, the touched-id sets * (which the generation store stages before-images for), and the * post-commit aggregation hooks. Planning performs reads and embedding * only — nothing touches storage until the generation store runs the * batch. */ private async planTransact(ops: TxOperation[]): Promise { const state: TxPlanState = { nouns: new Map(), removedNouns: new Set(), verbs: new Map(), removedVerbs: new Set() } const plan: PlannedTransact = { operations: [], ids: [], touchedNouns: [], touchedVerbs: [], postCommit: [], casUpdates: [], createdNouns: new Set(), changeEvents: [] } for (const op of ops) { switch (op.op) { case 'add': plan.ids.push(await this.planTxAdd(op, state, plan)) break case 'update': plan.ids.push(await this.planTxUpdate(op, state, plan)) break case 'remove': plan.ids.push(await this.planTxRemove(op, state, plan)) break case 'relate': plan.ids.push(await this.planTxRelate(op, state, plan)) break case 'unrelate': plan.ids.push(await this.planTxUnrelate(op, state, plan)) break default: { const exhaustive: never = op throw new Error(`transact(): unknown operation ${JSON.stringify(exhaustive)}`) } } } return plan } /** * Resolve an entity during planning: batch-pending writes win, then the * live store. `includeVectors: false` returns the stub vector exactly as * the live metadata-only `get()` does, so planned relationship vectors * match `relate()` byte-for-byte. */ private async planGetEntity( state: TxPlanState, id: string, options?: GetOptions ): Promise | null> { if (state.removedNouns.has(id)) { return null } const pending = state.nouns.get(id) if (pending) { const entity = await this.convertMetadataToEntity(id, pending.metadata) if (options?.includeVectors) { entity.vector = pending.vector } return entity } return this.get(id, options) } /** Plan one `{ op: 'add' }` — mirror of `add()`. Returns the entity id. */ private async planTxAdd( op: Extract, { op: 'add' }>, state: TxPlanState, plan: PlannedTransact ): Promise { const { op: _discriminator, ...rawParams } = op validateAddParams(rawParams as AddParams) // Same reserved-field normalization as add() — the metadata bag is // cleaned BEFORE enforcement so a remapped subtype participates in // subtype-pairing enforcement and only custom fields reach the index. const params = this.remapReservedAddMetadata(rawParams as AddParams) this.enforceTrackedFieldValues(params.metadata as Record | undefined, 'metadata') this.enforceTrackedFieldValues({ subtype: params.subtype } as Record, 'top-level') this.enforceSubtypeOnAdd('add', params.type, params.subtype, params.metadata) // Id normalization (8.0) — mirror of add(): a natural key coerces to a // STABLE UUID (v5), preserving the original under ORIGINAL_ID_KEY; a real // UUID passes through; no id mints a fresh v7. Done BEFORE ifAbsent so a // natural-key upsert is idempotent against the same key. const { id, originalId } = coerceNewEntityId(params.id) // ifAbsent and upsert resolve the same id collision in opposite ways // (skip vs. merge) — mirror of add()'s hard error. if (params.ifAbsent && params.upsert) { throw new Error( 'transact add: ifAbsent and upsert are mutually exclusive — ifAbsent skips an existing entity, upsert merges into it.' ) } // ifAbsent — idempotent by-id insert, resolved against batch + store. if (params.id && params.ifAbsent) { const existing = await this.planGetEntity(state, id) if (existing) { return id } } // upsert — by-id create-or-update, resolved against batch + store. When the id // already exists, delegate to planTxUpdate with a synthesized update op so the // supplied fields MERGE (mirror of add()'s live upsert path) instead of // overwriting. The id is already canonical here (coerceNewEntityId above). if (params.id && params.upsert) { const existing = await this.planGetEntity(state, id) if (existing) { return this.planTxUpdate( { op: 'update', id, ...(params.data !== undefined && { data: params.data }), ...(params.type !== undefined && { type: params.type }), ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && { visibility: params.visibility }), ...(params.metadata !== undefined && { metadata: params.metadata }), ...(params.vector !== undefined && { vector: params.vector }), ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }) } as Extract, { op: 'update' }>, state, plan ) } } const vector = params.vector || (await this.embed(params.data)) if (!this.dimensions) { this.dimensions = vector.length } else if (vector.length !== this.dimensions) { throw new Error( `Vector dimension mismatch: expected ${this.dimensions}, got ${vector.length}` ) } // isNew controls the operation's rollback strategy: a custom id may // collide with an existing entity (add() overwrite semantics), and a // failed batch must restore the overwritten bytes. const isNew = params.id ? !state.nouns.has(id) && (await this.storage.getNounMetadata(id)) === null : true // Either way (fresh create OR overwrite) this add resets the id's revision // baseline (`_rev: 1` below), so later in-batch updates to it sequence // against batch-local state — see PlannedTransact.createdNouns. plan.createdNouns.add(id) const now = Date.now() const storageMetadata = { ...params.metadata, // Preserve the caller's original (non-UUID) id when normalized — mirror // of add(). A real UUID passes through with no _originalId. ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }), data: params.data, noun: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), // visibility: stored only when not 'public' (absent === public, keeps records lean) ...(params.visibility !== undefined && params.visibility !== 'public' && { visibility: params.visibility }), service: params.service, createdAt: now, updatedAt: now, _rev: 1, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), ...(params.createdBy && { createdBy: params.createdBy }) } const entityForIndexing = { id, vector, connections: new Map(), level: 0, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && params.visibility !== 'public' && { visibility: params.visibility }), ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), createdAt: now, updatedAt: now, service: params.service, data: params.data, ...(params.createdBy && { createdBy: params.createdBy }), metadata: { ...(params.metadata || {}), ...(originalId !== undefined && { [ORIGINAL_ID_KEY]: originalId }) } } plan.operations.push( new SaveNounMetadataOperation(this.storage, id, storageMetadata, isNew), new SaveNounOperation(this.storage, { id, vector, connections: new Map(), level: 0 }, isNew), new AddToHNSWOperation(this.index, id, vector), new AddToMetadataIndexOperation(this.metadataIndex, id, entityForIndexing) ) plan.touchedNouns.push(id) plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityAdded(id, entityForIndexing) } }) if (this._changeFeed.hasListeners) { plan.changeEvents.push({ kind: 'entity', op: 'add', id, entity: { id, type: String(params.type), ...(params.subtype !== undefined && { subtype: String(params.subtype) }), metadata: (params.metadata as Record) ?? {}, ...(params.service !== undefined && { service: String(params.service) }) } }) } state.nouns.set(id, { metadata: storageMetadata, vector }) state.removedNouns.delete(id) return id } /** Plan one `{ op: 'update' }` — mirror of `update()`. Returns the entity id. */ private async planTxUpdate( op: Extract, { op: 'update' }>, state: TxPlanState, plan: PlannedTransact ): Promise { const { op: _discriminator, ...rawParams } = op validateUpdateParams(rawParams as UpdateParams) // Same reserved-field normalization as update() — user-mutable fields // remap to their dedicated param (top-level wins), system-managed fields // drop with a one-shot warning. const params = this.remapReservedUpdateMetadata(rawParams as UpdateParams) // Id normalization (8.0) — mirror of update(): a natural key resolves to the // canonical UUID add() stored. A real UUID passes through. params.id = resolveEntityId(params.id) this.enforceTrackedFieldValues(params.metadata as Record | undefined, 'metadata') if (params.subtype !== undefined) { this.enforceTrackedFieldValues({ subtype: params.subtype } as Record, 'top-level') } const existing = await this.planGetEntity(state, params.id, { includeVectors: true }) if (!existing) { throw new EntityNotFoundError(params.id) } if (params.subtype !== undefined || params.type !== undefined) { const effectiveType = params.type ?? existing.type const effectiveSubtype = params.subtype !== undefined ? params.subtype : existing.subtype const effectiveMetadata = params.metadata ?? existing.metadata this.enforceSubtypeOnAdd('update', effectiveType, effectiveSubtype, effectiveMetadata) } // ifRev CAS — identical resolution to update(); a conflict rejects the // WHOLE batch. This planning-time check is purely a FAST-FAIL (it can // interleave with concurrent writers); the authoritative re-verify runs in // transact()'s commit precondition under the commit mutex, via the // plan.casUpdates entry registered below. const currentRev = typeof existing._rev === 'number' ? existing._rev : 1 if (typeof params.ifRev === 'number' && params.ifRev !== currentRev) { throw new RevisionConflictError(params.id, params.ifRev, currentRev) } // Resolve the updated vector — mirror of update(): an explicit `vector` // always wins, new `data` re-embeds, otherwise the existing vector is // kept. Any vector change re-indexes HNSW below. let vector = existing.vector if (params.vector) { if (this.dimensions && params.vector.length !== this.dimensions) { throw new Error( `Vector dimension mismatch: expected ${this.dimensions}, got ${params.vector.length}` ) } vector = params.vector } else if (params.data) { vector = await this.embed(params.data) } const needsReindexing = Boolean(params.data || params.type || params.vector) const newMetadata = params.merge !== false ? { ...existing.metadata, ...params.metadata } : params.metadata || existing.metadata const now = Date.now() const updatedMetadata = { ...newMetadata, data: params.data !== undefined ? params.data : existing.data, noun: params.type || existing.type, service: existing.service, createdAt: existing.createdAt, updatedAt: now, _rev: currentRev + 1, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.weight !== undefined && { weight: params.weight }), ...(params.confidence === undefined && existing.confidence !== undefined && { confidence: existing.confidence }), ...(params.weight === undefined && existing.weight !== undefined && { weight: existing.weight }), ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.subtype === undefined && existing.subtype !== undefined && { subtype: existing.subtype }), // Visibility: new value if provided, else preserve existing; stored only when the // effective value is not 'public' (a change to 'public' drops the field). ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { visibility: params.visibility ?? existing.visibility }) } // Register for the authoritative under-mutex CAS re-verify + rev re-stamp // (see PlannedTransact.casUpdates). The staged UpdateNounMetadataOperation // holds `updatedMetadata` by reference, so the precommit re-stamp lands in // what execute() writes. plan.casUpdates.push({ id: params.id, ...(typeof params.ifRev === 'number' && { ifRev: params.ifRev }), updatedMetadata }) const entityForIndexing = { id: params.id, vector, connections: new Map(), level: 0, type: params.type || existing.type, subtype: params.subtype !== undefined ? params.subtype : existing.subtype, ...(((params.visibility ?? existing.visibility) ?? 'public') !== 'public' && { visibility: params.visibility ?? existing.visibility }), confidence: params.confidence !== undefined ? params.confidence : existing.confidence, weight: params.weight !== undefined ? params.weight : existing.weight, createdAt: existing.createdAt, updatedAt: now, service: existing.service, data: params.data !== undefined ? params.data : existing.data, createdBy: existing.createdBy, metadata: newMetadata } const removalMetadata = { type: existing.type, confidence: existing.confidence, weight: existing.weight, createdAt: existing.createdAt, updatedAt: existing.updatedAt, service: existing.service, data: existing.data, createdBy: existing.createdBy, metadata: existing.metadata } plan.operations.push( new UpdateNounMetadataOperation(this.storage, params.id, updatedMetadata), new SaveNounOperation(this.storage, { id: params.id, vector, connections: new Map(), level: 0 }) ) if (needsReindexing) { plan.operations.push( new RemoveFromHNSWOperation(this.index, params.id, existing.vector), new AddToHNSWOperation(this.index, params.id, vector) ) } plan.operations.push( new RemoveFromMetadataIndexOperation(this.metadataIndex, params.id, removalMetadata), new AddToMetadataIndexOperation(this.metadataIndex, params.id, entityForIndexing) ) plan.touchedNouns.push(params.id) const oldEntityForAgg = { type: existing.type, service: existing.service, data: existing.data, metadata: existing.metadata } plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityUpdated(params.id, entityForIndexing, oldEntityForAgg) } }) if (this._changeFeed.hasListeners) { plan.changeEvents.push({ kind: 'entity', op: 'update', id: params.id, entity: { id: params.id, type: String(entityForIndexing.type), ...(entityForIndexing.subtype !== undefined && { subtype: String(entityForIndexing.subtype) }), metadata: (newMetadata as Record) ?? {}, ...(entityForIndexing.service !== undefined && { service: String(entityForIndexing.service) }) } }) } state.nouns.set(params.id, { metadata: updatedMetadata, vector }) return params.id } /** * Plan one `{ op: 'remove' }` — mirror of `remove()`, including the * relationship cascade (every edge where the entity is source or target, * plus edges created earlier in this batch). Returns the entity id. */ private async planTxRemove( op: Extract, { op: 'remove' }>, state: TxPlanState, plan: PlannedTransact ): Promise { if (!op.id || typeof op.id !== 'string') { throw new Error(`transact(): remove operation requires an entity id (got ${JSON.stringify(op.id)})`) } // Id normalization (8.0) — mirror of remove(): a natural key resolves to the // canonical UUID add() stored. A real UUID passes through. const id = resolveEntityId(op.id) const pending = state.nouns.get(id) const metadata = pending ? pending.metadata : await this.storage.getNounMetadata(id) const noun = pending ? { id, vector: pending.vector, connections: new Map(), level: 0 } : await this.storage.getNoun(id) // Cascade set: stored edges touching the entity, plus batch-pending // edges, minus edges already removed in this batch. const storedVerbs = [ ...(await this.storage.getVerbsBySource(id)), ...(await this.storage.getVerbsByTarget(id)) ] const cascade = new Map() for (const verb of storedVerbs) { if (!state.removedVerbs.has(verb.id)) { cascade.set(verb.id, verb) } } for (const [verbId, verb] of state.verbs) { if ( !state.removedVerbs.has(verbId) && (verb.sourceId === id || verb.targetId === id) ) { cascade.set(verbId, verb) } } if (noun) { plan.operations.push(new RemoveFromHNSWOperation(this.index, id, noun.vector)) } if (metadata) { plan.operations.push(new RemoveFromMetadataIndexOperation(this.metadataIndex, id, metadata)) } // Pre-read metadata rides along: the count decrement must not depend on // re-reading the record being removed (see remove()). plan.operations.push(new DeleteNounMetadataOperation(this.storage, id, metadata)) for (const verb of cascade.values()) { plan.operations.push( // Endpoint ints resolve at EXECUTE time — a cascade verb (or its // endpoints) may have been created earlier in this same batch, so a // plan-time resolution would ask the id mapper about entities that do // not exist yet (the native mapper rightly refuses). new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration), new DeleteVerbMetadataOperation(this.storage, verb.id) ) plan.touchedVerbs.push(verb.id) state.verbs.delete(verb.id) state.removedVerbs.add(verb.id) if (this._changeFeed.hasListeners) { plan.changeEvents.push({ kind: 'relation', op: 'unrelate', id: verb.id, relation: { id: verb.id, from: verb.sourceId, to: verb.targetId, type: String(verb.verb) } }) } } plan.touchedNouns.push(id) if (this._changeFeed.hasListeners) { plan.changeEvents.push({ kind: 'entity', op: 'remove', id, ...(metadata && { entity: this.entityViewFromRawRecord(id, metadata as Record) }) }) } if (metadata) { // Canonical reserved/custom split — mirror of remove()'s aggregation hook. const { reserved, custom } = splitNounMetadataRecord(metadata) const entityForAgg = { type: reserved.noun, service: reserved.service, data: reserved.data, metadata: custom } plan.postCommit.push(() => { if (this._aggregationIndex) { this._aggregationIndex.onEntityDeleted(id, entityForAgg) } }) } state.nouns.delete(id) state.removedNouns.add(id) return id } /** * Plan one `{ op: 'relate' }` — mirror of `relate()`, including duplicate * deduplication (against the store AND earlier batch operations) and * `bidirectional`. Returns the (primary) relationship id — the existing id * when deduplicated. */ private async planTxRelate( op: Extract, { op: 'relate' }>, state: TxPlanState, plan: PlannedTransact ): Promise { const { op: _discriminator, ...rawParams } = op validateRelateParams(rawParams as RelateParams) // Same reserved-field normalization as relate(). const params = this.remapReservedRelateMetadata(rawParams as RelateParams) // Id normalization (8.0) — mirror of relate(): resolve BOTH endpoints to the // canonical UUID add() stored, so a relate op may reference either side by // natural key. Real UUIDs pass through. (Relationship ids are engine-minted.) params.from = resolveEntityId(params.from) params.to = resolveEntityId(params.to) this.enforceSubtypeOnRelate('relate', params.type, params.subtype, params.metadata) const fromEntity = await this.planGetEntity(state, params.from) const toEntity = await this.planGetEntity(state, params.to) if (!fromEntity) { throw new EntityNotFoundError(params.from, `Source entity ${params.from} not found`) } if (!toEntity) { throw new EntityNotFoundError(params.to, `Target entity ${params.to} not found`) } // Dedupe against earlier operations of this batch first… for (const [verbId, verb] of state.verbs) { if ( !state.removedVerbs.has(verbId) && verb.sourceId === params.from && verb.targetId === params.to && verb.verb === params.type ) { return verbId } } // …then against the committed graph (same lookup relate() performs). const dupSourceInt = this.graphEntityInt(params.from) const verbIds = dupSourceInt === undefined ? [] : await this.resolveVerbIntsToIds(await this.graphIndex.getVerbIdsBySource(dupSourceInt)) if (verbIds.length > 0) { const verbsMap = await this.graphIndex.getVerbsBatchCached(verbIds) for (const verb of verbsMap.values()) { if ( !state.removedVerbs.has(verb.id) && verb.targetId === params.to && verb.verb === params.type ) { return verb.id } } } const id = uuidv4() const relationVector = fromEntity.vector.map((v, i) => (v + toEntity.vector[i]) / 2) const now = Date.now() const verbMetadata = { ...(params.metadata || {}), verb: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), // visibility: stored only when not 'public' (absent === public, keeps records lean) ...(params.visibility !== undefined && params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.service !== undefined && { service: params.service }), createdAt: now, ...(params.data !== undefined && { data: params.data }) } const verb: GraphVerb = { id, vector: relationVector, sourceId: params.from, targetId: params.to, verb: params.type, type: params.type, ...(params.subtype !== undefined && { subtype: params.subtype }), ...(params.visibility !== undefined && params.visibility !== 'public' && { visibility: params.visibility }), weight: params.weight ?? 1.0, ...(params.confidence !== undefined && { confidence: params.confidence }), ...(params.service !== undefined && { service: params.service }), metadata: params.metadata, data: params.data, createdAt: now } plan.operations.push( new SaveVerbOperation(this.storage, { id, vector: relationVector, connections: new Map(), verb: params.type, sourceId: params.from, targetId: params.to }), new SaveVerbMetadataOperation(this.storage, id, verbMetadata), // Endpoint ints resolve at EXECUTE time — after any same-batch add of // an endpoint has applied. Plan-time resolution was a consumer-reported // native/JS parity bug: transact([add X, relate →X]) asked the native // id mapper to assign an int for an entity that did not exist yet. new AddToGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, id) ) ) plan.touchedVerbs.push(id) state.verbs.set(id, verb) state.removedVerbs.delete(id) if (this._changeFeed.hasListeners) { plan.changeEvents.push({ kind: 'relation', op: 'relate', id, relation: { id, from: params.from, to: params.to, type: String(params.type), ...(params.metadata && { metadata: params.metadata as Record }) } }) } if (params.bidirectional) { const reverseId = uuidv4() const reverseVerb: GraphVerb = { ...verb, id: reverseId, sourceId: params.to, targetId: params.from // sourceInt/targetInt are mirrored onto this object when the graph // operation resolves endpoints at execute time. } plan.operations.push( new SaveVerbOperation(this.storage, { id: reverseId, vector: relationVector, connections: new Map(), verb: params.type, sourceId: params.to, targetId: params.from }), new SaveVerbMetadataOperation(this.storage, reverseId, verbMetadata), new AddToGraphIndexOperation(this.graphIndex, reverseVerb, () => this.resolveVerbEndpointInts(reverseVerb), this.graphWriteGeneration, (verbInt) => this.cacheVerbInt(verbInt, reverseId) ) ) plan.touchedVerbs.push(reverseId) state.verbs.set(reverseId, reverseVerb) state.removedVerbs.delete(reverseId) if (this._changeFeed.hasListeners) { plan.changeEvents.push({ kind: 'relation', op: 'relate', id: reverseId, relation: { id: reverseId, from: params.to, to: params.from, type: String(params.type), ...(params.metadata && { metadata: params.metadata as Record }) } }) } } return id } /** Plan one `{ op: 'unrelate' }` — mirror of `unrelate()`. Returns the relationship id. */ private async planTxUnrelate( op: Extract, { op: 'unrelate' }>, state: TxPlanState, plan: PlannedTransact ): Promise { if (!op.id || typeof op.id !== 'string') { throw new Error(`transact(): unrelate operation requires a relationship id (got ${JSON.stringify(op.id)})`) } const id = op.id const verb = state.removedVerbs.has(id) ? null : (state.verbs.get(id) ?? (await this.storage.getVerb(id))) if (verb) { plan.operations.push( // Endpoint ints resolve at EXECUTE time — the verb (or its endpoints) // may have been created earlier in this same batch (forward refs). new RemoveFromGraphIndexOperation(this.graphIndex, verb, () => this.resolveVerbEndpointInts(verb), this.graphWriteGeneration) ) } plan.operations.push(new DeleteVerbMetadataOperation(this.storage, id)) plan.touchedVerbs.push(id) if (this._changeFeed.hasListeners) { plan.changeEvents.push({ kind: 'relation', op: 'unrelate', id, ...(verb && { relation: { id, from: verb.sourceId, to: verb.targetId, type: String(verb.verb) } }) }) } state.verbs.delete(id) state.removedVerbs.add(id) return id } /** * Get total count of nouns - O(1) operation * @returns Promise that resolves to the total number of nouns */ async getNounCount(): Promise { await this.ensureInitialized() return this.storage.getNounCount() } /** * Get total count of verbs - O(1) operation * @returns Promise that resolves to the total number of verbs */ async getVerbCount(): Promise { await this.ensureInitialized() return this.storage.getVerbCount() } /** * Get memory statistics and limits * * Returns detailed memory information including: * - Current heap usage * - Container memory limits (if detected) * - Query limits and how they were calculated * - Memory allocation recommendations * * Use this to debug why query limits are low or to understand * memory allocation in production environments. * * @returns Memory statistics and configuration * * @example * ```typescript * const stats = brain.getMemoryStats() * console.log(`Query limit: ${stats.limits.maxQueryLimit}`) * console.log(`Basis: ${stats.limits.basis}`) * console.log(`Free memory: ${Math.round(stats.memory.free / 1024 / 1024)}MB`) * ``` */ getMemoryStats(): { memory: { heapUsed: number heapTotal: number external: number rss: number free: number total: number containerLimit: number | null } limits: { maxQueryLimit: number maxQueryLength: number maxVectorDimensions: number basis: 'override' | 'reservedMemory' | 'containerMemory' | 'freeMemory' } config: { maxQueryLimit?: number reservedQueryMemory?: number } recommendations?: string[] } { const config = ValidationConfig.getInstance() const heapStats = process.memoryUsage ? process.memoryUsage() : { heapUsed: 0, heapTotal: 0, external: 0, rss: 0 } // Get system memory info let freeMemory = 0 let totalMemory = 0 try { const os = require('node:os') freeMemory = os.freemem() totalMemory = os.totalmem() } catch (e) { // OS module not available } const stats = { memory: { heapUsed: heapStats.heapUsed, heapTotal: heapStats.heapTotal, external: heapStats.external, rss: heapStats.rss, free: freeMemory, total: totalMemory, containerLimit: config.detectedContainerLimit }, limits: { maxQueryLimit: config.maxLimit, maxQueryLength: config.maxQueryLength, maxVectorDimensions: config.maxVectorDimensions, basis: config.limitBasis }, config: { maxQueryLimit: this.config.maxQueryLimit, reservedQueryMemory: this.config.reservedQueryMemory }, recommendations: [] as string[] } // Generate recommendations based on stats if (stats.limits.basis === 'freeMemory' && stats.memory.containerLimit) { stats.recommendations.push( `Container detected (${Math.round(stats.memory.containerLimit / 1024 / 1024)}MB) but limits based on free memory. ` + `Consider setting reservedQueryMemory config option for better limits.` ) } if (stats.limits.maxQueryLimit < 5000 && stats.memory.containerLimit && stats.memory.containerLimit > 2 * 1024 * 1024 * 1024) { stats.recommendations.push( `Query limit is low (${stats.limits.maxQueryLimit}) despite ${Math.round(stats.memory.containerLimit / 1024 / 1024 / 1024)}GB container. ` + `Consider: new Brainy({ reservedQueryMemory: 1073741824 }) to reserve 1GB for queries.` ) } if (stats.limits.basis === 'override') { stats.recommendations.push( `Using explicit maxQueryLimit override (${stats.limits.maxQueryLimit}). ` + `Auto-detection bypassed.` ) } return stats } // ============= SUB-APIS ============= /** * Natural Language Processing API */ nlp(): NaturalLanguageProcessor { if (!this._nlp) { this._nlp = new NaturalLanguageProcessor(this) } return this._nlp } /** * Entity Extraction API - Neural extraction with NounType taxonomy * * Extracts entities from text using: * - Pattern-based candidate detection * - Embedding-based type classification * - Context-aware confidence scoring * * @param text - Text to extract entities from * @param options - Extraction options * @returns Array of extracted entities with types and confidence * * Fast heuristic ensemble (pattern + type-embedding + context), not a trained NER — * each candidate is typed by its own span (no cross-candidate bleed). Confidences are * approximate; pass `types` to constrain results when precision matters. * * @param text - Text to extract entities from * @param options - Extraction options * @returns Array of extracted entities with types and confidence * * @example * const entities = await brain.extract('Sarah Chen founded Acme Corp') * // [ * // { text: 'Sarah Chen', type: NounType.Person, confidence: 0.68 }, * // { text: 'Acme Corp', type: NounType.Organization, confidence: 0.85 } * // ] */ async extract( text: string, options?: { types?: NounType[] confidence?: number includeVectors?: boolean neuralMatching?: boolean } ): Promise { if (!this._extractor) { this._extractor = new NeuralEntityExtractor(this) } return await this._extractor.extract(text, options) } /** * Extract entities from text (alias for extract()) * Added for API clarity — `extractEntities()` reads more naturally at call sites * * Uses NeuralEntityExtractor with SmartExtractor ensemble (4-signal architecture): * - ExactMatch (40%) - Dictionary lookups * - Embedding (35%) - Semantic similarity * - Pattern (20%) - Regex patterns * - Context (5%) - Contextual hints * * @param text - Text to extract entities from * @param options - Extraction options * @returns Array of extracted entities with types and confidence scores * * @example * ```typescript * const entities = await brain.extractEntities('John Smith founded Acme Corp', { * confidence: 0.7, * types: [NounType.Person, NounType.Organization], * neuralMatching: true * }) * ``` */ async extractEntities( text: string, options?: { types?: NounType[] confidence?: number includeVectors?: boolean neuralMatching?: boolean } ): Promise { return this.extract(text, options) } /** * Extract concepts from text * * Simplified interface for concept/topic extraction * Returns only concept names as strings for easy metadata population * * @param text - Text to extract concepts from * @param options - Extraction options * @returns Array of concept names * * @example * const concepts = await brain.extractConcepts('Using OAuth for authentication') * // ['oauth', 'authentication'] */ async extractConcepts( text: string, options?: { confidence?: number limit?: number } ): Promise { const entities = await this.extract(text, { types: [NounType.Concept], confidence: options?.confidence || 0.7, neuralMatching: true }) // Deduplicate and normalize const conceptSet = new Set(entities.map(e => e.text.toLowerCase())) const concepts = Array.from(conceptSet) // Apply limit if specified return options?.limit ? concepts.slice(0, options.limit) : concepts } /** * Import files with intelligent extraction and dual storage (VFS + Knowledge Graph) * * Unified import system that: * - Auto-detects format (Excel, PDF, CSV, JSON, Markdown) * - Extracts entities with AI-powered name/type detection * - Infers semantic relationships from context * - Stores in both VFS (organized files) and Knowledge Graph (connected entities) * - Links VFS files to graph entities * * @since 4.0.0 * * @example Quick Start (All AI features enabled by default) * ```typescript * const result = await brain.import('./glossary.xlsx') * // Auto-detects format, extracts entities, infers relationships * ``` * * @example Full-Featured Import (v4.x) * ```typescript * const result = await brain.import('./data.xlsx', { * // AI features * enableNeuralExtraction: true, // Extract entity names/metadata * enableRelationshipInference: true, // Detect semantic relationships * enableConceptExtraction: true, // Extract types/concepts * * // VFS features * vfsPath: '/imports/my-data', // Store in VFS directory * groupBy: 'type', // Organize by entity type * preserveSource: true, // Keep original file * * // Progress tracking (STANDARDIZED FOR ALL 7 FORMATS!) * onProgress: (p) => { * console.log(`[${p.stage}] ${p.message}`) * console.log(`Entities: ${p.entities || 0}, Rels: ${p.relationships || 0}`) * if (p.throughput) console.log(`Rate: ${p.throughput.toFixed(1)}/sec`) * } * }) * // THIS SAME HANDLER WORKS FOR CSV, PDF, Excel, JSON, Markdown, YAML, DOCX! * ``` * * @example Universal Progress Handler * ```typescript * // ONE handler for ALL 7 formats - no format-specific code needed! * const universalProgress = (p) => { * updateUI(p.stage, p.message, p.entities, p.relationships) * } * * await brain.import(csvBuffer, { onProgress: universalProgress }) * await brain.import(pdfBuffer, { onProgress: universalProgress }) * await brain.import(excelBuffer, { onProgress: universalProgress }) * // Works for JSON, Markdown, YAML, DOCX too! * ``` * * @example Performance Tuning (Large Files) * ```typescript * const result = await brain.import('./huge-file.csv', { * enableDeduplication: false, // Skip dedup for speed * confidenceThreshold: 0.8, // Higher threshold = fewer entities * onProgress: (p) => console.log(`${p.processed}/${p.total}`) * }) * ``` * * @example Import from Buffer or Object * ```typescript * // From buffer * const result = await brain.import(buffer, { format: 'pdf' }) * * // From object * const result = await brain.import({ entities: [...] }) * ``` * * @throws {Error} If invalid options are provided (v4.x breaking changes) * * @see {@link https://brainy.dev/docs/api/import API Documentation} * @see {@link https://brainy.dev/docs/guides/migrating-to-v4 Migration Guide} * @see {@link https://brainy.dev/docs/guides/standard-import-progress Standard Progress API} * * @remarks * **⚠️ Breaking Changes from v3.x:** * * The import API was redesigned for clarity and better feature control. * Old v3.x option names are **no longer recognized** and will throw errors. * * **Option Changes:** * - ❌ `extractRelationships` → ✅ `enableRelationshipInference` * - ❌ `createFileStructure` → ✅ `vfsPath: '/your/path'` * - ❌ `autoDetect` → ✅ *(removed - always enabled)* * - ❌ `excelSheets` → ✅ *(removed - all sheets processed)* * - ❌ `pdfExtractTables` → ✅ *(removed - always enabled)* * * **New Options:** * - ✅ `enableNeuralExtraction` - Extract entity names via AI * - ✅ `enableConceptExtraction` - Extract entity types via AI * - ✅ `preserveSource` - Save original file in VFS * * **If you get an error:** * The error message includes migration instructions and examples. * See the complete migration guide for all details. * * **Why these changes?** * - Clearer option names (explicitly describe what they do) * - Separation of concerns (neural, relationships, VFS are separate) * - Better defaults (AI features enabled by default) * - Reduced confusion (removed redundant options) */ async import( source: Buffer | string | object | PortableGraph, options?: { format?: 'excel' | 'pdf' | 'csv' | 'json' | 'markdown' | 'yaml' | 'docx' | 'image' vfsPath?: string groupBy?: 'type' | 'sheet' | 'flat' | 'custom' customGrouping?: (entity: any) => string createEntities?: boolean createRelationships?: boolean preserveSource?: boolean enableNeuralExtraction?: boolean enableRelationshipInference?: boolean enableConceptExtraction?: boolean confidenceThreshold?: number onProgress?: (progress: { stage: 'detecting' | 'extracting' | 'storing-vfs' | 'storing-graph' | 'relationships' | 'complete' phase?: 'extraction' | 'relationships' message: string processed?: number current?: number total?: number entities?: number relationships?: number throughput?: number eta?: number }) => void } & Partial ): Promise { // Portable graph round-trip: a PortableGraph document (produced by export()) is // restored via ONE atomic transaction — distinct from foreign-file ingestion below. if (isPortableGraph(source)) { return importGraph(this, this.storage, source, options as ImportOptions) } // Lazy load ImportCoordinator (foreign-file ingestion: CSV/PDF/Excel/JSON/…) const { ImportCoordinator } = await import('./import/ImportCoordinator.js') const coordinator = new ImportCoordinator(this) await coordinator.init() return await coordinator.import(source as Buffer | string | object, options) } /** * Virtual File System API - Knowledge Operating System * * Returns a cached VFS instance that is auto-initialized during brain.init(). * No separate initialization needed! * * @example After import * ```typescript * await brain.import('./data.xlsx', { vfsPath: '/imports/data' }) * // VFS ready immediately - no init() call needed! * const files = await brain.vfs.readdir('/imports/data') * ``` * * @example Direct VFS usage * ```typescript * await brain.init() // VFS auto-initialized here! * await brain.vfs.writeFile('/docs/readme.md', 'Hello World') * const content = await brain.vfs.readFile('/docs/readme.md') * ``` * * **Pattern:** The VFS instance is cached, so multiple calls to brain.vfs * return the same instance. This ensures import and user code share state. * */ get vfs(): VirtualFileSystem { if (!this._vfs) { // VFS is initialized during brain.init() // If not initialized yet, create instance but user should call brain.init() first this._vfs = new VirtualFileSystem(this) } // Warn if VFS accessed before init() completed if (!this._vfsInitialized && this.initialized) { console.warn('[Brainy] VFS accessed before initialization complete. Call await brain.init() first.') } return this._vfs } /** * Integration Hub for external tools (Excel, Power BI, Google Sheets) * * Provides HTTP endpoints that external tools can connect to: * - OData API for Excel Power Query, Power BI, Tableau * - REST API for Google Sheets custom functions * - SSE streaming for real-time dashboards * - Webhooks for push notifications * * Only available when `integrations: true` is set in config. * * @example Basic usage * ```typescript * const brain = new Brainy({ integrations: true }) * await brain.init() * * // Get endpoint URLs * console.log(brain.hub.endpoints) * // { odata: '/odata', sheets: '/sheets', sse: '/events', webhooks: '/webhooks' } * * // Handle requests (use with Express, Hono, etc.) * app.all('/odata/*', async (req, res) => { * const response = await brain.hub.handleRequest({ * method: req.method, * path: req.path, * query: req.query, * headers: req.headers, * body: req.body * }) * res.status(response.status).set(response.headers).json(response.body) * }) * ``` * * @throws Error if integrations are not enabled in config */ get hub(): IntegrationHub { if (!this._hub) { throw new Error( 'Integration Hub not enabled. Set integrations: true in config:\n' + 'new Brainy({ integrations: true })' ) } return this._hub } /** * @description Subscribe to this brain's committed mutations — the * authoritative in-process change feed. Fires exactly once per affected * record for EVERY canonical write, regardless of origin: direct API calls, * batch methods, `transact()`, imports, the Virtual Filesystem, and * native-accelerated deployments all funnel through the same commit point * this feed is emitted from. * * Delivery contract (see {@link BrainyChangeEvent}): * - **Post-commit only** — an aborted write (e.g. a losing `ifRev` * compare-and-swap) never emits. * - **Commit-ordered, asynchronous** — events arrive in the order writes * became durable, dispatched in a microtask so a slow listener never * delays a write. A throwing listener is isolated (logged, others * unaffected). * - **Fully described** — `remove`/`unrelate` events carry the record's * LAST committed state, and batch operations emit one event per item. * - **Fire-and-forget** — no replay or backpressure. Each event carries its * committed `generation`, so catch-up after a gap can be built on * {@link transactionLog} / {@link asOf}. * - **Zero overhead when unused** — with no subscribers the write path does * no event work at all. * * @param listener - Called once per committed mutation. * @returns An unsubscribe function — call it when done (e.g. on teardown of * a pooled instance) to stop delivery. * @example * const off = brain.onChange((e) => { * if (e.kind === 'entity') console.log(e.op, e.entity?.type, e.id) * }) * await brain.add({ data: 'hello', type: 'document' }) // → "add document " * off() */ onChange(listener: ChangeListener): () => void { return this._changeFeed.subscribe(listener) } /** * Get Triple Intelligence System * Advanced pattern recognition and relationship analysis */ getTripleIntelligence(): TripleIntelligenceSystem { if (!this._tripleIntelligence) { // Use core components directly - no lazy loading needed. // TripleIntelligenceSystem speaks UUIDs; adapt the BigInt graph-index // contract through the coordinator's UUID-level helper so the int // conversion stays at this boundary. this._tripleIntelligence = new TripleIntelligenceSystem( this.metadataIndex, this.index, { getNeighbors: (id: string, direction?: 'in' | 'out' | 'both') => this.getNeighborUuids(id, { direction }), size: () => this.graphIndex.size() }, async (text: string) => this.embedder(text), this.storage ) } return this._tripleIntelligence } // ============= METADATA INTELLIGENCE API ============= /** * Get all indexed field names currently in the metadata index * Essential for dynamic query building and NLP field discovery */ async getAvailableFields(): Promise { await this.ensureInitialized() return this.metadataIndex.getFilterFields() } /** * Get field statistics including cardinality and query patterns * Used for query optimization and understanding data distribution */ async getFieldStatistics(): Promise> { await this.ensureInitialized() return this.metadataIndex.getFieldStatistics() } /** * Get fields sorted by cardinality for optimal filtering * Lower cardinality fields are better for initial filtering */ async getFieldsWithCardinality(): Promise> { await this.ensureInitialized() return this.metadataIndex.getFieldsWithCardinality() } /** * Get optimal query plan for a given set of filters * Returns field processing order and estimated cost */ async getOptimalQueryPlan(filters: Record): Promise<{ strategy: 'exact' | 'range' | 'hybrid' fieldOrder: string[] estimatedCost: number }> { await this.ensureInitialized() return this.metadataIndex.getOptimalQueryPlan(filters) } /** * Get filter values for a specific field (for UI dropdowns, etc) */ async getFieldValues(field: string): Promise { await this.ensureInitialized() return this.metadataIndex.getFilterValues(field) } /** * Get fields that commonly appear with a specific entity type * Essential for type-aware NLP parsing */ async getFieldsForType(nounType: NounType): Promise> { await this.ensureInitialized() return this.metadataIndex.getFieldsForType(nounType) } /** * Create a streaming pipeline */ stream() { const { Pipeline } = require('./streaming/pipeline.js') return new Pipeline(this) } /** * Get insights about the data */ async insights(): Promise<{ entities: number relationships: number types: Record services: string[] density: number }> { await this.ensureInitialized() // O(1) entity counting using existing MetadataIndexManager const entities = this.metadataIndex.getTotalEntityCount() // O(1) count by type using existing index tracking const typeCountsMap = this.metadataIndex.getAllEntityCounts() const types: Record = Object.fromEntries(typeCountsMap) // O(1) relationships count using GraphAdjacencyIndex const relationships = this.graphIndex.getTotalRelationshipCount() // Get unique services - O(log n) using index const serviceValues = await this.metadataIndex.getFilterValues('service') const services = serviceValues.filter(Boolean) // Calculate density (relationships per entity) const density = entities > 0 ? relationships / entities : 0 return { entities, relationships, types, services, density } } /** * Flush all indexes and caches to persistent storage * CRITICAL FIX: Ensures data survives server restarts * * Flushes all 4 core indexes: * 1. Storage counts (entity/verb counts by type) * 2. Metadata index (field indexes + EntityIdMapper) * 3. Graph adjacency index (relationship cache) * 4. HNSW vector index (deferred dirty nodes) * * @example * // Flush after bulk operations * await brain.import('./data.xlsx') * await brain.flush() * * // Flush before shutdown * process.on('SIGTERM', async () => { * await brain.flush() * process.exit(0) * }) */ async flush(): Promise { await this.ensureInitialized() // Read-only instances have no buffered writes to flush. close() may call // flush() defensively, so we early-return instead of throwing. if (this.isReadOnly) { return } console.log('Flushing Brainy indexes and caches to disk...') const startTime = Date.now() // 8.0 MVCC: persist the in-memory pending single-op generation history to // disk first (async group-commit), so a flush makes every single-op write's // history durable, not just the live data. await this.generationStore.flushPendingSingleOps() // Flush all components in parallel for performance await Promise.all([ // 1. Flush storage adapter counts (entity/verb counts by type) (async () => { if (this.storage && typeof this.storage.flushCounts === 'function') { await this.storage.flushCounts() } })(), // 2. Flush metadata index (field indexes + EntityIdMapper) this.metadataIndex.flush(), // 3. Flush graph adjacency index (relationship cache + LSM trees) this.graphIndex.flush(), // 4. Flush HNSW dirty nodes (deferred persistence mode) (async () => { if (this.index && typeof this.index.flush === 'function') { await this.index.flush() } })(), // 5. Persist the generation counter (8.0 MVCC — coalesced single-op // bumps become durable on every explicit flush) this.generationStore.persistCounterNow() ]) // 6. Auto-compact generational history per config.retention (default on). // Runs after the flush so durable state is in place; respects live // Db pins and an explicit autoCompact: false. await this.autoCompactHistory() // 7. Stamp the entity tree: which source generation the canonical tree // reflects + the rollup invariants that verify it whole (the counters // persisted in step 1). Written at flush boundaries — the tree tracks // every commit by construction, so the stamp is a durable checkpoint, // not a per-commit cost. Open compares stamp vs log head + rollups. await this.stampEntityTree() const elapsed = Date.now() - startTime console.log(`All indexes flushed to disk in ${elapsed}ms`) } /** * @description Write the entity tree's FAMILY STAMP: `sourceGeneration` (the * committed generation the canonical tree reflects — equal by construction, * since the tree is written by the commit itself) plus the rollup invariants * (entity/relationship counts) that verify the tree whole where per-file * checks cannot scale. Verified at open by {@link verifyEntityTreeStamp}; * healed by `repairIndex()`, whose unconditional recount rebuilds the * rollups from a canonical walk and re-stamps. Best-effort: a stamp-write * fault warns loudly but never fails the flush that carried real data. */ private async stampEntityTree(): Promise { if (this.isReadOnly) return try { const [nounCount, verbCount] = await Promise.all([ this.storage.getNounCount(), this.storage.getVerbCount() ]) await writeFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH, { family: 'entity-tree', sourceGeneration: this.generationStore.generation(), members: { mode: 'rollup', invariants: { nounCount, verbCount } } }) } catch (error) { prodLog.warn( `[Brainy] entity-tree stamp write failed (coherence checking degrades until the ` + `next successful flush): ${(error as Error).message}` ) } } /** * @description Open-time coherence check for the entity tree's family stamp: * compare `sourceGeneration` against the log head and the stamped rollup * invariants against the live counters. Verdicts: * - `coherent` / `absent` (legacy store; first flush stamps) → silent. * - `behind` → benign for the tree (it is written BY the commit; only the * stamp is stale — a crash landed between commit and flush). Refreshed at * the next flush. * - `incoherent` → LOUD: the tree or its counters diverged from what was * stamped — `repairIndex()` recounts from canonical and re-stamps. * Never blocks open; a fault reading the stamp is surfaced as unverifiable, * never conflated with absence. */ private async verifyEntityTreeStamp(): Promise { let stamp: FamilyStamp | null try { stamp = await readFamilyStamp(this.storage, ENTITY_TREE_STAMP_PATH) } catch (error) { prodLog.warn( `[Brainy] entity-tree stamp is UNVERIFIABLE (read fault, not absence): ` + `${(error as Error).message}` ) return } const [nounCount, verbCount] = await Promise.all([ this.storage.getNounCount(), this.storage.getVerbCount() ]) const verdict = verifyFamilyStamp(stamp, this.generationStore.generation(), { nounCount, verbCount }) if (verdict.state === 'incoherent') { prodLog.warn( `[Brainy] entity-tree stamp INCOHERENT at open: ${verdict.failures.join('; ')}. ` + `The canonical tree or its counters diverged from the stamped state — run ` + `brain.repairIndex() to recount from canonical and re-stamp.` ) } else if (verdict.state === 'behind') { prodLog.debug( `[Brainy] entity-tree stamp is behind the log head (${verdict.stampSource} < ` + `${verdict.head}) — benign (stamped at last flush); refreshes at the next flush.` ) } } /** * Ask the writer process serving this data directory to flush its in-memory * indexes to disk, so a read-only inspector can observe fresh state. * * - **Same-process call** (this instance owns the writer lock): equivalent * to `this.flush()`. * - **Different process** (typical inspector flow): writes a request file * into the lock directory and polls for an ack file. The writer's * flush-request watcher (started in `init()` for writer instances) sees * the request, calls `flush()`, and writes the ack. * - **No writer running**: times out and returns `false`. Callers can * proceed with last-flushed disk state and warn the operator. * * @param options.timeoutMs - How long to wait for an ack (default 5000ms). * @returns `true` if the writer flushed in response to this request, * `false` on timeout (no writer or unresponsive). * * @example * ```typescript * const reader = await Brainy.openReadOnly({ storage: { type: 'filesystem', path: '/data/brain' } }) * const fresh = await reader.requestFlush({ timeoutMs: 3000 }) * if (!fresh) { * console.warn('Writer did not respond; results reflect last natural flush.') * } * const bookings = await reader.find({ where: { entityType: 'booking' } }) * ``` */ async requestFlush(options?: { timeoutMs?: number }): Promise { await this.ensureInitialized() const timeoutMs = options?.timeoutMs ?? 5000 // In-process shortcut: if this instance can write, just flush directly. if (!this.isReadOnly) { await this.flush() return true } if (typeof this.storage.requestFlushOverFilesystem !== 'function') { return false } return this.storage.requestFlushOverFilesystem(timeoutMs) } /** * Get index loading status (Diagnostic for lazy loading) * * Returns detailed information about index population and lazy loading state. * Useful for debugging empty query results or performance troubleshooting. * * @example * ```typescript * const status = await brain.getIndexStatus() * console.log(`HNSW Index: ${status.hnswIndex.size} entities`) * console.log(`Metadata Index: ${status.metadataIndex.entries} entries`) * console.log(`Graph Index: ${status.graphIndex.relationships} relationships`) * console.log(`Lazy rebuild completed: ${status.lazyRebuildCompleted}`) * ``` */ async getIndexStatus(): Promise<{ initialized: boolean lazyRebuildCompleted: boolean disableAutoRebuild: boolean /** `true` while a native provider runs the one-time 7.x → 8.0 rebuild LOCK. * A readiness probe should map this to HTTP 503 + Retry-After (transiently * not-ready), NOT 200-ready and NOT 500-broken. Never gated by the lock. */ migrating: boolean /** Structured progress while {@link migrating}; absent otherwise. */ migration?: MigrationProgress /** `true` when a non-fatal index rebuild failed at init() (degraded — queries * may be incomplete). Folds in the same `_indexRebuildFailed` signal that * {@link validateIndexConsistency} / {@link checkHealth} already expose. */ rebuildFailed: boolean /** The rebuild failure message when {@link rebuildFailed}; absent otherwise. */ rebuildError?: string /** Count of records committed via adopt-forward failed-rollback recovery whose * derived index may be incomplete until repairIndex() (the 8.2.6 degraded * set). Non-zero = degraded — a readiness probe should not report 200-ready. */ degradedIds: number hnswIndex: { size: number /** Honest "serving" — the provider's `isReady()` when exposed, else `size>0`. */ populated: boolean /** The provider's honest `isReady()` signal; absent when unexposed. */ ready?: boolean } metadataIndex: { entries: number populated: boolean ready?: boolean } graphIndex: { relationships: number populated: boolean ready?: boolean } storage: { totalEntities: number } }> { // Callable before init() — a readiness/liveness probe may fire during // construction or a fired-not-awaited init(). Report a safe not-initialized // snapshot rather than dereferencing providers that init() has not assigned. if (!this.initialized || this.metadataIndex == null || this.index == null || this.graphIndex == null) { return { initialized: false, lazyRebuildCompleted: this.lazyRebuildCompleted, disableAutoRebuild: this.config.disableAutoRebuild || false, migrating: false, rebuildFailed: this._indexRebuildFailed != null, ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}), degradedIds: this._indexDegradedIds.size, hnswIndex: { size: 0, populated: false }, metadataIndex: { entries: 0, populated: false }, graphIndex: { relationships: 0, populated: false }, storage: { totalEntities: 0 } } } const metadataStats = await this.metadataIndex.getStats() const hnswSize = this.index.size() const graphSize = await this.graphIndex.size() // Honest readiness: when a provider exposes isReady(), it is the truth of // whether the index actually SERVES (a native index can report a non-zero // size/count yet not have loaded its serving structure). `populated` reflects // that honest signal when present, falling back to size>0 only for providers // that do not expose isReady(). Mirrors validateIndexConsistency/checkHealth, // which already fold in the same signals. const readyOf = (p: unknown): boolean | undefined => { const r = assessIndexReadiness(p) return r === 'unknown' ? undefined : r === 'ready' } const hnswReady = readyOf(this.index) const metadataReady = readyOf(this.metadataIndex) const graphReady = readyOf(this.graphIndex) // Check storage entity count let storageEntityCount = 0 try { const entities = await this.storage.getNouns({ pagination: { limit: 1 } }) storageEntityCount = entities.totalCount || 0 } catch (e) { // Ignore errors } return { initialized: this.initialized, lazyRebuildCompleted: this.lazyRebuildCompleted, disableAutoRebuild: this.config.disableAutoRebuild || false, // A non-fatal index-rebuild failure recorded at init(), or adopt-forward // degraded ids, are degraded states (queries may be incomplete) — surface // them here alongside the same signals validateIndexConsistency()/ // checkHealth() already expose, so a readiness probe never reports 200-ready // over a known-degraded index. rebuildFailed: this._indexRebuildFailed != null, ...(this._indexRebuildFailed ? { rebuildError: this._indexRebuildFailed.message } : {}), degradedIds: this._indexDegradedIds.size, ...this.migrationSnapshot(), hnswIndex: { size: hnswSize, populated: hnswReady ?? hnswSize > 0, ...(hnswReady !== undefined ? { ready: hnswReady } : {}) }, metadataIndex: { entries: metadataStats.totalEntries, populated: metadataReady ?? metadataStats.totalEntries > 0, ...(metadataReady !== undefined ? { ready: metadataReady } : {}) }, graphIndex: { relationships: graphSize, populated: graphReady ?? graphSize > 0, ...(graphReady !== undefined ? { ready: graphReady } : {}) }, storage: { totalEntities: storageEntityCount } } } /** * Run a battery of cheap invariant checks suitable for an operator-facing * health probe. Reports counts of suspicious states alongside the basic * size invariants — designed so an incident responder can spot the typical * failure modes (mismatched index sizes, lots of `_seeded` records hanging * around, missing field stats) without scanning the whole store. * * @example * ```typescript * const h = await brain.health() * for (const c of h.checks) { * console.log(`${c.status === 'pass' ? '✓' : '!'} ${c.name}: ${c.message}`) * } * ``` */ async health(): Promise<{ overall: 'pass' | 'warn' | 'fail' checks: Array<{ name: string status: 'pass' | 'warn' | 'fail' message: string details?: Record }> }> { // Lock-exempt: an operator must be able to probe health WHILE the brain // upgrades — that is the whole point of an observable migration. await this.ensureInitialized({ bypassMigrationLock: true }) const checks: Array<{ name: string status: 'pass' | 'warn' | 'fail' message: string details?: Record }> = [] // Migration LOCK (#18): while a native provider rebuilds the derived indexes, // the parity/field checks below would report transient mismatches that read // as failures but are not — surface the upgrade as the single honest signal. // `warn` (not `fail`) so a readiness probe treats it as "not ready yet, retry". const migSnap = this.migrationSnapshot() if (migSnap.migrating) { const pct = migSnap.migration?.percent return { overall: 'warn', checks: [ { name: 'migration', status: 'warn', message: `Brain is upgrading (7.x → 8.0)${pct !== undefined ? `, ${Math.round(pct)}% complete` : ''}. ` + 'Reads and writes are blocked until it completes; retry shortly.', details: { ...migSnap.migration } } ] } } const hnswSize = this.index.size() const metadataStats = await this.metadataIndex.getStats() const graphSize = await this.graphIndex.size() // 1. Index size parity. HNSW must hold at least one node per indexed entity. if (hnswSize === metadataStats.totalEntries) { checks.push({ name: 'index-parity', status: 'pass', message: `HNSW (${hnswSize}) and metadata index (${metadataStats.totalEntries}) agree.`, details: { hnswSize, metadataEntries: metadataStats.totalEntries, graphRelationships: graphSize } }) } else { const drift = Math.abs(hnswSize - metadataStats.totalEntries) checks.push({ name: 'index-parity', status: drift > Math.max(10, metadataStats.totalEntries * 0.01) ? 'fail' : 'warn', message: `HNSW (${hnswSize}) and metadata (${metadataStats.totalEntries}) differ by ${drift}. Run a rebuild if the gap is unexpected.`, details: { hnswSize, metadataEntries: metadataStats.totalEntries, drift } }) } // 2. Field registry sanity. Empty field set + non-zero entities = stale reader. let fieldCount = 0 try { if (typeof this.metadataIndex.getFieldStatistics === 'function') { const fieldStats = await this.metadataIndex.getFieldStatistics() fieldCount = fieldStats.size } } catch { // ignore — metadata index doesn't expose stats } if (metadataStats.totalEntries > 0 && fieldCount === 0) { checks.push({ name: 'field-registry', status: 'warn', message: `${metadataStats.totalEntries} entities present but the field registry is empty. Reader may be stale; consider requestFlush().`, details: { entities: metadataStats.totalEntries, fields: fieldCount } }) } else { checks.push({ name: 'field-registry', status: 'pass', message: `${fieldCount} fields registered for ${metadataStats.totalEntries} entities.`, details: { fields: fieldCount, entities: metadataStats.totalEntries } }) } // 3. Seeded entity sweep — operators often want to know if demo seed data // is still in a brain that's also serving real traffic (a common root // cause of duplicate-ID and stale-content bugs). Uses the metadata index, // not a full scan. let seededIds: string[] = [] try { seededIds = await this.metadataIndex.getIds('_seeded', true) } catch { // ignore — field may not be indexed } if (seededIds.length > 0) { checks.push({ name: 'seeded-records', status: 'warn', message: `${seededIds.length} entities tagged _seeded:true. Verify this is the demo data you expect.`, details: { count: seededIds.length, sample: seededIds.slice(0, 5) } }) } else { checks.push({ name: 'seeded-records', status: 'pass', message: 'No _seeded:true entities found.' }) } // 4. Lock heartbeat freshness — only meaningful for readers inspecting a // running writer. If the lock exists but the heartbeat is old, the writer // probably crashed. if (typeof this.storage.readWriterLock === 'function') { const lock = await this.storage.readWriterLock() if (lock) { const age = Date.now() - new Date(lock.lastHeartbeat).getTime() if (age > 60_000) { checks.push({ name: 'writer-heartbeat', status: 'warn', message: `Writer lock heartbeat is ${Math.round(age / 1000)}s old (PID ${lock.pid} on ${lock.hostname}). Writer may be hung.`, details: { lock, ageMs: age } }) } else { checks.push({ name: 'writer-heartbeat', status: 'pass', message: `Writer healthy (PID ${lock.pid} on ${lock.hostname}, heartbeat ${Math.round(age / 1000)}s ago).`, details: { lock } }) } } } const worst = checks.reduce<'pass' | 'warn' | 'fail'>((acc, c) => { if (c.status === 'fail') return 'fail' if (c.status === 'warn' && acc !== 'fail') return 'warn' return acc }, 'pass') return { overall: worst, checks } } /** * Explain how a `find` query's `where` clause will be served. For each * field, returns whether it will hit the column store (best), a sparse * chunked index (legacy fallback), or has no index entries at all (silently * empty result — usually a bug or a stale reader). Designed to be the very * first thing an operator runs when `find()` returns surprising results. * * @example * ```typescript * const plan = await brain.explain({ where: { entityType: 'booking', status: 'paid' } }) * for (const f of plan.fieldPlan) { * console.log(`${f.field} -> ${f.path}: ${f.notes ?? ''}`) * } * ``` */ async explain(params: FindParams): Promise<{ query: FindParams fieldPlan: Array<{ field: string; path: 'column-store' | 'sparse-chunked' | 'none'; notes?: string }> warnings: string[] }> { await this.ensureInitialized() const fieldPlan: Array<{ field: string; path: 'column-store' | 'sparse-chunked' | 'none'; notes?: string }> = [] const warnings: string[] = [] const where = params?.where if (where && typeof where === 'object') { for (const field of Object.keys(where)) { const result = await this.metadataIndex.explainField(field) fieldPlan.push({ field, path: result.path, notes: result.notes }) if (result.path === 'none') { warnings.push( `Field "${field}" has no index entries. find() will return [] silently. ` + `Run brain.requestFlush() or check the writer's field registry.` ) } } } else { warnings.push('No `where` clause provided; nothing to explain.') } return { query: params, fieldPlan, warnings } } /** * Operator-facing summary of what's in this Brainy store. Designed to be * the first thing a human runs during an incident: counts, mode, lock owner, * indexed field list, index health flags. * * Safe to call on a read-only instance. All counts come from already-loaded * indexes (no extra storage scans), so this is fast (<10ms typical). * * @example * ```typescript * const s = await brain.stats() * console.log(`${s.entityCount} entities (${Object.entries(s.entitiesByType).map(([t,n])=>`${t}:${n}`).join(', ')})`) * if (s.writerLock) { * console.log(`Writer PID ${s.writerLock.pid} on ${s.writerLock.hostname}`) * } * ``` */ async stats(): Promise { await this.ensureInitialized() const { NounTypeEnum, VerbTypeEnum } = await import('./types/graphTypes.js') const nounCounts = typeof this.storage.getNounCountsByType === 'function' ? this.storage.getNounCountsByType() : new Uint32Array(0) const verbCounts = typeof this.storage.getVerbCountsByType === 'function' ? this.storage.getVerbCountsByType() : new Uint32Array(0) const entitiesByType: Record = {} let entityCount = 0 for (let i = 0; i < nounCounts.length; i++) { if (nounCounts[i] === 0) continue const name = NounTypeEnum[i as number] if (name) entitiesByType[name] = nounCounts[i] entityCount += nounCounts[i] } const relationsByType: Record = {} let relationCount = 0 for (let i = 0; i < verbCounts.length; i++) { if (verbCounts[i] === 0) continue const name = VerbTypeEnum[i as number] if (name) relationsByType[name] = verbCounts[i] relationCount += verbCounts[i] } const metadataStats = await this.metadataIndex.getStats() let fieldRegistry: string[] = [] try { if (typeof this.metadataIndex.getFieldStatistics === 'function') { const fieldStats = await this.metadataIndex.getFieldStatistics() fieldRegistry = Array.from(fieldStats.keys()).sort() } } catch { // Field stats unavailable on this metadata index implementation — leave empty. } const writerLock = typeof this.storage.readWriterLock === 'function' ? await this.storage.readWriterLock() : null const storageBackend = this.storage.constructor.name // FileSystemStorage keeps its root directory in a `rootDir` member that // BaseStorage doesn't declare; surfaced opportunistically for operators. const rootDir = (this.storage as BaseStorage & { rootDir?: unknown }).rootDir return { mode: this.isReadOnly ? 'reader' : 'writer', entityCount, entitiesByType, relationCount, relationsByType, fieldRegistry, indexHealth: await (async () => { const graphSize = await this.graphIndex.size() return { vector: this.index.size() > 0 || entityCount === 0, metadata: metadataStats.totalEntries > 0 || entityCount === 0, graph: graphSize > 0 || relationCount === 0 } })(), storage: { backend: storageBackend, rootDir: typeof rootDir === 'string' ? rootDir : undefined }, writerLock: writerLock || undefined, version: getBrainyVersion() } } /** * Plugin and provider diagnostics — shows what's active and how subsystems are wired. * * @example * ```typescript * const diag = brain.diagnostics() * console.log(diag.providers) // { vector: { source: 'plugin' }, ... } * console.log(diag.indexes.graph.wiredToStorage) // true * ``` */ diagnostics(): DiagnosticsResult { const wellKnownKeys = [ 'metadataIndex', 'graphIndex', 'entityIdMapper', 'cache', 'vector', 'roaring', 'embeddings', 'embedBatch', 'distance', 'msgpack' ] as const const providers: Record = {} for (const key of wellKnownKeys) { providers[key] = { source: this.pluginRegistry.hasProvider(key) ? 'plugin' : 'default' } } const hnswSize = this.index.size() const metadataInitialized = !!this.metadataIndex const graphInitialized = !!this.graphIndex // Boundary: identity-compares against BaseStorage's protected `graphIndex` // member to report whether the live graph index is the storage-wired one. // Diagnostics-only peek — the public accessor (`getGraphIndex()`) is async // and lazily *creates* the index, which would defeat the wiring check. const storageGraphIndex = (this.storage as unknown as { graphIndex?: GraphAdjacencyIndex }).graphIndex return { version: getBrainyVersion(), plugins: { active: this.pluginRegistry.getActivePlugins(), count: this.pluginRegistry.getActivePlugins().length }, providers, indexes: { hnsw: { size: hnswSize, type: this.index.constructor.name }, metadata: { type: this.metadataIndex?.constructor.name || 'none', initialized: metadataInitialized }, graph: { type: this.graphIndex?.constructor.name || 'none', initialized: graphInitialized, wiredToStorage: graphInitialized && storageGraphIndex === this.graphIndex } } } } /** * Assert that specific providers are supplied by a plugin (not using JS fallback). * * Call after init() in production to fail fast if a paid plugin (e.g. cor) * isn't providing the expected acceleration. Throws if any listed key is using * the default JavaScript implementation. * * @param keys - Provider keys that MUST come from a plugin * @throws Error listing which providers are falling back to defaults * * @example * ```typescript * const brain = new Brainy() * await brain.init() * * // Fail fast if cor isn't providing these * brain.requireProviders(['distance', 'embeddings', 'metadataIndex', 'graphIndex']) * ``` */ requireProviders(keys: string[]): void { const missing = keys.filter(k => !this.pluginRegistry.hasProvider(k)) if (missing.length > 0) { const active = this.pluginRegistry.getActivePlugins() const pluginInfo = active.length > 0 ? `Active plugins: ${active.join(', ')}` : 'No plugins active' throw new Error( `[brainy] Required providers using JS fallback: ${missing.join(', ')}. ` + `${pluginInfo}. ` + `These providers must be supplied by a plugin for this deployment. ` + `Check plugin installation, license, and native module availability.` ) } } /** * Efficient Pagination API - Production-scale pagination using index-first approach * Automatically optimizes based on query type and applies pagination at the index level */ get pagination() { return { // Get paginated results with automatic optimization find: async (params: FindParams & { page?: number, pageSize?: number }) => { const page = params.page || 1 const pageSize = params.pageSize || 10 const offset = (page - 1) * pageSize return this.find({ ...params, limit: pageSize, offset }) }, // Get total count for pagination UI (O(1) when possible) count: async (params: Omit, 'limit' | 'offset'>) => { // For simple type queries, use O(1) index counting if (params.type && !params.subtype && !params.query && !params.where && !params.connected) { const types = Array.isArray(params.type) ? params.type : [params.type] return types.reduce((sum, type) => sum + this.metadataIndex.getEntityCountByType(type), 0) } // For complex queries, use metadata index for efficient counting if (params.where || params.subtype || params.service) { let filter: any = {} if (params.where) { Object.assign(filter, params.where) // Alias: where.type → where.noun (storage field name for entity type) if ('type' in filter && !('noun' in filter)) { filter.noun = filter.type delete filter.type } } if (params.service) filter.service = params.service if (params.subtype !== undefined) { filter.subtype = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype } if (params.type) { const types = Array.isArray(params.type) ? params.type : [params.type] if (types.length === 1) { filter.noun = types[0] } else { const baseFilter = { ...filter } filter = { anyOf: types.map(type => ({ noun: type, ...baseFilter })) } } } const filteredIds = await this.metadataIndex.getIdsForFilter(filter) return filteredIds.length } // Fallback: total entity count return this.metadataIndex.getTotalEntityCount() }, // Get pagination metadata meta: async (params: FindParams & { page?: number, pageSize?: number }) => { const page = params.page || 1 const pageSize = params.pageSize || 10 const totalCount = await this.pagination.count(params) const totalPages = Math.ceil(totalCount / pageSize) return { page, pageSize, totalCount, totalPages, hasNext: page < totalPages, hasPrev: page > 1 } } } } /** * Streaming API - Process millions of entities with constant memory using existing Pipeline * Integrates with index-based optimizations for maximum efficiency */ get streaming(): { entities: (filter?: Partial>) => AsyncGenerator> search: (params: FindParams, batchSize?: number) => AsyncGenerator<{ id: string; score: number; entity: Entity }> relationships: (filter?: { type?: string; sourceId?: string; targetId?: string }) => AsyncGenerator pipeline: (source: AsyncIterable) => any process: (processor: (entity: Entity) => Promise>, filter?: Partial>, options?: { batchSize: number; parallel: number }) => Promise } { return { // Stream all entities with optional filtering entities: async function* (this: Brainy, filter?: Partial>) { if (filter?.type || filter?.subtype || filter?.where || filter?.service) { // Use MetadataIndexManager for efficient filtered streaming let filterObj: any = {} if (filter.where) { Object.assign(filterObj, filter.where) // Alias: where.type → where.noun (storage field name for entity type) if ('type' in filterObj && !('noun' in filterObj)) { filterObj.noun = filterObj.type delete filterObj.type } } if (filter.service) filterObj.service = filter.service if (filter.subtype !== undefined) { filterObj.subtype = Array.isArray(filter.subtype) ? { oneOf: filter.subtype } : filter.subtype } if (filter.type) { const types = Array.isArray(filter.type) ? filter.type : [filter.type] if (types.length === 1) { filterObj.noun = types[0] } else { const baseFilterObj = { ...filterObj } filterObj = { anyOf: types.map(type => ({ noun: type, ...baseFilterObj })) } } } const filteredIds = await this.metadataIndex.getIdsForFilter(filterObj) // Stream filtered entities in batches for memory efficiency const batchSize = 100 for (let i = 0; i < filteredIds.length; i += batchSize) { const batchIds = filteredIds.slice(i, i + batchSize) for (const id of batchIds) { const entity = await this.get(id) if (entity) yield entity as Entity } } } else { // Stream all entities using storage adapter pagination let offset = 0 const batchSize = 100 let hasMore = true while (hasMore) { const result = await this.storage.getNouns({ pagination: { offset, limit: batchSize } }) for (const noun of result.items) { // Convert HNSWNoun to Entity yield noun as unknown as Entity } hasMore = result.hasMore offset += batchSize } } }.bind(this), // Stream search results efficiently search: async function* (this: Brainy, params: FindParams, batchSize = 50) { const originalLimit = params.limit let offset = 0 let hasMore = true while (hasMore) { const batchResults = await this.find({ ...params, limit: batchSize, offset }) for (const result of batchResults) { yield result } hasMore = batchResults.length === batchSize offset += batchSize // Respect original limit if specified if (originalLimit && offset >= originalLimit) { break } } }.bind(this), // Stream relationships efficiently. Cursor-based when the adapter supports it // (resumes after the last verb — O(N) for a full walk) and falls back to offset // otherwise. Offset paging here re-scanned from the start every page (O(N²)). relationships: async function* (this: Brainy, filter?: { type?: string, sourceId?: string, targetId?: string }) { let offset = 0 let cursor: string | undefined const batchSize = 100 for (;;) { const result = await this.storage.getVerbs({ pagination: cursor ? { limit: batchSize, cursor } : { offset, limit: batchSize }, filter }) for (const verb of result.items) { yield verb } if (!result.hasMore || result.items.length === 0) break if (result.nextCursor) { cursor = result.nextCursor } else { offset += result.items.length } } }.bind(this), // Create processing pipeline from stream pipeline: (source: AsyncIterable) => { return createPipeline(this).source(source) }, // Batch process entities with Pipeline system process: async function (this: Brainy, processor: (entity: Entity) => Promise>, filter?: Partial>, options = { batchSize: 50, parallel: 4 } ) { return createPipeline(this) .source(this.streaming.entities(filter)) .batch(options.batchSize) .parallelSink(async (batch: Entity[]) => { await Promise.all(batch.map(processor)) }, options.parallel) .run() }.bind(this) } } /** * O(1) Count API - Production-scale counting using existing indexes * Works across all storage adapters (FileSystem, OPFS, S3, Memory) * * Phase 1b Enhancement: Type-aware methods with 99.2% memory reduction */ get counts() { return { // O(1) total entity count entities: () => this.metadataIndex.getTotalEntityCount(), // O(1) total relationship count relationships: () => this.graphIndex.getTotalRelationshipCount(), // O(1) count by type (string-based, backward compatible) // Added optional excludeVFS using Roaring bitmap intersection byType: async (typeOrOptions?: string | { excludeVFS?: boolean }, options?: { excludeVFS?: boolean }) => { // Handle overloaded signature: byType(type), byType({ excludeVFS }), byType(type, { excludeVFS }) let type: string | undefined let excludeVFS = false if (typeof typeOrOptions === 'string') { type = typeOrOptions excludeVFS = options?.excludeVFS ?? false } else if (typeOrOptions && typeof typeOrOptions === 'object') { excludeVFS = typeOrOptions.excludeVFS ?? false } if (excludeVFS) { const allCounts = this.metadataIndex.getAllEntityCounts() // Uses Roaring bitmap intersection - hardware accelerated const vfsCounts = await this.metadataIndex.getAllVFSEntityCounts() if (type) { const total = allCounts.get(type) || 0 const vfs = vfsCounts.get(type) || 0 return total - vfs } // Return all counts with VFS subtracted const result: Record = {} for (const [t, total] of allCounts) { const vfs = vfsCounts.get(t) || 0 const nonVfs = total - vfs if (nonVfs > 0) { result[t] = nonVfs } } return result } // Default path (unchanged) - synchronous for backward compatibility if (type) { return this.metadataIndex.getEntityCountByType(type) } return Object.fromEntries(this.metadataIndex.getAllEntityCounts()) }, // Phase 1b: O(1) count by type enum (Uint32Array-based, more efficient) // Uses fixed-size type tracking: 676 bytes vs ~35KB with Maps (98.1% reduction) byTypeEnum: (type: NounType) => { return this.metadataIndex.getEntityCountByTypeEnum(type) }, // Phase 1b: Get top N noun types by entity count (useful for cache warming) topTypes: (n: number = 10) => { return this.metadataIndex.getTopNounTypes(n) }, /** * O(1) subtype counts for a given NounType. * * Returns the count for a single (type, subtype) pair when `subtype` is * passed; returns the full subtype → count map for that NounType when omitted. * Backed by the persisted `_system/subtype-statistics.json` rollup — no * scan, no storage round-trip. * * @param type - The NounType to count subtypes within * @param subtype - Optional specific subtype string for O(1) point count * @returns A number when `subtype` is given, otherwise a `Record` (empty `{}` if none) * * @example Get all subtypes of Person * const counts = brain.counts.bySubtype(NounType.Person) * // → { employee: 56, customer: 847, vendor: 34 } * * @example O(1) point count * const employees = brain.counts.bySubtype(NounType.Person, 'employee') * // → 56 */ bySubtype: (type: NounType, subtype?: string): number | Record => { const subtypeMap = typeof this.storage.getSubtypeCountsByType === 'function' ? this.storage.getSubtypeCountsByType() : null if (!subtypeMap) { return subtype !== undefined ? 0 : {} } const typeIdx = TypeUtils.getNounIndex(type) const inner = subtypeMap.get(typeIdx) if (!inner) { return subtype !== undefined ? 0 : {} } if (subtype !== undefined) { return inner.get(subtype) || 0 } const result: Record = {} for (const [k, v] of inner.entries()) result[k] = v return result }, /** * Top N subtypes for a NounType, sorted by count (descending). * * @param type - The NounType to rank subtypes within * @param n - Maximum number of (subtype, count) pairs to return (default: 10) * @returns Array of `[subtype, count]` tuples, highest count first * * @example * const top3 = brain.counts.topSubtypes(NounType.Person, 3) * // → [['customer', 847], ['employee', 56], ['vendor', 34]] */ topSubtypes: (type: NounType, n: number = 10): Array<[string, number]> => { const subtypeMap = typeof this.storage.getSubtypeCountsByType === 'function' ? this.storage.getSubtypeCountsByType() : null if (!subtypeMap) return [] const typeIdx = TypeUtils.getNounIndex(type) const inner = subtypeMap.get(typeIdx) if (!inner) return [] return Array.from(inner.entries()) .sort((a, b) => b[1] - a[1]) .slice(0, n) }, // Phase 1b: Get top N verb types by count topVerbTypes: (n: number = 10) => { return this.metadataIndex.getTopVerbTypes(n) }, // Phase 1b: Get all noun type counts as typed Map // More efficient than byType() for type-aware queries allNounTypeCounts: () => { return this.metadataIndex.getAllNounTypeCounts() }, // Phase 1b: Get all verb type counts as typed Map allVerbTypeCounts: () => { return this.metadataIndex.getAllVerbTypeCounts() }, // O(1) count by relationship type byRelationshipType: (type?: string) => { if (type) { return this.graphIndex.getRelationshipCountByType(type) } return Object.fromEntries(this.graphIndex.getAllRelationshipCounts()) }, /** * O(1) subtype counts for a given VerbType. Verb-side mirror of * `bySubtype`. Returns the count for a single (verb, subtype) pair when * `subtype` is passed; returns the full subtype → count map when omitted. * Backed by the persisted `_system/verb-subtype-statistics.json` rollup. * * @param verb - The VerbType to count subtypes within * @param subtype - Optional specific subtype string for O(1) point count * * @example * brain.counts.byRelationshipSubtype(VerbType.ReportsTo) * // → { direct: 12, 'dotted-line': 3 } * * brain.counts.byRelationshipSubtype(VerbType.ReportsTo, 'direct') * // → 12 */ byRelationshipSubtype: (verb: VerbType, subtype?: string): number | Record => { const verbSubtypeMap = typeof this.storage.getVerbSubtypeCountsByType === 'function' ? this.storage.getVerbSubtypeCountsByType() : null if (!verbSubtypeMap) { return subtype !== undefined ? 0 : {} } const verbIdx = TypeUtils.getVerbIndex(verb) const inner = verbSubtypeMap.get(verbIdx) if (!inner) { return subtype !== undefined ? 0 : {} } if (subtype !== undefined) { return inner.get(subtype) || 0 } const result: Record = {} for (const [k, v] of inner.entries()) result[k] = v return result }, /** * Top N subtypes for a VerbType, sorted by count (descending). Mirror of * `topSubtypes` for verbs. * * @example * brain.counts.topRelationshipSubtypes(VerbType.ReportsTo, 5) * // → [['direct', 12], ['dotted-line', 3]] */ topRelationshipSubtypes: (verb: VerbType, n: number = 10): Array<[string, number]> => { const verbSubtypeMap = typeof this.storage.getVerbSubtypeCountsByType === 'function' ? this.storage.getVerbSubtypeCountsByType() : null if (!verbSubtypeMap) return [] const verbIdx = TypeUtils.getVerbIndex(verb) const inner = verbSubtypeMap.get(verbIdx) if (!inner) return [] return Array.from(inner.entries()) .sort((a, b) => b[1] - a[1]) .slice(0, n) }, // O(1) count by field-value criteria byCriteria: async (field: string, value: any) => { return this.metadataIndex.getCountForCriteria(field, value) }, /** * Counts by value for a field registered via `brain.trackField()`. Reads * from the backing `__fieldCounts__` aggregate (Layer 2 of the * subtype-and-facets primitive). Backfill-on-define means the first call * scans existing entities, subsequent calls are O(groups). * * Without `options.type`: returns the cross-NounType total per value. * With `options.type`: returns per-value counts for that NounType only — * requires the field to have been registered with `perType: true`. * * @param name - The tracked field name * @param options.type - Optional NounType filter (requires perType registration) * @returns `{ value: count }` map. Empty when the field wasn't tracked * (no aggregate exists) or no entities have set it yet. * @throws If `options.type` is passed but the field was not registered with `perType: true` * * @example * brain.trackField('status', { perType: true }) * await brain.add({ data: 'Ship it', type: NounType.Task, metadata: { status: 'todo' } }) * await brain.counts.byField('status') * // → { todo: 1 } * await brain.counts.byField('status', { type: NounType.Task }) * // → { todo: 1 } */ byField: async ( name: string, options?: { type?: NounType } ): Promise> => { const tracked = this._trackedFields.get(name) if (!tracked) return {} if (options?.type !== undefined && !tracked.perType) { throw new Error( `counts.byField('${name}'): per-type breakdown requested but the field was registered without perType:true. Re-call trackField('${name}', { perType: true }).` ) } const aggregateName = this.fieldCountsAggregateName(name) if (!this._aggregationIndex || !this._aggregationIndex.hasAggregate(aggregateName)) { return {} } const rows = await this.queryAggregate(aggregateName) const result: Record = {} for (const row of rows) { const value = row.groupKey?.[name] // Skip the aggregation engine's "missing-value" sentinel: entities that // don't have the tracked field at all (e.g. the VFS root) bucket under // '__null__' and would otherwise pollute the count map. if (value === undefined || value === null || value === '__null__') continue if (options?.type !== undefined && row.groupKey?.['noun'] !== options.type) continue const key = String(value) result[key] = (result[key] || 0) + (typeof row.metrics?.count === 'number' ? row.metrics.count : row.count) } return result }, // Get all type counts as Map for performance-critical operations getAllTypeCounts: () => this.metadataIndex.getAllEntityCounts(), // Get complete statistics // Added optional excludeVFS using Roaring bitmap intersection getStats: async (options?: { excludeVFS?: boolean }) => { if (options?.excludeVFS) { const allCounts = this.metadataIndex.getAllEntityCounts() // Uses Roaring bitmap intersection - hardware accelerated const vfsCounts = await this.metadataIndex.getAllVFSEntityCounts() // Compute non-VFS counts via subtraction const byType: Record = {} let total = 0 for (const [type, count] of allCounts) { const vfs = vfsCounts.get(type) || 0 const nonVfs = count - vfs if (nonVfs > 0) { byType[type] = nonVfs total += nonVfs } } const entityStats = { total, byType } const relationshipStats = this.graphIndex.getRelationshipStats() return { entities: entityStats, relationships: relationshipStats, density: total > 0 ? relationshipStats.totalRelationships / total : 0 } } // Default path (unchanged) - synchronous for backward compatibility const entityStats = { total: this.metadataIndex.getTotalEntityCount(), byType: Object.fromEntries(this.metadataIndex.getAllEntityCounts()) } const relationshipStats = this.graphIndex.getRelationshipStats() return { entities: entityStats, relationships: relationshipStats, density: entityStats.total > 0 ? relationshipStats.totalRelationships / entityStats.total : 0 } } } } /** * Get complete statistics - convenience method * For more granular counting, use brain.counts API * Added optional excludeVFS using Roaring bitmap intersection * @param options Optional settings - excludeVFS: filter out VFS entities * @returns Complete statistics including entities, relationships, and density */ async getStats(options?: { excludeVFS?: boolean }) { return this.counts.getStats(options) } /** * Distinct subtypes seen for a given NounType. * * Reads from the subtype-statistics rollup — no scan, no storage round-trip. * The returned list is the vocabulary actually observed in the data, not a * registered schema (Brainy doesn't validate subtype vocabulary; that's a * consumer concern). * * @param type - The NounType to enumerate subtypes for * @returns Sorted list of distinct subtype strings (empty if none) * * @example * const personSubtypes = brain.subtypesOf(NounType.Person) * // → ['customer', 'employee', 'vendor'] */ subtypesOf(type: NounType): string[] { const subtypeMap = typeof this.storage.getSubtypeCountsByType === 'function' ? this.storage.getSubtypeCountsByType() : null if (!subtypeMap) return [] const typeIdx = TypeUtils.getNounIndex(type) const inner = subtypeMap.get(typeIdx) if (!inner) return [] return Array.from(inner.keys()).sort() } /** * Distinct subtypes seen for a given VerbType. Mirror of `subtypesOf` for * relationships. Reads from the verb subtype-statistics rollup — no scan, no * storage round-trip. Vocabulary is observed (what's actually in the data), * not registered. * * @param verb - The VerbType to enumerate subtypes for * @returns Sorted list of distinct subtype strings (empty if none) * * @example * const variants = brain.relationshipSubtypesOf(VerbType.ReportsTo) * // → ['direct', 'dotted-line'] */ relationshipSubtypesOf(verb: VerbType): string[] { const verbSubtypeMap = typeof this.storage.getVerbSubtypeCountsByType === 'function' ? this.storage.getVerbSubtypeCountsByType() : null if (!verbSubtypeMap) return [] const verbIdx = TypeUtils.getVerbIndex(verb) const inner = verbSubtypeMap.get(verbIdx) if (!inner) return [] return Array.from(inner.keys()).sort() } /** * Find entities and relationships missing a `subtype` value, grouped by type. * * The diagnostic pair to `fillSubtypes()` / `migrateField()` — answers the * question "what would strict subtype enforcement reject?". 8.0 makes * `requireSubtype: true` the default, so run this when opening a pre-8.0 * brain (with `requireSubtype: false` as the temporary escape hatch), then * back-fill the reported gaps with `fillSubtypes(rules)`. * * Streams the brain via the same paginated `storage.getNouns()` / * `storage.getVerbs()` pattern `fillSubtypes()` uses — safe for large brains * but linear in `O(N)`. A native index provider may serve this from a * column-store null-subtype bitmap in the future for sub-linear performance * on billion-scale brains. * * @param options.includeVFS - When `false` (default), entities marked with * `metadata.isVFSEntity` or `metadata.isVFS` are excluded from the report — * these bypass enforcement anyway, so counting them is noise. Pass `true` * to include them. * @param options.batchSize - Pagination batch size (default `200`). * @param options.onProgress - Optional progress callback invoked after each batch. * @returns Report with per-type counts of entities/relationships without a * subtype, plus the overall total and a one-line recommendation pointing at * `fillSubtypes()`. * * @example Find pre-existing gaps before turning on strict mode * const report = await brain.audit() * if (report.total > 0) { * console.warn('Found ' + report.total + ' entities/edges without subtype:') * console.warn(report.entitiesWithoutSubtype) * console.warn(report.relationshipsWithoutSubtype) * } * * @since 7.30.1 */ async audit(options: { includeVFS?: boolean batchSize?: number onProgress?: (progress: { scanned: number; missingSubtype: number }) => void } = {}): Promise<{ entitiesWithoutSubtype: Record relationshipsWithoutSubtype: Record total: number scanned: number recommendation: string }> { await this.ensureInitialized() const includeVFS = options.includeVFS === true const batchSize = Math.max(1, options.batchSize ?? 200) const entitiesWithoutSubtype: Record = {} const relationshipsWithoutSubtype: Record = {} let scanned = 0 let missingSubtype = 0 const reportProgress = (): void => { if (options.onProgress) options.onProgress({ scanned, missingSubtype }) } // Scan nouns let offset = 0 while (true) { const page = await this.storage.getNouns({ pagination: { offset, limit: batchSize } }) if (page.items.length === 0) break for (const noun of page.items) { scanned++ // Legacy stored shapes carried the entity type under `noun` — read both. const n: HNSWNounWithMetadata & { noun?: NounType } = noun // Skip VFS infrastructure entities unless includeVFS is set — they // bypass enforcement via the isVFSEntity marker anyway, so listing // them in the report would mislead consumers into thinking they have // a migration gap they actually don't. if (!includeVFS && (n.metadata?.isVFSEntity === true || n.metadata?.isVFS === true)) continue const subtype = typeof n.subtype === 'string' ? n.subtype : undefined if (!subtype || subtype.length === 0) { missingSubtype++ const typeKey = String(n.type ?? n.noun ?? 'unknown') entitiesWithoutSubtype[typeKey] = (entitiesWithoutSubtype[typeKey] || 0) + 1 } } reportProgress() if (!page.hasMore) break offset += page.items.length } // Scan verbs offset = 0 while (true) { const page = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } }) if (page.items.length === 0) break for (const verb of page.items) { scanned++ // Legacy stored shapes carried the verb type under `type` — read both. const v: HNSWVerbWithMetadata & { type?: VerbType } = verb if (!includeVFS && (v.metadata?.isVFSEntity === true || v.metadata?.isVFS === true)) continue const subtype = typeof v.subtype === 'string' ? v.subtype : undefined if (!subtype || subtype.length === 0) { missingSubtype++ const verbKey = String(v.verb ?? v.type ?? 'unknown') relationshipsWithoutSubtype[verbKey] = (relationshipsWithoutSubtype[verbKey] || 0) + 1 } } reportProgress() if (!page.hasMore) break offset += page.items.length } const recommendation = missingSubtype === 0 ? 'No subtype gaps detected — this brain is strict-mode-ready.' : 'Found ' + missingSubtype + ' entries without subtype. Back-fill with `brain.fillSubtypes(rules)` — one rule per NounType/VerbType, literal default or per-entry function. To lift an existing field into `subtype` instead, use `brain.migrateField({ from, to: \'subtype\' })`.' return { entitiesWithoutSubtype, relationshipsWithoutSubtype, total: missingSubtype, scanned, recommendation } } /** * Back-fill missing `subtype` values across the whole brain — the 8.0 * migration helper for data written before subtype became required. * * 8.0 enforces a non-empty `subtype` on every write by default * (`requireSubtype: true`). A brain created on 7.x typically carries entities * and relationships without one; this method clears that debt in a single * idempotent pass so the opt-out (`requireSubtype: false`) can be removed. * The intended upgrade flow: * * 1. Open the brain with `requireSubtype: false` (temporary escape hatch). * 2. `await brain.audit()` — see what's missing, grouped by type. * 3. `await brain.fillSubtypes(rules)` — back-fill with one rule per type. * 4. Re-run `audit()` until `total === 0`, then drop the opt-out. * * **Rules.** One rule per NounType (entities) and/or VerbType * (relationships) — the two vocabularies don't overlap, so a single map * covers both sides. A rule is either a literal subtype string (blanket * default) or a function deriving the subtype from the entry; functions * subsume `where`-style filtering by returning `undefined` for entries they * decline (those stay untouched and count as `skipped`, so a later run with * a stricter rule can pick them up). * * **What is never touched:** entries that already carry a non-empty * `subtype` (re-running is a no-op on them), and — unless * `includeVFS: true` — Brainy's own VFS infrastructure entries * (`metadata.isVFSEntity` / `metadata.isVFS`), which bypass enforcement * anyway and are not migration debt. * * **Write strategy (deliberate):** each fill goes through the public * `update()` / `updateRelation()` paths, so every write is individually * atomic (storage record + indexes + subtype rollups commit together) and * bumps the entry's `_rev` like any other update. The pass is *not* one * whole-brain transaction: a migration over millions of entries inside a * single transaction would hold an unbounded working set and turn one bad * entry into an all-or-nothing failure. Idempotence is the recovery model — * a crashed or partially-failed run is resumed safely by re-running, because * only entries still missing a subtype are written. * * Streams via the same paginated `storage.getNouns()` / `storage.getVerbs()` * walk `audit()` uses — `O(N)` but constant memory. The noun pass is skipped * entirely when the map has no NounType rules, and vice versa. * * @param rules - Map of NounType/VerbType → literal subtype or rule function. * Must contain at least one valid type key; invalid keys, empty-string * literals, and non-string/non-function values throw before any data is * touched. * @param options.includeVFS - Also fill VFS infrastructure entries (default * `false` — they bypass enforcement and don't need a subtype). * @param options.batchSize - Pagination batch size (default `200`). * @param options.onProgress - Optional callback invoked after each batch. * @returns `{ scanned, filled, skipped, errors, byType }` — see * {@link FillSubtypesResult}. After a clean run, `skipped` equals the * remaining `audit().total`. * @throws If the brain is read-only, the rule map is empty/malformed, or a * key is not a valid NounType/VerbType. Per-entry write failures do NOT * throw — they are collected in `errors` and the pass continues. * * @example Back-fill entities and relationships in one pass * const report = await brain.fillSubtypes({ * [NounType.Person]: (e) => e.metadata?.kind ?? 'unspecified', * [NounType.Document]: 'general', * [VerbType.RelatedTo]: 'unspecified' * }) * // → { scanned: 5200, filled: 1429, skipped: 0, errors: [], byType: { person: 800, document: 600, relatedTo: 29 } } * * @example Selective fill — decline entries a rule can't classify * await brain.fillSubtypes({ * [NounType.Person]: (e) => e.metadata?.department ? 'employee' : undefined * }) * // Persons without a department stay untouched (counted as skipped). * * @since 8.0.0 */ async fillSubtypes( rules: FillSubtypeRules, options: { includeVFS?: boolean batchSize?: number onProgress?: (progress: { scanned: number; filled: number; skipped: number }) => void } = {} ): Promise { this.assertWritable('fillSubtypes') await this.ensureInitialized() // Validate the rule map up front — fail fast on shape errors before any // data is touched. if (!rules || typeof rules !== 'object' || Array.isArray(rules)) { throw new Error( 'fillSubtypes: rules must be a map of NounType/VerbType → subtype string or rule function' ) } const nounTypeValues = new Set(Object.values(NounType)) const verbTypeValues = new Set(Object.values(VerbType)) const nounRules = new Map>>() const verbRules = new Map>>() for (const [key, rule] of Object.entries(rules)) { if (rule === undefined) continue if (typeof rule !== 'string' && typeof rule !== 'function') { throw new Error( `fillSubtypes: rule for '${key}' must be a subtype string or a function (got ${typeof rule})` ) } if (typeof rule === 'string' && rule.length === 0) { throw new Error( `fillSubtypes: rule for '${key}' is an empty string — a subtype must be non-empty` ) } if (nounTypeValues.has(key)) { nounRules.set(key, rule as FillSubtypeRule>) } else if (verbTypeValues.has(key)) { verbRules.set(key, rule as FillSubtypeRule>) } else { throw new Error(`fillSubtypes: '${key}' is not a valid NounType or VerbType`) } } if (nounRules.size === 0 && verbRules.size === 0) { throw new Error( 'fillSubtypes: rules map is empty — provide at least one NounType or VerbType rule' ) } const includeVFS = options.includeVFS === true const batchSize = Math.max(1, options.batchSize ?? 200) let scanned = 0 let filled = 0 let skipped = 0 const errors: Array<{ id: string; error: string }> = [] const byType: Record = {} const reportProgress = (): void => { if (options.onProgress) options.onProgress({ scanned, filled, skipped }) } // Normalize a rule's output: only a non-empty string is a fill; anything // else (undefined, empty string) is a decline. An empty subtype would fail // the very strict-mode check this helper exists to satisfy. const normalize = (value: string | undefined): string | undefined => typeof value === 'string' && value.length > 0 ? value : undefined // Pass 1: entities (skipped entirely when the map has no NounType rules). if (nounRules.size > 0) { let offset = 0 while (true) { const page = await this.storage.getNouns({ pagination: { offset, limit: batchSize } }) if (page.items.length === 0) break for (const noun of page.items) { scanned++ // VFS infrastructure entries bypass enforcement via their marker, so // they're not migration debt — leave them alone unless asked. if ( !includeVFS && (noun.metadata?.isVFSEntity === true || noun.metadata?.isVFS === true) ) { continue } if (typeof noun.subtype === 'string' && noun.subtype.length > 0) continue // already filled — never overwrite const rule = nounRules.get(noun.type) if (rule === undefined) { skipped++ continue } try { let subtype: string | undefined if (typeof rule === 'function') { const entity = await this.convertNounToEntity(noun) subtype = normalize(rule(entity)) } else { subtype = rule } if (subtype === undefined) { skipped++ continue } await this.update({ id: noun.id, subtype }) filled++ byType[noun.type] = (byType[noun.type] || 0) + 1 } catch (err) { errors.push({ id: noun.id, error: err instanceof Error ? err.message : String(err) }) } } reportProgress() if (!page.hasMore) break offset += page.items.length } } // Pass 2: relationships (skipped entirely when the map has no VerbType rules). if (verbRules.size > 0) { let offset = 0 while (true) { const page = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } }) if (page.items.length === 0) break for (const verb of page.items) { scanned++ if ( !includeVFS && (verb.metadata?.isVFSEntity === true || verb.metadata?.isVFS === true) ) { continue } if (typeof verb.subtype === 'string' && verb.subtype.length > 0) continue // already filled — never overwrite const rule = verbRules.get(verb.verb) if (rule === undefined) { skipped++ continue } try { let subtype: string | undefined if (typeof rule === 'function') { // Project the stored verb onto the public Relation shape the // rule function is typed against. const relation: Relation = { id: verb.id, from: verb.sourceId, to: verb.targetId, type: verb.verb, weight: verb.weight, data: verb.data, metadata: verb.metadata as T, service: verb.service, createdAt: verb.createdAt, updatedAt: verb.updatedAt, confidence: verb.confidence } subtype = normalize(rule(relation)) } else { subtype = rule } if (subtype === undefined) { skipped++ continue } await this.updateRelation({ id: verb.id, subtype }) filled++ byType[verb.verb] = (byType[verb.verb] || 0) + 1 } catch (err) { errors.push({ id: verb.id, error: err instanceof Error ? err.message : String(err) }) } } reportProgress() if (!page.hasMore) break offset += page.items.length } } return { scanned, filled, skipped, errors, byType } } /** * Stream-and-rewrite a field across every entity in the brain. * * Layer 3 of the subtype-and-facets primitive. Reads the value at `from`, * writes it to `to`, and (unless `readBoth: true`) clears the source. Use this * when migrating between field-name conventions or moving a value from * `metadata.*` / `data.*` up to a top-level standard field like `subtype`. * * **Path forms:** * - `'subtype'`, `'type'`, `'confidence'`, etc. — top-level standard fields * (whatever appears in `STANDARD_ENTITY_FIELDS`) * - `'metadata.X'` — a key under `entity.metadata` * - `'data.X'` — a key under `entity.data` (when `data` is an object) * - `'X'` (bare, not standard) — shorthand for `metadata.X` * * **Behavior:** * - Streams entities in batches and rewrites in-place via `brain.update()`. * Aggregations and indexes refresh automatically. * - Entities where the source path is absent or where the target already * holds the same value are skipped (idempotent — safe to re-run). * - With `readBoth: true`, the source value is preserved alongside the new * target so legacy consumers can keep querying the old path during a * deprecation window. Re-run with `readBoth: false` when ready to clear. * * @param options.from - Source path * @param options.to - Destination path * @param options.readBoth - If true, leave the source value in place (default false → source cleared) * @param options.batchSize - Entities per batch (default 100) * @param options.onProgress - Optional callback invoked after each batch * @returns Migration summary — counts and per-entity errors * * @example One-shot migration * await brain.migrateField({ from: 'metadata.kind', to: 'subtype' }) * // → { scanned: 1500, migrated: 1500, skipped: 0, errors: [] } * * @example Deprecation window — keep both fields readable * await brain.migrateField({ from: 'data.kind', to: 'subtype', readBoth: true }) * // ...consumers switch reads to subtype over time... * await brain.migrateField({ from: 'data.kind', to: 'subtype' }) * // ...source cleared in the final sweep. */ async migrateField(options: { from: string to: string readBoth?: boolean batchSize?: number onProgress?: (progress: { scanned: number; migrated: number }) => void /** * Which entity kind to walk. Defaults to `'noun'` for backward compat with * 7.29.0. Use `'verb'` to migrate relationship fields, or `'both'` to walk * nouns then verbs in one pass. Path forms (`'subtype'`, `'metadata.X'`, * `'data.X'`) are identical on both sides. */ entityKind?: 'noun' | 'verb' | 'both' }): Promise<{ scanned: number migrated: number skipped: number errors: Array<{ id: string; error: string }> }> { this.assertWritable('migrateField') await this.ensureInitialized() if (!options.from || typeof options.from !== 'string') { throw new Error('migrateField: `from` must be a non-empty string path') } if (!options.to || typeof options.to !== 'string') { throw new Error('migrateField: `to` must be a non-empty string path') } if (options.from === options.to) { throw new Error('migrateField: `from` and `to` are identical — nothing to do') } const fromPath = this.parseMigrationPath(options.from) const toPath = this.parseMigrationPath(options.to) const batchSize = Math.max(1, options.batchSize ?? 100) const readBoth = options.readBoth === true const entityKind = options.entityKind ?? 'noun' let scanned = 0 let migrated = 0 let skipped = 0 const errors: Array<{ id: string; error: string }> = [] const reportProgress = (): void => { if (options.onProgress) { options.onProgress({ scanned, migrated }) } } // Walk nouns when entityKind is 'noun' or 'both'. if (entityKind === 'noun' || entityKind === 'both') { let offset = 0 while (true) { const page = await this.storage.getNouns({ pagination: { offset, limit: batchSize } }) if (page.items.length === 0) break for (const noun of page.items) { scanned++ try { const entity = await this.convertNounToEntity(noun) const sourceValue = this.readPath(entity, fromPath) if (sourceValue === undefined || sourceValue === null) { skipped++ continue } const targetValue = this.readPath(entity, toPath) if (targetValue === sourceValue && (readBoth || !this.pathExists(entity, fromPath))) { skipped++ continue } const update = this.buildMigrationUpdate(entity, fromPath, toPath, sourceValue, readBoth) await this.update(update) migrated++ } catch (err) { errors.push({ id: noun.id ?? '', error: err instanceof Error ? err.message : String(err) }) } } reportProgress() if (!page.hasMore) break offset += page.items.length } } // Walk verbs when entityKind is 'verb' or 'both'. Mirror of the noun loop — // the path forms (`'subtype'`, `'metadata.X'`, `'data.X'`) and the // readPath / buildMigrationUpdate helpers all work for verbs because // `Relation` carries the same shape (top-level standard fields + // metadata bag + optional data object). updateRelation() is the verb-side // mutator. if (entityKind === 'verb' || entityKind === 'both') { let offset = 0 while (true) { const page = await this.storage.getVerbs({ pagination: { offset, limit: batchSize } }) if (page.items.length === 0) break for (const verb of page.items) { scanned++ try { const edge = this.verbToRelationLike(verb) const sourceValue = this.readPath(edge, fromPath) if (sourceValue === undefined || sourceValue === null) { skipped++ continue } const targetValue = this.readPath(edge, toPath) if (targetValue === sourceValue && (readBoth || !this.pathExists(edge, fromPath))) { skipped++ continue } const update = this.buildRelationMigrationUpdate(edge, fromPath, toPath, sourceValue, readBoth) await this.updateRelation(update) migrated++ } catch (err) { errors.push({ id: verb.id ?? '', error: err instanceof Error ? err.message : String(err) }) } } reportProgress() if (!page.hasMore) break offset += page.items.length } } return { scanned, migrated, skipped, errors } } /** * Project a storage verb shape onto the Entity-shaped surface that * `readPath` / `pathExists` already understand. The migration helpers were * written for nouns; making them work for verbs is just a shape projection — * `subtype` / `metadata` / `data` live at the same paths on both sides. */ private verbToRelationLike(verb: any): Entity { return { id: verb.id, vector: verb.vector, // Deliberate projection: the edge's VerbType occupies Entity.type so the // noun-oriented path helpers (readPath/pathExists) work unchanged. type: (verb.verb ?? verb.type) as NounType, subtype: verb.subtype, data: verb.data, metadata: verb.metadata as T, service: verb.service, createdAt: verb.createdAt, updatedAt: verb.updatedAt, createdBy: verb.createdBy, confidence: verb.confidence, weight: verb.weight } } /** * Build an `UpdateRelationParams` payload that mirrors `buildMigrationUpdate` * for verbs. Top-level path → top-level on update; metadata path → metadata * bag with merge:false; data path → data bag. */ private buildRelationMigrationUpdate( edge: Entity, from: { kind: 'top' | 'metadata' | 'data'; field: string }, to: { kind: 'top' | 'metadata' | 'data'; field: string }, value: unknown, readBoth: boolean ): UpdateRelationParams { const id = edge.id // Record intersection: `to.kind === 'top'` writes a dynamically-named // standard field (subtype/weight/confidence/...) onto the update payload. const update: UpdateRelationParams & Record = { id } if (to.kind === 'top') { update[to.field] = value } else if (to.kind === 'metadata') { const base = (edge.metadata as unknown as Record) ?? {} const nextMeta: Record = { ...base, [to.field]: value } if (!readBoth && from.kind === 'metadata') { delete nextMeta[from.field] } update.metadata = nextMeta as UpdateRelationParams['metadata'] update.merge = false } else { const base = (edge.data && typeof edge.data === 'object') ? { ...(edge.data as Record) } : {} base[to.field] = value if (!readBoth && from.kind === 'data') { delete base[from.field] } update.data = base } if (!readBoth) { if (from.kind === 'metadata' && to.kind !== 'metadata') { const base = (edge.metadata as unknown as Record) ?? {} const nextMeta: Record = { ...base } delete nextMeta[from.field] update.metadata = nextMeta as UpdateRelationParams['metadata'] update.merge = false } else if (from.kind === 'data' && to.kind !== 'data') { const base = (edge.data && typeof edge.data === 'object') ? { ...(edge.data as Record) } : null if (base) { delete base[from.field] update.data = base } } else if (from.kind === 'top' && from.field === 'subtype') { update.subtype = undefined } } return update } /** * Parse a dotted path used by `migrateField` into its routing kind + field name. * `'subtype'` → top-level standard; `'metadata.X'` → metadata; `'data.X'` → data; * bare non-standard names → metadata (matching `resolveEntityField`'s fallback). */ private parseMigrationPath(path: string): { kind: 'top' | 'metadata' | 'data'; field: string } { const dotIdx = path.indexOf('.') if (dotIdx === -1) { if (STANDARD_ENTITY_FIELDS.has(path)) return { kind: 'top', field: path } return { kind: 'metadata', field: path } } const head = path.slice(0, dotIdx) const tail = path.slice(dotIdx + 1) if (!tail) throw new Error(`migrateField: invalid path '${path}' (trailing dot)`) if (head === 'metadata') return { kind: 'metadata', field: tail } if (head === 'data') return { kind: 'data', field: tail } throw new Error( `migrateField: unsupported path prefix '${head}'. Supported: 'metadata.X', 'data.X', or a bare field name.` ) } /** Read the value at a parsed migration path. Returns `undefined` if absent. */ private readPath(entity: Entity, path: { kind: 'top' | 'metadata' | 'data'; field: string }): unknown { if (path.kind === 'top') return (entity as Entity & Record)[path.field] if (path.kind === 'metadata') { const bag = entity.metadata as unknown as Record | undefined return bag?.[path.field] } const data = entity.data if (data && typeof data === 'object') { return (data as Record)[path.field] } return undefined } /** Whether a parsed path resolves to a defined (non-undefined) value. */ private pathExists(entity: Entity, path: { kind: 'top' | 'metadata' | 'data'; field: string }): boolean { return this.readPath(entity, path) !== undefined } /** * Build an `UpdateParams` payload that copies the value from the source path * to the destination, and (unless `readBoth`) clears the source. Uses * `merge: false` on the metadata bag when clearing so we can omit the source * key authoritatively instead of relying on `undefined` round-trip behavior. */ private buildMigrationUpdate( entity: Entity, from: { kind: 'top' | 'metadata' | 'data'; field: string }, to: { kind: 'top' | 'metadata' | 'data'; field: string }, value: unknown, readBoth: boolean ): UpdateParams { const id = entity.id // Record intersection: `to.kind === 'top'` writes a dynamically-named // standard field (subtype/weight/confidence/...) onto the update payload. const update: UpdateParams & Record = { id } // Apply destination write. if (to.kind === 'top') { update[to.field] = value } else if (to.kind === 'metadata') { const base = (entity.metadata as unknown as Record) ?? {} const nextMeta: Record = { ...base, [to.field]: value } if (!readBoth && from.kind === 'metadata') { delete nextMeta[from.field] } update.metadata = nextMeta as UpdateParams['metadata'] update.merge = false } else { // data path — only meaningful when data is an object const base = (entity.data && typeof entity.data === 'object') ? { ...(entity.data as Record) } : {} base[to.field] = value if (!readBoth && from.kind === 'data') { delete base[from.field] } update.data = base } // Apply source clear if not already handled by the destination bag write. if (!readBoth) { if (from.kind === 'metadata' && to.kind !== 'metadata') { const base = (entity.metadata as unknown as Record) ?? {} const nextMeta: Record = { ...base } delete nextMeta[from.field] update.metadata = nextMeta as UpdateParams['metadata'] update.merge = false } else if (from.kind === 'data' && to.kind !== 'data') { const base = (entity.data && typeof entity.data === 'object') ? { ...(entity.data as Record) } : null if (base) { delete base[from.field] update.data = base } } else if (from.kind === 'top' && (from.field === 'subtype')) { // Only subtype currently supports clearing at the top level via UpdateParams. // Other standard fields aren't user-mutable through this path. update.subtype = undefined } } return update } // ============= NEW EMBEDDING & ANALYSIS APIs ============= /** * Batch embed multiple texts at once * * More efficient than calling embed() multiple times due to * WASM batch processing optimizations. * * @param texts Array of texts to embed * @returns Array of embedding vectors (384 dimensions each) * * @example * const embeddings = await brain.embedBatch([ * 'Machine learning is fascinating', * 'Deep neural networks', * 'Natural language processing' * ]) * // embeddings.length === 3 * // embeddings[0].length === 384 */ async embedBatch(texts: string[], options?: { signal?: AbortSignal }): Promise { await this.ensureInitialized() if (texts.length === 0) { return [] } // Plugin provides native batch embedding — single forward pass for all texts const batchProvider = this.pluginRegistry.getProvider<(texts: string[]) => Promise>('embedBatch') if (batchProvider) { return batchProvider(texts) } // Plugin provides single-text embedding engine — map through it if (this.pluginRegistry.hasProvider('embeddings')) { return Promise.all(texts.map(t => this.embedder(t))) } // Default: WASM batch API (single forward pass, more efficient than N calls) return await embeddingManager.embedBatch(texts, options) } /** * Calculate semantic similarity between two texts * * Returns a score from 0 (completely different) to 1 (identical meaning). * Uses cosine similarity on embedding vectors. * * @param textA First text * @param textB Second text * @returns Similarity score between 0 and 1 * * @example * const score = await brain.similarity( * 'The cat sat on the mat', * 'A feline was resting on the rug' * ) * // score ≈ 0.85 (high semantic similarity) */ async similarity(textA: string, textB: string): Promise { await this.ensureInitialized() // Embed both texts const [vectorA, vectorB] = await Promise.all([ this.embedder(textA), this.embedder(textB) ]) // Calculate cosine similarity (convert from distance) // cosineDistance returns 1 - similarity, so similarity = 1 - distance const distance = this.distance(vectorA, vectorB) return 1 - distance } /** * Zero-config hybrid highlighting * * Returns both exact text matches AND semantically similar concepts. * Perfect for UI highlighting at different levels: * - matchType: 'text' = exact word match (highlight strongly) * - matchType: 'semantic' = concept match (highlight softly) * * @param params.query - The search query * @param params.text - The text to highlight (e.g., entity.data) * @param params.granularity - 'word' | 'phrase' | 'sentence' (default: 'word') * @param params.threshold - Minimum similarity for semantic matches (default: 0.5) * @returns Array of highlights with text, score, position, and matchType * * @example * ```typescript * const highlights = await brain.highlight({ * query: "david the warrior", * text: "David Smith is a brave fighter who battles dragons" * }) * // Returns: [ * // { text: "David", score: 1.0, position: [0, 5], matchType: 'text' }, // Exact * // { text: "fighter", score: 0.78, position: [25, 32], matchType: 'semantic' }, // Concept * // { text: "battles", score: 0.72, position: [37, 44], matchType: 'semantic' } // Concept * // ] * ``` */ async highlight(params: import('./types/brainy.types.js').HighlightParams): Promise { await this.ensureInitialized() const { query, text, granularity = 'word', threshold = 0.5, contentType, contentExtractor } = params if (!query || !text) { return [] } // Extract text from structured content (JSON, HTML, Markdown) // Custom extractor takes priority, then built-in detection type ChunkWithCategory = { text: string, position: [number, number], contentCategory?: import('./types/brainy.types.js').ContentCategory } let segments: import('./types/brainy.types.js').ExtractedSegment[] if (contentExtractor) { segments = contentExtractor(text) } else { segments = extractForHighlighting(text, contentType) } // Build concatenated text from segments for position tracking // and split each segment into chunks based on granularity const allChunks: ChunkWithCategory[] = [] let offset = 0 for (const segment of segments) { const segmentChunks = this.splitForHighlighting(segment.text, granularity) for (const chunk of segmentChunks) { allChunks.push({ text: chunk.text, position: [chunk.position[0] + offset, chunk.position[1] + offset], contentCategory: segment.contentCategory }) } offset += segment.text.length + 1 // +1 for space between segments } if (allChunks.length === 0) { return [] } // Production safety: Limit chunks to prevent memory explosion // At 500 words × 384 dimensions × 4 bytes = 768KB temp memory (acceptable) const MAX_HIGHLIGHT_CHUNKS = 500 const chunks = allChunks.slice(0, MAX_HIGHLIGHT_CHUNKS) // Track all highlights (keyed by position to avoid duplicates) const highlightMap = new Map() // === PHASE 1: Find exact text matches (score = 1.0, matchType = 'text') === const queryWords = this.metadataIndex.tokenize(query) const queryWordsLower = new Set(queryWords.map(w => w.toLowerCase())) for (const chunk of chunks) { const chunkLower = chunk.text.toLowerCase().replace(/[^\w\s]/g, '') if (queryWordsLower.has(chunkLower)) { const key = `${chunk.position[0]}-${chunk.position[1]}` highlightMap.set(key, { text: chunk.text, score: 1.0, position: chunk.position, matchType: 'text', contentCategory: chunk.contentCategory }) } } // === PHASE 2: Find semantic matches with timeout fallback === // AbortController ensures the background semantic work (WASM batch embedding) // is cancelled on timeout or error, preventing event loop saturation and // WASM engine crashes from abandoned promises. const SEMANTIC_TIMEOUT_MS = 10_000 const abortController = new AbortController() try { const semanticResult = await Promise.race([ this.highlightSemanticPhase(query, chunks, threshold, highlightMap, abortController.signal), new Promise<'timeout'>((resolve) => setTimeout(() => resolve('timeout'), SEMANTIC_TIMEOUT_MS)) ]) if (semanticResult === 'timeout') { abortController.abort() const textHighlights = Array.from(highlightMap.values()) return textHighlights.sort((a, b) => b.score - a.score) } } catch { abortController.abort() const textHighlights = Array.from(highlightMap.values()) return textHighlights.sort((a, b) => b.score - a.score) } // Sort by score descending (text matches will be first with score=1.0) const highlights = Array.from(highlightMap.values()) return highlights.sort((a, b) => b.score - a.score) } /** * Phase 2 of highlight(): semantic matching with batch embedding * @internal */ private async highlightSemanticPhase( query: string, chunks: Array<{ text: string, position: [number, number], contentCategory?: import('./types/brainy.types.js').ContentCategory }>, threshold: number, highlightMap: Map, signal?: AbortSignal ): Promise { if (signal?.aborted) return // Get query embedding const queryVector = await this.embed(query) if (signal?.aborted) return // Batch embed all chunks using native WASM batch API const chunkTexts = chunks.map(c => c.text) const chunkVectors = await this.embedBatch(chunkTexts, { signal }) if (signal?.aborted) return // Calculate semantic similarities for (let i = 0; i < chunks.length; i++) { const key = `${chunks[i].position[0]}-${chunks[i].position[1]}` // Skip if already a text match (text matches take priority) if (highlightMap.has(key)) continue const distance = this.distance(queryVector, chunkVectors[i]) const similarity = 1 - distance if (similarity >= threshold) { highlightMap.set(key, { text: chunks[i].text, score: similarity, position: chunks[i].position, matchType: 'semantic', contentCategory: chunks[i].contentCategory }) } } } /** * Split text into chunks for highlighting * @internal */ private splitForHighlighting(text: string, granularity: string): Array<{ text: string, position: [number, number] }> { const results: Array<{ text: string, position: [number, number] }> = [] if (granularity === 'word') { // Split on whitespace, track positions const regex = /\S+/g let match while ((match = regex.exec(text)) !== null) { // Skip stopwords if (!STOPWORDS.has(match[0].toLowerCase())) { results.push({ text: match[0], position: [match.index, match.index + match[0].length] }) } } } else if (granularity === 'sentence') { // Split on sentence boundaries const regex = /[^.!?]+[.!?]+/g let match while ((match = regex.exec(text)) !== null) { results.push({ text: match[0].trim(), position: [match.index, match.index + match[0].length] }) } // Handle text without sentence-ending punctuation if (results.length === 0 && text.trim()) { results.push({ text: text.trim(), position: [0, text.length] }) } } else if (granularity === 'phrase') { // Sliding window of 2-4 words const words: Array<{ text: string, start: number, end: number }> = [] const regex = /\S+/g let match while ((match = regex.exec(text)) !== null) { words.push({ text: match[0], start: match.index, end: match.index + match[0].length }) } // Generate 2-4 word phrases for (let windowSize = 2; windowSize <= 4; windowSize++) { for (let i = 0; i <= words.length - windowSize; i++) { const phraseWords = words.slice(i, i + windowSize) const phraseText = phraseWords.map(w => w.text).join(' ') const start = phraseWords[0].start const end = phraseWords[phraseWords.length - 1].end results.push({ text: phraseText, position: [start, end] }) } } } return results } /** * Get comprehensive index statistics * * Returns detailed stats about all internal indexes including * entity counts, vector index size, graph relationships, and * estimated memory usage. * * @returns Index statistics object * * @example * const stats = await brain.indexStats() * console.log(`Entities: ${stats.entities}`) * console.log(`Vectors: ${stats.vectors}`) * console.log(`Relationships: ${stats.relationships}`) */ async indexStats(): Promise<{ entities: number vectors: number relationships: number metadataFields: string[] memoryUsage: { vectors: number graph: number metadata: number total: number } }> { await this.ensureInitialized() const metadataStats = await this.metadataIndex.getStats() const graphStats = this.graphIndex.getStats() const vectorCount = this.index.size() // Get unique metadata field names const metadataFields = metadataStats.fieldsIndexed || [] return { entities: metadataStats.totalEntries, vectors: vectorCount, relationships: graphStats.totalRelationships, metadataFields, memoryUsage: { vectors: vectorCount * 384 * 4, // 384 dimensions * 4 bytes per float32 graph: graphStats.memoryUsage, metadata: metadataStats.indexSize || 0, total: (vectorCount * 384 * 4) + graphStats.memoryUsage + (metadataStats.indexSize || 0) } } } /** * Validate metadata index consistency and detect corruption * * Returns health status and recommendations for repair. Corruption typically * manifests as high avg entries/entity (expected ~30, corrupted can be 100+) * caused by the update() field asymmetry bug (fixed). * * @returns Promise resolving to validation results * * @example * const validation = await brain.validateIndexConsistency() * if (!validation.healthy) { * console.log(validation.recommendation) * // Run brain.rebuildIndex() to repair * } */ async validateIndexConsistency(): Promise<{ healthy: boolean avgEntriesPerEntity: number entityCount: number indexEntryCount: number recommendation: string | null /** Cross-layer: each provider's own invariant self-report, when * it exposes validateInvariants(). Absent providers are simply not listed. */ providers?: ProviderInvariantReport[] }> { await this.ensureInitialized() const result = await this.metadataIndex.validateConsistency() // Cross-layer integrity: validateConsistency() above only sees the // JS metadata index — it is BLIND to a native provider whose manifest ↔ // segments ↔ counts have diverged. Delegate to each provider's own // validateInvariants() and aggregate, so "healthy-while-broken" is impossible. const providers = await this.collectProviderInvariants() const brokenProviders = providers.filter((p) => !p.healthy) let healthy = result.healthy const notes: string[] = [] if (result.recommendation) notes.push(result.recommendation) if (this._indexRebuildFailed) { healthy = false notes.unshift( `Index rebuild failed at init() (degraded — queries may be incomplete): ` + `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` ) } if (this._indexDegradedIds.size > 0) { healthy = false notes.unshift( `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + `failed-rollback recovery — the derived index may be incomplete for them. ` + `Run repairIndex() to reconcile.` ) } for (const p of brokenProviders) { healthy = false const failing = p.invariants.filter((i) => !i.holds) notes.unshift( `Provider '${p.provider}' reports ${failing.length} failing invariant(s): ` + failing.map((i) => `${i.name} (${i.detail}; heal=${i.heal})`).join('; ') + `. Run repairIndex() to reconcile from canonical.` ) } return { ...result, healthy, recommendation: notes.length > 0 ? notes.join(' ') : null, ...(providers.length > 0 ? { providers } : {}) } } /** * @description Feature-detect + call each index provider's OPTIONAL * `validateInvariants()` and collect the reports. The contract is * that `validateInvariants()` NEVER throws and is bounded <50ms — but if a * provider violates that, the throw is turned into an UNHEALTHY synthetic * report (loud), never swallowed into "healthy". Providers that do not expose * the hook are simply omitted (the JS baseline validates via * `metadataIndex.validateConsistency()`). * @returns One report per provider that exposes `validateInvariants()`. */ private async collectProviderInvariants(): Promise { const reports: ProviderInvariantReport[] = [] const providers: unknown[] = [this.metadataIndex, this.index, this.graphIndex] for (const provider of providers) { const fn = (provider as { validateInvariants?: () => Promise } | null) ?.validateInvariants if (typeof fn !== 'function') continue try { const report = await fn.call(provider) if (report && Array.isArray(report.invariants)) reports.push(report) } catch (err) { reports.push({ provider: 'unknown', healthy: false, serving: false, invariants: [ { name: 'validate-invariants-threw', holds: false, detail: `validateInvariants() threw (contract violation — it must never throw): ${(err as Error).message}`, heal: 'rebuild' } ], checkedAt: Date.now(), durationMs: 0 }) } } return reports } /** * Get metadata index statistics * * Returns detailed statistics about the metadata index including * total entries, IDs indexed, and fields indexed. * * @returns Promise resolving to index statistics */ async getIndexStats(): Promise<{ totalEntries: number totalIds: number fieldsIndexed: string[] lastRebuild: number indexSize: number }> { await this.ensureInitialized() return this.metadataIndex.getStats() } /** * Get graph neighbors of an entity * * Traverses the relationship graph to find connected entities. * Supports filtering by direction and relationship type. * * @param entityId The entity to get neighbors for * @param options Optional traversal options * @returns Array of neighbor entity IDs * * @example * // Get all connected entities * const allNeighbors = await brain.neighbors(entityId) * * // Get only outgoing connections * const outgoing = await brain.neighbors(entityId, { direction: 'outgoing' }) * * // Get incoming connections with specific verb type * const incoming = await brain.neighbors(entityId, { * direction: 'incoming', * verbType: VerbType.RELATES_TO * }) */ async neighbors( entityId: string, options?: { direction?: 'outgoing' | 'incoming' | 'both' depth?: number verbType?: VerbType | VerbType[] limit?: number } ): Promise { // Graph read (see related): adjacency traversal only. await this.ensureInitialized({ needs: ['graph'] }) await this.verifyGraphAdjacencyLive() const direction = options?.direction || 'both' const limit = options?.limit const verbTypes = options?.verbType === undefined ? undefined : new Set(Array.isArray(options.verbType) ? options.verbType : [options.verbType]) // Map our API direction to graphIndex direction const graphDirection = direction === 'outgoing' ? 'out' : direction === 'incoming' ? 'in' : 'both' let neighbors = await this.getTypedNeighbors(entityId, graphDirection, verbTypes, limit) // Handle depth > 1 (multi-hop traversal). The verb-type filter is applied at EVERY hop // (previously only the first), and the BFS is bounded by `limit` so a dense graph can't // expand without limit before the final slice. if (options?.depth && options.depth > 1) { const visited = new Set([entityId, ...neighbors]) let currentLevel = neighbors for (let d = 1; d < options.depth; d++) { if (limit && neighbors.length >= limit) break const nextLevel: string[] = [] outer: for (const nodeId of currentLevel) { const nodeNeighbors = await this.getTypedNeighbors(nodeId, graphDirection, verbTypes) for (const neighbor of nodeNeighbors) { if (!visited.has(neighbor)) { visited.add(neighbor) nextLevel.push(neighbor) neighbors.push(neighbor) if (limit && neighbors.length >= limit) break outer } } } if (nextLevel.length === 0) break currentLevel = nextLevel } if (limit && neighbors.length > limit) { neighbors = neighbors.slice(0, limit) } } return neighbors } /** * Neighbours of a single node, optionally filtered to a set of verb types. * * Extracted so depth traversal can apply the verb-type filter at every hop (not just the * first). For `direction: 'both'` the verb scan unions out-edges and in-edges so the filter * isn't silently limited to outgoing edges. */ private async getTypedNeighbors( nodeId: string, graphDirection: 'in' | 'out' | 'both', verbTypes?: Set, limit?: number ): Promise { // 8.0 BigInt boundary: unmapped node → no relations. const nodeInt = this.graphEntityInt(nodeId) if (nodeInt === undefined) return [] const neighbors = this.entityIntsToUuids( await this.graphIndex.getNeighbors(nodeInt, { direction: graphDirection, limit }) ) if (!verbTypes || verbTypes.size === 0) return neighbors // Gather candidate edges in the relevant direction(s). const verbInts: bigint[] = [] if (graphDirection !== 'in') verbInts.push(...await this.graphIndex.getVerbIdsBySource(nodeInt)) if (graphDirection !== 'out') verbInts.push(...await this.graphIndex.getVerbIdsByTarget(nodeInt)) const verbIds = await this.resolveVerbIntsToIds(verbInts) const verbs = await this.graphIndex.getVerbsBatchCached(verbIds) const neighborSet = new Set(neighbors) const filtered: string[] = [] for (const [, verb] of verbs) { if (verbTypes.has(verb.type as VerbType) || verbTypes.has(verb.verb as VerbType)) { const neighborId = verb.sourceId === nodeId ? verb.targetId : verb.sourceId if (neighborSet.has(neighborId)) filtered.push(neighborId) } } return filtered } /** * Find semantic duplicates in the database * * Uses embedding similarity to identify entities that may be * duplicates or near-duplicates based on their content. * * @param options Optional search options * @returns Array of duplicate groups with similarity scores * * @example * // Find all duplicates with default threshold (0.85) * const duplicates = await brain.findDuplicates() * * // Find duplicates of a specific type with custom threshold * const personDupes = await brain.findDuplicates({ * type: NounType.PERSON, * threshold: 0.9, * limit: 100 * }) */ async findDuplicates(options?: { threshold?: number type?: NounType limit?: number }): Promise duplicates: Array<{ entity: Entity; similarity: number }> }>> { await this.ensureInitialized() const threshold = options?.threshold ?? 0.85 const limit = options?.limit ?? 100 // Get entities to check const findParams: FindParams = { limit: Math.min(limit * 10, 1000), // Get more entities to find duplicates within type: options?.type } const entities = await this.find(findParams) const results: Array<{ entity: Entity duplicates: Array<{ entity: Entity; similarity: number }> }> = [] const processedIds = new Set() for (const result of entities) { if (processedIds.has(result.id)) continue // Find similar entities const similar = await this.similar({ to: result.id, limit: 20, // Check top 20 similar entities type: options?.type }) // Filter to those above threshold (excluding self) const duplicates = similar .filter(s => s.id !== result.id && s.score >= threshold) .map(s => ({ entity: s.entity, similarity: s.score })) if (duplicates.length > 0) { results.push({ entity: result.entity, duplicates }) // Mark all duplicates as processed to avoid reverse matches duplicates.forEach(d => processedIds.add(d.entity.id)) } processedIds.add(result.id) // Stop if we have enough results if (results.length >= limit) break } return results } /** * Cluster entities by semantic similarity * * Groups entities into clusters based on their embedding similarity. * Uses a greedy algorithm that finds densely connected components * using the HNSW index for efficient neighbor lookup. * * @param options Optional clustering options * @returns Array of clusters with entities and optional centroids * * @example * // Find all clusters with default threshold * const clusters = await brain.cluster() * * // Find document clusters with higher threshold * const docClusters = await brain.cluster({ * type: NounType.Document, * threshold: 0.85, * minClusterSize: 3 * }) * * for (const cluster of docClusters) { * console.log(`Cluster ${cluster.clusterId}: ${cluster.entities.length} entities`) * } */ async cluster(options?: { threshold?: number type?: NounType minClusterSize?: number limit?: number includeCentroid?: boolean }): Promise[] centroid?: number[] }>> { await this.ensureInitialized() const threshold = options?.threshold ?? 0.8 const minClusterSize = options?.minClusterSize ?? 2 const limit = options?.limit ?? 100 const includeCentroid = options?.includeCentroid ?? false // Get entities to cluster const findParams: FindParams = { limit: 1000, // Process up to 1000 entities type: options?.type } const allEntities = await this.find(findParams) const clustered = new Set() const clusters: Array<{ clusterId: string entities: Entity[] centroid?: number[] }> = [] // Greedy clustering: for each unclustered entity, find its similar neighbors for (const result of allEntities) { if (clustered.has(result.id)) continue // Find similar entities to this one const similar = await this.similar({ to: result.id, limit: 50, threshold, type: options?.type }) // Filter to unclustered entities (including self) const clusterMembers = similar.filter(s => !clustered.has(s.id)) // Only create cluster if it meets minimum size if (clusterMembers.length >= minClusterSize) { const entities = clusterMembers.map(s => s.entity) // Mark all as clustered clusterMembers.forEach(s => clustered.add(s.id)) // Calculate centroid if requested let centroid: number[] | undefined if (includeCentroid && entities.length > 0) { const vectors = entities .filter(e => e.vector && e.vector.length > 0) .map(e => e.vector as number[]) if (vectors.length > 0) { // Average all vectors to get centroid const dim = vectors[0].length centroid = new Array(dim).fill(0) for (const vec of vectors) { for (let i = 0; i < dim; i++) { centroid[i] += vec[i] } } for (let i = 0; i < dim; i++) { centroid[i] /= vectors.length } } } clusters.push({ clusterId: `cluster-${clusters.length + 1}`, entities, centroid }) if (clusters.length >= limit) break } else { // Mark single entity as processed (not in a cluster) clustered.add(result.id) } } return clusters } /** * Storage adapter (internal API) * Direct access to the underlying BaseStorage for diagnostics and tests * (e.g. reading the transaction log). Not part of the supported public * surface. * @internal */ get storageAdapter(): BaseStorage { return this.storage } // ============= HELPER METHODS ============= /** * Parse natural language query using advanced NLP with 220+ patterns * The embedding model is always available as it's core to Brainy's functionality */ private async parseNaturalQuery(query: string): Promise> { // Initialize NLP processor if needed (lazy loading) if (!this._nlp) { this._nlp = new NaturalLanguageProcessor(this) await this._nlp.init() // Ensure pattern library is loaded } // Process with our advanced pattern library (220+ patterns with embeddings) const tripleQuery = await this._nlp.processNaturalQuery(query) // Convert TripleQuery to FindParams const params: FindParams = {} // Handle vector search if (tripleQuery.like || tripleQuery.similar) { params.query = typeof tripleQuery.like === 'string' ? tripleQuery.like : typeof tripleQuery.similar === 'string' ? tripleQuery.similar : query } else if (!tripleQuery.where && !tripleQuery.connected) { // Default to vector search if no other criteria specified params.query = query } // Handle metadata filtering if (tripleQuery.where) { params.where = tripleQuery.where as Partial } // Handle graph relationships if (tripleQuery.connected) { params.connected = { to: Array.isArray(tripleQuery.connected.to) ? tripleQuery.connected.to[0] : tripleQuery.connected.to, from: Array.isArray(tripleQuery.connected.from) ? tripleQuery.connected.from[0] : tripleQuery.connected.from, via: tripleQuery.connected.type as VerbType | undefined, depth: tripleQuery.connected.depth, direction: tripleQuery.connected.direction } } // Handle other options if (tripleQuery.limit) params.limit = tripleQuery.limit if (tripleQuery.offset) params.offset = tripleQuery.offset return this.enhanceNLPResult(params, query) } /** * Enhance NLP results with fusion scoring */ private enhanceNLPResult(params: FindParams, _originalQuery: string): FindParams { // Add fusion scoring for complex queries if (params.query && params.where && Object.keys(params.where).length > 0) { params.fusion = params.fusion || { strategy: 'adaptive', weights: { vector: 0.6, field: 0.3, graph: 0.1 } } } return params } /** * @description Build the MetadataIndex filter object from a query's structured * criteria (`where` / `type` / `subtype` / `service` / `excludeVFS`) — the shared * filter shape consumed by `getIdsForFilter` / `getIdSetForFilter`. Applies the * `where.type → noun` alias and expands a multi-type filter into an `anyOf`. * @param params - The structured query criteria. * @returns The filter object, or `null` when no structured criteria are present. */ private buildMetadataFilter(params: { where?: any type?: NounType | NounType[] subtype?: string | string[] service?: string excludeVFS?: boolean }): any | null { if (!(params.where || params.type || params.subtype || params.service || params.excludeVFS)) { return null } let filter: any = {} if (params.where) { Object.assign(filter, params.where) // Alias: where.type → where.noun (storage field name for entity type) if ('type' in filter && !('noun' in filter)) { filter.noun = filter.type delete filter.type } } if (params.service) filter.service = params.service if (params.excludeVFS === true) { filter.vfsType = { exists: false } filter.isVFSEntity = { ne: true } } // Subtype (top-level standard field — fast path). Assigned BEFORE the type-array // expansion below so the spread into each anyOf branch carries it through. if (params.subtype !== undefined) { filter.subtype = Array.isArray(params.subtype) ? { oneOf: params.subtype } : params.subtype } if (params.type) { const types = Array.isArray(params.type) ? params.type : [params.type] if (types.length === 1) { filter.noun = types[0] } else { filter = { anyOf: types.map((type) => ({ noun: type, ...filter })) } } } return filter } /** * Execute vector search component * * @param params Find parameters * @param candidateIds Optional pre-resolved metadata filter IDs for metadata-first search. * When provided, HNSW search is restricted to these candidates: * - NativeHNSWWrapper: uses native searchWithCandidates (Rust bitmap filtering) * - JS JsHnswVectorIndex: converts to filter function with O(1) Set lookups * @param allowedIds Optional 8.0 #46 predicate-pushdown universe as an * {@link OpaqueIdSet} (native/cor roaring Buffer) — forwarded straight into the * beam walk with no id materialization. The native vector provider consumes it; * the JS index ignores the opaque form and relies on `candidateIds`. */ private async executeVectorSearch( params: FindParams, candidateIds?: string[], allowedIds?: OpaqueIdSet ): Promise[]> { // Vector cold-read guard: before trusting a semantic/vector result, verify the // vector index actually SERVES a known persisted vector (one-shot per brain). // A pure semantic find({ query }) has no filter, so verifyMetadataLive never // fires — a cold native index that loaded its COUNT but not its serving // structure would return a silent []. This self-heals (rebuild) or throws // VectorIndexNotReadyError instead. await this.verifyVectorLive() const vector = params.vector || (await this.embed(params.query!)) const limit = params.limit || 10 // Build search options for metadata-first candidate filtering. Pass both forms // when available: the native provider prefers the opaque `allowedIds` (zero // crossing), the JS index uses the materialized `candidateIds`. const searchOptions = candidateIds || allowedIds ? { ...(candidateIds && { candidateIds }), ...(allowedIds && { allowedIds }) } : undefined // HNSW search with optional metadata-first candidate filtering const searchResults: [string, number][] = await this.index.search(vector, limit * 2, undefined, searchOptions) // Batch-load entities for 10-50x faster cloud storage performance // GCS: 10 results = 1×50ms vs 10×50ms = 500ms (10x faster) const ids = searchResults.map(([id]) => id) const entitiesMap = await this.batchGet(ids) const results: Result[] = [] for (const [id, distance] of searchResults) { const entity = entitiesMap.get(id) if (entity) { const score = Math.max(0, Math.min(1, 1 / (1 + distance))) results.push(this.createResult(id, score, entity)) } } return results } /** * Execute proximity search component */ private async executeProximitySearch(params: FindParams): Promise[]> { if (!params.near) return [] // Vector cold-read guard (see executeVectorSearch): proximity search also // hits this.index.search — verify it serves before trusting an empty result. await this.verifyVectorLive() // Teaching error: without an anchor id the constraint is meaningless, and // letting it fall through produces an opaque storage-layer sharding error. if (!params.near.id) { throw new Error( "find({ near }): 'near.id' is required — pass the entity to search around, " + 'e.g. near: { id, threshold }. To impose a minimum score on plain semantic ' + 'results, filter on result.score instead.' ) } const nearEntity = await this.get(params.near.id) if (!nearEntity) return [] const nearResults: [string, number][] = await this.index.search(nearEntity.vector, params.limit || 10) // Filter by threshold first to minimize batch fetch const threshold = params.near.threshold || 0.7 const filteredResults = nearResults.filter(([, distance]) => { const score = Math.max(0, Math.min(1, 1 / (1 + distance))) return score >= threshold }) // Batch-load entities for 10-50x faster cloud storage performance const ids = filteredResults.map(([id]) => id) const entitiesMap = await this.batchGet(ids) const results: Result[] = [] for (const [id, distance] of filteredResults) { const entity = entitiesMap.get(id) if (entity) { const score = Math.max(0, Math.min(1, 1 / (1 + distance))) results.push(this.createResult(id, score, entity)) } } return results } /** * Execute graph search component. * * Honors the full `GraphConstraints` contract: multi-hop `depth` (breadth-first via * `neighbors()`), `via`/`type` verb-type filtering, and `direction`. Previously this read * only `from`/`to`/`direction` and did a single 1-hop `getNeighbors()`, so `depth` and `via` * were silently ignored — `find({ connected: { from, depth: 3 } })` returned only the * immediate neighbour at every depth. */ private async executeGraphSearch(params: FindParams, existingResults: Result[]): Promise[]> { if (!params.connected) return existingResults const { from, to, depth, direction = 'both' } = params.connected const via = params.connected.via ?? params.connected.type const subtypeFilter = params.connected.subtype const effectiveDepth = depth ?? 1 // GraphConstraints speaks 'in' | 'out' | 'both'; neighbors() speaks 'incoming' | 'outgoing' | 'both'. const toNeighborDir = (d: 'in' | 'out' | 'both'): 'incoming' | 'outgoing' | 'both' => d === 'in' ? 'incoming' : d === 'out' ? 'outgoing' : 'both' const connectedIds = new Set() // Population is extracted into a closure so it can be re-run VERBATIM after a // cold-load heal (verdict 'rebuilt') — re-executing the SAME traversal once the // adjacency is actually loaded, without changing any collection semantics. const populate = async (): Promise => { connectedIds.clear() if (subtypeFilter !== undefined) { // Multi-hop BFS with per-hop (verbType, subtype) predicate. Works on the // open-core JS path at any depth. When a graph-index provider exposes a // faster `findConnectedSubtype` (native BFS with the same semantics), // `nativeSubtypeBfs` routes through it transparently — same pattern as // every other provider hook. const subtypeArr = Array.isArray(subtypeFilter) ? subtypeFilter : [subtypeFilter] const subtypeSet = new Set(subtypeArr) // Native fast path (D.3): single verb type + single subtype + outgoing // walk match the native BFS semantics exactly (out-edges only, source // excluded, visited-set cycle guard). Entity ints in, entity ints out — // UUID conversion stays at this boundary. Returns null when the query // shape (or provider) can't take the native route. const nativeSubtypeBfs = async ( anchor: string, walk: 'in' | 'out' | 'both' ): Promise | null> => { if (walk !== 'out') return null if (via === undefined || Array.isArray(via)) return null if (subtypeArr.length !== 1) return null const provider = this.graphIndex as Partial<{ findConnectedSubtype( sourceInt: bigint, verbTypeIndex: number, subtype: string | null, depth: number, limit?: number | null ): Promise }> if (typeof provider.findConnectedSubtype !== 'function') return null const anchorInt = this.graphEntityInt(anchor) if (anchorInt === undefined) return new Set() // unmapped → no relations const verbTypeIndex = TypeUtils.getVerbIndex(via as VerbType) // No limit: match the JS BFS exactly — overall result limiting happens // downstream against existingResults. const reachedInts = await provider.findConnectedSubtype( anchorInt, verbTypeIndex, subtypeArr[0], effectiveDepth, null ) return new Set(this.entityIntsToUuids(reachedInts)) } const bfsWithSubtype = async (anchor: string, walk: 'in' | 'out' | 'both'): Promise> => { const nativeResult = await nativeSubtypeBfs(anchor, walk) if (nativeResult !== null) return nativeResult const visited = new Set([anchor]) const reached = new Set() let frontier = new Set([anchor]) for (let hop = 0; hop < effectiveDepth && frontier.size > 0; hop++) { const nextFrontier = new Set() for (const node of frontier) { // Walk outgoing edges (when direction includes outbound) and incoming // edges (when direction includes inbound). Both = union. const dirs: Array<'from' | 'to'> = [] if (walk === 'out' || walk === 'both') dirs.push('from') if (walk === 'in' || walk === 'both') dirs.push('to') for (const dir of dirs) { const edges = await this.related({ ...(dir === 'from' ? { from: node } : { to: node }), ...(via && { type: via as VerbType | VerbType[] }), subtype: subtypeArr, limit: 10000 }) for (const edge of edges) { // Defensive: related() honors the subtype filter, but check // again in case a future impl widens it. if (edge.subtype !== undefined && !subtypeSet.has(edge.subtype)) continue const neighbor = dir === 'from' ? edge.to : edge.from if (visited.has(neighbor)) continue visited.add(neighbor) reached.add(neighbor) nextFrontier.add(neighbor) } } } frontier = nextFrontier } return reached } if (from) { for (const id of await bfsWithSubtype(from, direction)) connectedIds.add(id) } if (to) { const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both' for (const id of await bfsWithSubtype(to, reverse)) connectedIds.add(id) } } else { // No subtype filter — fast path via neighbors(), which does the same // depth-aware BFS but only filters by verbType. const collect = (id: string, dir: 'incoming' | 'outgoing' | 'both'): Promise => this.neighbors(id, { direction: dir, depth: effectiveDepth, verbType: via }) if (from) { for (const id of await collect(from, toNeighborDir(direction))) connectedIds.add(id) } if (to) { const reverse: 'in' | 'out' | 'both' = direction === 'in' ? 'out' : direction === 'out' ? 'in' : 'both' for (const id of await collect(to, toNeighborDir(reverse))) connectedIds.add(id) } } } await populate() // Cold-load guard: an empty connected set is suspicious. The native adjacency can report // size()>0 (or isReady()===false) on a cold open yet have loaded NO source→target edges — so // traversal silently returns []. Re-verify against the honest isReady() signal (or, for older // providers, a GLOBAL known-edge sample — NOT the queried anchor, which may be genuinely // edgeless). If the adjacency was dead and a rebuild healed it, re-collect; if it stays dead, // verifyGraphAdjacencyLive() throws GraphIndexNotReadyError. A genuinely edgeless anchor // verifies 'live' and the empty result stands — no spurious rebuild/throw. if (connectedIds.size === 0) { const verdict = await this.verifyGraphAdjacencyLive() if (verdict === 'rebuilt') { await populate() } } // Filter existing results to only connected entities if (existingResults.length > 0) { return existingResults.filter(r => connectedIds.has(r.id)) } // Batch-load connected entities for fast cloud-storage performance const results: Result[] = [] const ids = [...connectedIds] const entitiesMap = await this.batchGet(ids) for (const id of ids) { const entity = entitiesMap.get(id) if (entity) { results.push(this.createResult(id, 1.0, entity)) } } return results } /** * Apply fusion scoring for multi-source results */ private applyFusionScoring(results: Result[], fusionType: any): Result[] { // Implement different fusion strategies const strategy = typeof fusionType === 'string' ? fusionType : fusionType.strategy || 'weighted' switch (strategy) { case 'max': // Use maximum score from any source return results case 'average': // Average scores from multiple sources const scoreMap = new Map() for (const result of results) { const scores = scoreMap.get(result.id) || [] scores.push(result.score) scoreMap.set(result.id, scores) } return results.map(r => ({ ...r, score: scoreMap.get(r.id)!.reduce((a, b) => a + b, 0) / scoreMap.get(r.id)!.length })) case 'weighted': default: // Weighted combination based on source importance const weights = fusionType.weights || { vector: 0.7, metadata: 0.2, graph: 0.1 } return results.map(r => ({ ...r, score: r.score * (weights.vector || 1.0) })) } } /** * Execute text search using word index * * Performs keyword-based search using the __words__ index in MetadataIndexManager. * Returns results ranked by word match count. * * @param query - Text query to search for * @param limit - Maximum results to return * @returns Array of Results with scores based on match count */ private async executeTextSearch(query: string, limit: number): Promise[]> { const textMatches = await this.metadataIndex.getIdsForTextQuery(query) if (textMatches.length === 0) return [] // Take top matches and load entities const topMatches = textMatches.slice(0, limit * 2) // Get more for filtering const ids = topMatches.map(m => m.id) const entitiesMap = await this.batchGet(ids) // Create results with scores based on match count const maxMatches = topMatches[0]?.matchCount || 1 const results: Result[] = [] for (const match of topMatches) { const entity = entitiesMap.get(match.id) if (entity) { // Normalize score to 0-1 range based on match count const score = match.matchCount / maxMatches results.push(this.createResult(match.id, score, entity)) } } return results } /** * Auto-detect optimal alpha for hybrid search * * Short queries (1-2 words) favor text search (lower alpha) * Long queries (5+ words) favor semantic search (higher alpha) * * @param query - The search query * @returns Alpha value between 0 (text only) and 1 (semantic only) */ private autoAlpha(query: string): number { const wordCount = query.trim().split(/\s+/).filter(w => w.length > 0).length if (wordCount <= 2) return 0.3 // Favor text for short queries if (wordCount <= 5) return 0.5 // Balanced return 0.7 // Favor semantic for long queries } /** * Reciprocal Rank Fusion (RRF) for combining search results * * RRF is a proven fusion algorithm that: * - Doesn't require score normalization * - Handles different score distributions * - Gives higher weight to top-ranked items * * Formula: score(d) = sum(1 / (k + rank(d))) for each list * * Now includes match visibility (textMatches, textScore, semanticScore, matchSource) * * @param textResults - Results from text search * @param semanticResults - Results from semantic search * @param alpha - Weight for semantic (0=text only, 1=semantic only) * @param queryWords - Original query words for match tracking * @param k - RRF constant (default: 60, standard in literature) * @returns Fused results sorted by combined score with match visibility */ private async rrfFusion( textResults: Result[], semanticResults: Result[], alpha: number, queryWords: string[], k: number = 60 ): Promise[]> { // Track scores and match details per entity interface MatchData { rrf: number textScore?: number semanticScore?: number textMatches: string[] hasText: boolean hasSemantic: boolean } const matchData = new Map() const entityMap = new Map>() // Text contribution (1 - alpha weight) const textWeight = 1 - alpha textResults.forEach((r, rank) => { const rrfScore = textWeight * (1 / (k + rank + 1)) const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false } existing.rrf += rrfScore existing.textScore = r.score // Original text search score (0-1) existing.hasText = true matchData.set(r.id, existing) if (r.entity) entityMap.set(r.id, r.entity) }) // Semantic contribution (alpha weight) semanticResults.forEach((r, rank) => { const rrfScore = alpha * (1 / (k + rank + 1)) const existing = matchData.get(r.id) || { rrf: 0, textMatches: [], hasText: false, hasSemantic: false } existing.rrf += rrfScore existing.semanticScore = r.score // Original semantic search score (0-1) existing.hasSemantic = true matchData.set(r.id, existing) if (r.entity) entityMap.set(r.id, r.entity) }) // Sort by fused score const sortedIds = Array.from(matchData.entries()) .sort((a, b) => b[1].rrf - a[1].rrf) .map(([id, data]) => ({ id, data })) // Build results - need to load any missing entities const missingIds = sortedIds.filter(s => !entityMap.has(s.id)).map(s => s.id) if (missingIds.length > 0) { const loaded = await this.batchGet(missingIds) for (const [id, entity] of loaded) { entityMap.set(id, entity) } } // Performance: Build set of text result IDs for O(1) lookup // This avoids re-extracting text for entities that weren't in text results const textResultIds = new Set(textResults.map(r => r.id)) // Create final results with match visibility const results: Result[] = [] for (const { id, data } of sortedIds) { const entity = entityMap.get(id) if (entity) { // Find which query words matched - uses fast path if entity wasn't in text results const textMatches = this.findMatchingWords(entity, queryWords, textResultIds) // Determine match source let matchSource: 'text' | 'semantic' | 'both' if (data.hasText && data.hasSemantic) { matchSource = 'both' } else if (data.hasText) { matchSource = 'text' } else { matchSource = 'semantic' } // Create result with match visibility const result = this.createResult(id, data.rrf, entity) result.textMatches = textMatches result.textScore = data.textScore result.semanticScore = data.semanticScore result.matchSource = matchSource results.push(result) } } return results } /** * Find which query words match in an entity's text content * * Performance: O(query_words × text_length) - only called when needed * At scale: Use textResultIds set for O(1) lookup instead of re-extracting * * @param entity - Entity to check * @param queryWords - Words from the search query * @param textResultIds - Optional: Set of IDs from text search (O(1) lookup) * @returns Array of matching query words */ private findMatchingWords( entity: Entity, queryWords: string[], textResultIds?: Set ): string[] { // Fast path: if entity wasn't in text results, no words matched if (textResultIds && !textResultIds.has(entity.id)) { return [] } // Slow path: extract text and check each word // Only happens for entities that DID match text search const textContent = this.metadataIndex.extractTextContent({ data: entity.data, metadata: entity.metadata }).toLowerCase() return queryWords.filter(word => textContent.includes(word.toLowerCase())) } /** * Apply graph constraints using O(1) GraphAdjacencyIndex - TRUE Triple Intelligence! */ private async applyGraphConstraints( results: Result[], constraints: any ): Promise[]> { // Filter by graph connections using fast graph index if (constraints.to || constraints.from) { const filtered: Result[] = [] for (const result of results) { let hasConnection = false if (constraints.to) { // Check if this entity connects TO the target (O(1) lookup) const outgoingNeighbors = await this.getNeighborUuids(result.id, { direction: 'out' }) hasConnection = outgoingNeighbors.includes(constraints.to) } if (constraints.from && !hasConnection) { // Check if this entity connects FROM the source (O(1) lookup) const incomingNeighbors = await this.getNeighborUuids(result.id, { direction: 'in' }) hasConnection = incomingNeighbors.includes(constraints.from) } if (hasConnection) { filtered.push(result) } } return filtered } return results } /** * Convert storage-shape verbs to public `Relation`s. The storage layer has * already done the canonical reserved/custom split (every combine goes * through `hydrateVerbWithMetadata` — see src/types/reservedFields.ts), so * this is a pure top-level field mapping: `metadata` carries ONLY custom * fields, and every reserved field (`subtype`, `weight`, `confidence`, * timestamps, `service`, `data`) surfaces at top level. */ private verbsToRelations(verbs: GraphVerb[]): Relation[] { return verbs.map((v) => { return { id: v.id, from: v.sourceId, to: v.targetId, type: (v.verb || v.type) as VerbType, ...(v.subtype !== undefined && { subtype: v.subtype }), ...(v.visibility !== undefined && { visibility: v.visibility }), weight: v.weight ?? 1.0, ...(typeof v.confidence === 'number' && { confidence: v.confidence }), data: v.data, metadata: v.metadata, service: v.service as string, createdAt: typeof v.createdAt === 'number' ? v.createdAt : Date.now(), ...(typeof v.updatedAt === 'number' && { updatedAt: v.updatedAt }) } }) } /** * Embed data into vector representation * Handles any data type by intelligently converting to string representation * * @param data - Any data to convert to vector (string, object, array, etc.) * @returns Promise that resolves to a numerical vector representation * * @example * // Basic string embedding * const vector = await brainy.embed('machine learning algorithms') * console.log('Vector dimensions:', vector.length) * * @example * // Object embedding with intelligent field extraction * const documentVector = await brainy.embed({ * title: 'AI Research Paper', * content: 'This paper discusses neural networks...', * author: 'Dr. Smith', * category: 'machine-learning' * }) * // Uses 'content' field for embedding by default * * @example * // Different object field priorities * // Priority: data > content > text > name > title > description * const vectors = await Promise.all([ * brainy.embed({ data: 'primary content' }), // Uses 'data' * brainy.embed({ content: 'main content' }), // Uses 'content' * brainy.embed({ text: 'text content' }), // Uses 'text' * brainy.embed({ name: 'entity name' }), // Uses 'name' * brainy.embed({ title: 'document title' }), // Uses 'title' * brainy.embed({ description: 'description text' }) // Uses 'description' * ]) * * @example * // Array embedding for batch processing * const batchVectors = await brainy.embed([ * 'first document', * 'second document', * { content: 'third document as object' }, * { title: 'fourth document' } * ]) * // Returns vector representing all items combined * * @example * // Complex object handling * const complexData = { * user: { name: 'John', role: 'developer' }, * project: { name: 'AI Assistant', status: 'active' }, * metrics: { score: 0.95, performance: 'excellent' } * } * const vector = await brainy.embed(complexData) * // Converts entire object to JSON for embedding * * @example * // Pre-computing vectors for performance optimization * const documents = [ * { id: 'doc1', content: 'Document 1 content...' }, * { id: 'doc2', content: 'Document 2 content...' }, * { id: 'doc3', content: 'Document 3 content...' } * ] * * // Pre-compute all vectors * const vectors = await Promise.all( * documents.map(doc => brainy.embed(doc.content)) * ) * * // Add entities with pre-computed vectors (faster) * for (let i = 0; i < documents.length; i++) { * await brainy.add({ * data: documents[i], * type: NounType.Document, * vector: vectors[i] // Skip embedding computation * }) * } * * @example * // Custom embedding for search queries * async function searchWithCustomEmbedding(query: string) { * // Enhance query for better matching * const enhancedQuery = `search: ${query} relevant information` * const queryVector = await brainy.embed(enhancedQuery) * * // Use pre-computed vector for search * return brainy.find({ * vector: queryVector, * limit: 10 * }) * } * * @example * // Handling edge cases gracefully * const edgeCases = await Promise.all([ * brainy.embed(null), // Returns vector for empty string * brainy.embed(undefined), // Returns vector for empty string * brainy.embed(''), // Returns vector for empty string * brainy.embed(42), // Converts number to string * brainy.embed(true), // Converts boolean to string * brainy.embed([]), // Empty array handling * brainy.embed({}) // Empty object handling * ]) * * @example * // Using with similarity comparisons * const doc1Vector = await brainy.embed('artificial intelligence research') * const doc2Vector = await brainy.embed('machine learning algorithms') * * // Find entities similar to doc1Vector * const similar = await brainy.find({ * vector: doc1Vector, * limit: 5 * }) */ async embed(data: any): Promise { // Handle different data types intelligently let textToEmbed: string | string[] if (typeof data === 'string') { textToEmbed = data } else if (Array.isArray(data)) { // Array of items - convert each to string textToEmbed = data.map(item => { if (typeof item === 'string') return item if (typeof item === 'number' || typeof item === 'boolean') return String(item) if (item && typeof item === 'object') { // For objects, try to extract meaningful text if (item.data) return String(item.data) if (item.content) return String(item.content) if (item.text) return String(item.text) if (item.name) return String(item.name) if (item.title) return String(item.title) if (item.description) return String(item.description) // Fallback to JSON for complex objects try { return JSON.stringify(item) } catch { return String(item) } } return String(item) }) } else if (data && typeof data === 'object') { // Single object - extract meaningful text if (data.data) textToEmbed = String(data.data) else if (data.content) textToEmbed = String(data.content) else if (data.text) textToEmbed = String(data.text) else if (data.name) textToEmbed = String(data.name) else if (data.title) textToEmbed = String(data.title) else if (data.description) textToEmbed = String(data.description) else { // For complex objects, create a descriptive string try { textToEmbed = JSON.stringify(data) } catch { textToEmbed = String(data) } } } else if (data === null || data === undefined) { // Handle null/undefined gracefully textToEmbed = '' } else { // Numbers, booleans, etc - convert to string textToEmbed = String(data) } return this.embedder(textToEmbed) } /** * Explicitly warm up the embedding engine * * Use this to pre-initialize the Candle WASM embedding engine before * processing requests. The WASM module (93MB with embedded model) takes * 90-140 seconds to compile on throttled CPU environments like Cloud Run. * * Calling this during container startup ensures the first real request * doesn't pay the compilation cost. * * @example * ```typescript * // Option 1: Use eagerEmbeddings config (automatic during init) * const brain = new Brainy({ eagerEmbeddings: true }) * await brain.init() // Embedding engine initialized here * * // Option 2: Manual warmup (more control) * const brain = new Brainy() * await brain.init() * await brain.warmupEmbeddings() // Explicit control over timing * ``` * * @returns Promise that resolves when embedding engine is ready */ async warmupEmbeddings(): Promise { if (!this.initialized) { throw new Error('Brain must be initialized before warming up embeddings. Call init() first.') } // Plugin-provided embeddings are already ready (native, no WASM warmup needed) if (this.pluginRegistry.hasProvider('embeddings')) { return } console.log('Warming up embedding engine...') const start = Date.now() await embeddingManager.init() const elapsed = Date.now() - start console.log(`Embedding engine ready in ${elapsed}ms`) } /** * Check if embedding engine is initialized * * @returns true if embedding engine is ready for immediate use */ isEmbeddingReady(): boolean { // Plugin-provided embeddings are always ready (native, no WASM init required) if (this.pluginRegistry.hasProvider('embeddings')) { return true } return embeddingManager.isInitialized() } /** * Setup embedder */ private setupEmbedder(): EmbeddingFunction { // Custom model loading removed - not implemented // Only 'fast' and 'accurate' model types are supported return defaultEmbeddingFunction } /** * Setup storage */ /** * @description 7.x → 8.0 on-disk LAYOUT migration, run BEFORE `setupStorage()`. * * 7.x stored every object branch-scoped under `branches//`; * 8.0 stores the IDENTICAL entity structure at the storage ROOT * (`entities////…`) plus the generational `_system` + * `_generations` layer. `GenerationStore.open()` is tolerant of a missing * manifest (opens at generation 0), so a naive 8.0 open of a 7.x directory * reports zero entities and SILENTLY LOSES ALL DATA. This phase collapses the * HEAD branch's canonical entities to the root so the rest of init (and any * native plugin storage factory, whose own legacy guard would otherwise THROW) * sees a clean flat-v8 store; 8.0 rebuilds all derived state from those entities. * * Runs on a temporary BUILT-IN `FileSystemStorage` (never the plugin adapter), * is idempotent (a `_system/migration-layout.json` marker short-circuits re-open), * resume-safe (per-object read→write→delete), and a strict NO-OP for any brain * that is not a filesystem store carrying a legacy `branches/` layout. */ private async legacyLayoutMigrationPhase(): Promise { // Pre-built adapter instances and non-filesystem stores have no on-disk 7.x // layout to migrate. if (isStorageAdapterInstance(this.config.storage)) return const storageConfig = (this.config.storage || {}) as StorageOptions & Record const type = storageConfig.type || 'auto' if (type !== 'filesystem' && type !== 'auto') return // Fast path: the only on-disk shape that needs migrating has a top-level // `branches/` directory. A native 8.0 brain and a fresh directory do not, so // skip the probe-storage construction entirely on the init hot path. (The // migration is node-filesystem-only; `fs.existsSync` is absent/throwing // elsewhere, which the catch treats as "nothing to migrate".) // // Resolve the root through the SHARED resolver so the migration probe checks // the SAME directory createStorage (and any plugin factory) will open, and a // removed pre-8.0 alias (`rootDirectory`, `options.*`, `fileSystemStorage.*`) // throws here too — consistent with createStorage, never a silent skip. const rootDir = resolveFilesystemRoot(storageConfig) try { if (!fs.existsSync(`${rootDir}/branches`)) return } catch { return } // Probe with a BUILT-IN filesystem storage over the same root (never the // plugin factory — cor's adapter would throw on the legacy layout). In a // non-filesystem environment this throws; that just means "nothing to migrate". let probe: BaseStorage try { probe = (await createStorage({ ...storageConfig, type: 'filesystem' })) as BaseStorage await probe.init() } catch { return } try { // Idempotency: a completed migration leaves a marker → re-open is a no-op. const marker = await probe.readRawObject('_system/migration-layout.json') if (marker && (marker as { layout?: string }).layout === 'flat-v8') return // DETECT: HEAD-branch entities present under branches//entities/. const head = (storageConfig.branch as string) || 'main' const branchPaths = await probe.listRawObjects('branches') const headEntityPrefix = `branches/${head}/entities/` const legacyEntityPaths = branchPaths.filter((p) => p.startsWith(headEntityPrefix)) if (legacyEntityPaths.length === 0) { // Already flat (root entities, no head-branch entities) → stamp the marker // so future opens short-circuit. A genuinely empty/fresh dir gets no marker. const rootEntities = await probe.listRawObjects('entities') if (rootEntities.length > 0) { await probe.writeRawObject('_system/migration-layout.json', { layout: 'flat-v8', version: 8, fromBranch: null, entitiesMoved: 0 }) } return } // GUARD: an explicit autoMigrate:false must not silently lose 7.x data. if (this.config.autoMigrate === false) { throw new Error( `Brainy 8.0 found a 7.x branch layout at this storage directory ` + `(${legacyEntityPaths.length} entities under branches/${head}/). Opening it as-is ` + `would report zero entities and lose your data. Set autoMigrate: true (the default) ` + `to migrate the layout in place on open, or export the data on 7.x first.` ) } // Acquire the writer lock for the duration of the move (idempotent marker // makes a lost race harmless, but the lock prevents two concurrent collapses). const canLock = typeof (probe as unknown as { supportsMultiProcessLocking?: () => boolean }) .supportsMultiProcessLocking === 'function' && (probe as unknown as { supportsMultiProcessLocking: () => boolean }).supportsMultiProcessLocking() const lockable = probe as unknown as { acquireWriterLock?: (o: { force?: boolean }) => Promise releaseWriterLock?: () => Promise } if (canLock && typeof lockable.acquireWriterLock === 'function') { await lockable.acquireWriterLock({ force: this.config.force }) } if (!this.config.silent) { console.warn( `[brainy] Migrating a 7.x branch layout (branches/${head}) to the 8.0 flat layout ` + `in place — ${legacyEntityPaths.length} entity files. This runs once; back up the ` + `directory first if you need a rollback (8.0 does not keep the old layout).` ) } try { // COLLAPSE: branches//entities/* → entities/* via the .gz-transparent // raw primitives (NOT fs.rename — listRawObjects returns logical paths and // the in-memory adapter has no real files). read→write→delete is idempotent, // so a crash mid-move is recovered by simply re-running. let moved = 0 for (const src of legacyEntityPaths) { const target = src.slice(`branches/${head}/`.length) // → entities/... const data = await probe.readRawObject(src) if (data === null) continue // already moved by an interrupted prior run await probe.writeRawObject(target, data) await probe.deleteRawObject(src) moved++ } // REBUILD persisted derived state from the now-flat canonical entities. // (The three indexes — HNSW, metadata, graph — are rebuilt by the normal // init's rebuildIndexesIfNeeded after storage is set up; the COUNT rollups // are NOT, so they must be rebuilt + persisted here or getNounCount()/stats() // read 0 on the migrated store.) // - rebuildCounts → totalNounCount/totalVerbCount + counts.json (getNounCount/getVerbCount) // - rebuildTypeCounts → per-type type-statistics.json (stats().entitiesByType) await rebuildCounts(probe) await probe.rebuildTypeCounts() if (typeof (probe as unknown as { rebuildSubtypeCounts?: () => Promise }).rebuildSubtypeCounts === 'function') { await (probe as unknown as { rebuildSubtypeCounts: () => Promise }).rebuildSubtypeCounts() } // FINALIZE: mark migrated (makes re-open a no-op), then drain the head // branch. Non-head branches under branches/ are 7.x version history 8.0's // MVCC does not import — they are left in place. await probe.writeRawObject('_system/migration-layout.json', { layout: 'flat-v8', version: 8, fromBranch: head, entitiesMoved: moved }) // Parity guard (defense-in-depth; the VFS `_cow/` stranding lesson). The // entity move deleted every `branches//entities/*` key, so anything // STILL under `branches//` is non-entity durable state a 7.x engine // wrote outside `entities/` (branch-scoped blobs, an index dir, a field // registry). Draining it unconditionally would silently DELETE it — the // same failure class as the VFS content blobs stranded in `_cow/`. So // drain only a clean branch (nothing but the moved entities); if any // non-entity object survives, PRESERVE the branch (skip the drain) so the // state stays recoverable, and surface it loudly. const residual = ( await probe.listRawObjects(`branches/${head}`) ).filter((k: string) => !k.startsWith(`branches/${head}/entities/`)) if (residual.length === 0) { await probe.removeRawPrefix(`branches/${head}`) } else if (!this.config.silent) { const sample = residual.slice(0, 8).join(', ') console.warn( `[brainy] 7→8 migration preserved ${residual.length} non-entity object(s) under ` + `branches/${head}/ — branch-scoped durable state written outside entities/. The ` + `branch was NOT drained, so this state is recoverable; inspect it before removing ` + `branches/ manually. Keys: ${sample}${residual.length > 8 ? ', …' : ''}` ) } } finally { if (canLock && typeof lockable.releaseWriterLock === 'function') { await lockable.releaseWriterLock() } } } finally { if (typeof (probe as unknown as { close?: () => Promise }).close === 'function') { await (probe as unknown as { close: () => Promise }).close() } } } /** * @description On-open recovery for VFS content blobs a 7→8 upgrade left in the * removed branch system's copy-on-write area (`_cow/`). If a `_cow/` area is * present and a completed adoption has not already been recorded, adopt every * orphaned blob into the 8.0 content-addressed store (`_cas/`) and stamp a * marker so later opens skip the scan. Because it runs on every open right * after {@link legacyLayoutMigrationPhase}, it heals BOTH a fresh 7→8 migration * (where `_cow/` still holds the blobs) AND a brain already migrated by an * earlier 8.0.x that stranded them — no operator action needed. * * Filesystem-only; a native-8.0 / fresh brain has no `_cow/` and returns on the * cheap existence check. Non-destructive and idempotent (see * {@link BaseStorage.adoptLegacyCowBlobs}). If any blob cannot be fully adopted * (`incomplete > 0`), the marker is NOT stamped (the next open retries) and * {@link _vfsBlobAdoptionIncomplete} is set so the pre-upgrade backup is * retained — the upgrade is not verifiably complete for the VFS. */ private async autoAdoptLegacyVfsBlobsIfNeeded(): Promise { if (this.getStorageType() !== 'filesystem') return // Cheapest gate: no copy-on-write area → a native-8.0 / fresh brain, nothing // a 7.x brain could have stranded here. try { const root = resolveFilesystemRoot( (this.config.storage || {}) as StorageOptions & Record ) if (!fs.existsSync(`${root}/_cow`)) return } catch { return } const storage = this.storage as unknown as { adoptLegacyCowBlobs?: () => Promise<{ cowBlobs: number adopted: number alreadyPresent: number incomplete: number }> readRawObject?: (p: string) => Promise writeRawObject?: (p: string, o: unknown) => Promise } if (typeof storage.adoptLegacyCowBlobs !== 'function') return const MARKER = '_system/vfs-blob-adoption.json' try { const done = (await storage.readRawObject?.(MARKER)) as { complete?: boolean } | null if (done && done.complete) return // already fully adopted on a prior open } catch { // no marker yet — proceed } const result = await storage.adoptLegacyCowBlobs() if (result.incomplete > 0) { // Partial: retry on future opens, and hold the pre-upgrade backup. this._vfsBlobAdoptionIncomplete = true if (!this.config.silent) { console.error( `[brainy] VFS blob recovery adopted ${result.adopted} blob(s) but ${result.incomplete} ` + `orphaned _cow/ blob(s) are missing their bytes or metadata and were left in place. ` + `The pre-upgrade backup is being retained; inspect _cow/ before discarding it.` ) } return } // Clean sweep — record it so later opens skip the _cow/ scan entirely. try { await storage.writeRawObject?.(MARKER, { complete: true, adopted: result.adopted }) } catch { // marker write is best-effort; adoption already succeeded } if (result.adopted > 0 && !this.config.silent) { console.warn( `[brainy] Recovered ${result.adopted} VFS content blob(s) stranded by a 7→8 upgrade ` + `(adopted _cow/ → _cas/ in place).` ) } } private async setupStorage(): Promise { // If the caller passed a pre-constructed storage adapter (e.g. // `storage: new MemoryStorage()`, or the historical materializer's // pre-populated snapshot storage), use it directly instead of routing // through the factory. Otherwise the factory's `type === 'auto'` branch // would silently create a FileSystemStorage at `./brainy-data`, ignoring // the instance and surprising anyone who wrote `new MemoryStorage()` // expecting it to be honoured. if (isStorageAdapterInstance(this.config.storage)) { return this.config.storage as BaseStorage } // The config-object branch of `BrainyConfig['storage']` is a structural // subset of the factory's `StorageOptions`; the Record view serves the // plugin factory contract (`create(config: Record)`). const storageConfig = (this.config.storage || {}) as StorageOptions & Record const storageType = storageConfig.type || 'auto' // Check plugin-provided storage factories (e.g., a native filesystem/mmap // override registered by an acceleration plugin). const pluginFactory = this.pluginRegistry.getStorageFactory(storageType) if (pluginFactory) { // Hand the plugin factory a NORMALIZED config whose canonical `path` is // the SAME root our built-in resolver computed. The plugin's native side // (e.g. getBinaryBlobPath / mmap files) re-resolves the directory itself; // without this, an alias-only config (`{ rootDirectory }`, `{ options: // { path } }`, …) would resolve to the alias here but to ./brainy-data on // the plugin side — a brainy-data/cor split-brain where the two halves // write to different directories. Stamping the resolved `path` keeps both // resolvers on the identical root. const normalizedConfig = { ...storageConfig, path: resolveFilesystemRoot(storageConfig) } const adapter = await pluginFactory.create(normalizedConfig) return adapter as BaseStorage } // Fall through to built-in storage types const storage = await createStorage(storageConfig) return storage as BaseStorage } /** * Detect storage type from the storage instance class name * * Fixes storage type detection for HNSW persistence mode. * Previously relied on this.config.storage.type which was often not set * after storage creation, causing cloud storage to use 'immediate' mode * and resulting in 50-100x slower add() operations. * * @returns Storage type string ('gcs', 's3', 'memory', etc.) */ private getStorageType(): string { if (!this.storage) return 'memory' const className = this.storage.constructor.name if (className.includes('Gcs') || className.includes('GCS')) return 'gcs' if (className.includes('S3')) return 's3' if (className.includes('R2')) return 'r2' if (className.includes('Azure')) return 'azure' if (className.includes('OPFS')) return 'opfs' if (className.includes('FileSystem')) return 'filesystem' if (className.includes('Memory')) return 'memory' return 'unknown' } /** * Setup index — single unified vector graph. * * Persistence mode is auto-selected from the storage adapter (see * {@link resolveHNSWPersistMode}): `'immediate'` on filesystem storage, * `'deferred'` on memory storage. Operators can override with an explicit * `config.vector.persistMode` (e.g. `'deferred'` for bulk-ingest speed on * filesystem storage). */ private setupIndex(): JsHnswVectorIndex { // 8.0 config surface: config.vector.{recall, persistMode}. // The recall preset translates to HNSW knobs (M / efConstruction / efSearch) // via resolveJsHnswConfig. No algorithm-internal knobs are exposed at the // public surface. The JS index computes exact float32 distances throughout — // there is no quantization path. const recallKnobs = resolveJsHnswConfig(this.config.vector) const indexConfig = { ...recallKnobs, distanceFunction: this.distance } const persistMode = this.resolveHNSWPersistMode() return new JsHnswVectorIndex(indexConfig, this.distance, { storage: this.storage, useParallelization: true, persistMode }) } /** * Create the vector index, using the plugin-provided engine when available. * Shared by init() and clear() to avoid duplication. * * Selection order (first match wins): * 1. `'vector'` provider if registered — the canonical 8.0 provider key. * A native plugin registers one internally-adaptive engine here; * in-memory / hybrid / on-disk selection is the provider's job, not * Brainy's, so there is no engine-specific branching on this side. * 2. Brainy's built-in TS JsHnswVectorIndex (the always-available fallback). */ private createIndex(): JsHnswVectorIndex { const persistMode = this.resolveHNSWPersistMode() const vectorFactory = this.pluginRegistry.getProvider<(config: any, distance: DistanceFunction, options: any) => any>('vector') if (vectorFactory) { // Native providers receive the algorithm-neutral surface: the resolved // recall preset name (per the 8.0 provider contract they translate it // to their own internal knobs), plus the JS HNSW knob tuple for // providers that mirror the open-core implementation. Quantization at // scale is the native provider's own concern (e.g. DiskANN PQ) — Brainy // exposes no quantization knob. return vectorFactory( { ...resolveJsHnswConfig(this.config.vector), recall: this.config.vector?.recall ?? DEFAULT_RECALL, distanceFunction: this.distance }, this.distance, { storage: this.storage, persistMode } ) } return this.setupIndex() } /** * Resolve the vector persistence mode from the storage adapter. * * Auto-selected when `config.vector.persistMode` is omitted: * - **filesystem** (and any persistent adapter): `'immediate'` — per-add * durability is the point of a persistent backend. * - **memory**: `'deferred'` — nothing survives the process anyway, so * per-add persistence writes are pure overhead; deferred batches them * into `flush()` / `close()`. * * An explicit `config.vector.persistMode` always wins (the operator * escape hatch for bulk-ingest pipelines on filesystem storage). */ private resolveHNSWPersistMode(): 'immediate' | 'deferred' { if (this.config.vector?.persistMode) { return this.config.vector.persistMode } return this.getStorageType() === 'memory' ? 'deferred' : 'immediate' } /** * Normalize and validate configuration */ private normalizeConfig(config?: BrainyConfig): ResolvedBrainyConfig { // Validate storage configuration. Brainy 8.0 ships two adapters only — // FileSystemStorage and MemoryStorage (cloud + OPFS adapters were removed). // Cloud backup remains supported via operator tooling (db.persist() + // gsutil / aws s3 cp / rclone / azcopy). Pre-constructed adapter // instances bypass the type check (they ARE the storage). const storageConfig = config?.storage if ( storageConfig && !isStorageAdapterInstance(storageConfig) && storageConfig.type && !['auto', 'memory', 'filesystem'].includes(storageConfig.type) ) { throw new Error( `Invalid storage type: ${storageConfig.type}. Brainy 8.0 ships 'auto', 'memory', and 'filesystem' only. ` + `Cloud storage adapters (GCS / S3 / R2 / Azure) and OPFS were removed in 8.0; ` + `back up locally with db.persist() and sync the on-disk artefact with your tool of choice.` ) } return { storage: config?.storage || { type: 'auto' }, verbose: config?.verbose ?? false, silent: config?.silent ?? false, // false = auto-decide based on dataset size (inline vs lazy rebuild) disableAutoRebuild: config?.disableAutoRebuild ?? false, // Memory management escape hatches over the RAM-derived auto limits maxQueryLimit: config?.maxQueryLimit ?? undefined, reservedQueryMemory: config?.reservedQueryMemory ?? undefined, // Migration LOCK wait budget — left undefined when omitted so // awaitMigrationLock() applies its 30 s default at the read site. migrationWaitTimeoutMs: config?.migrationWaitTimeoutMs ?? undefined, // Pre-upgrade backup — default-on; opt out with `migrationBackup: false`. migrationBackup: config?.migrationBackup ?? true, // Vector index configuration (8.0) — algorithm-neutral surface with // `recall` preset + `persistMode` (folded in from 7.x's hnswPersistMode). // See BRAINY-8.0-RENAME-COORDINATION § A.2 + § G.2. vector: config?.vector ?? undefined, // Generational-history retention — the `retention` knob; defaults applied // at the read site (resolveRetentionPolicy) so a partial object inherits // per-field defaults and `undefined` resolves to ADAPTIVE. retention: config?.retention ?? undefined, // Embedding initialization — left undefined when omitted so the adaptive // default resolves at the init() read site (eager when the WASM embedder // is the active one on a non-reader writer outside unit tests). Explicit // true/false always wins. See the eagerEmbeddings JSDoc in brainy.types.ts. eagerEmbeddings: config?.eagerEmbeddings ?? undefined, // Plugin configuration - undefined = auto-detect plugins: config?.plugins ?? undefined, // Integration Hub - undefined/false = disabled integrations: config?.integrations ?? undefined, // Migration — auto-run by default. Pending data migrations apply inline // during init() for small datasets (<10K entities); larger datasets // defer with a notice so the operator can schedule brain.migrate(). autoMigrate: config?.autoMigrate ?? true, // Subtype required-by-default (8.0 — per BRAINY-8.0-SUBTYPE-CONTRACT § C-1). // Every write path now requires `subtype` on every type unless the consumer // opts out explicitly. Opt-out: `requireSubtype: false` (last-resort migration // hatch for old data or legacy fixtures) or `requireSubtype: { except: [...] }` // (per-type allowlist). requireSubtype: config?.requireSubtype ?? true, // Multi-process safety mode: config?.mode ?? 'writer', force: config?.force ?? false, // Reserved-field-in-metadata-bag policy (8.0 — no silent failures). // Default 'throw': an untyped caller that smuggles a reserved key past // the compile guard gets a loud Error naming the correct write path. // 'warn' = remap + one-shot warning per key; 'remap' = legacy silent remap. reservedFieldPolicy: config?.reservedFieldPolicy ?? 'throw' } } /** * Ensure indexes are loaded (Production-scale lazy loading) * * Called by query methods (find, search, get, etc.) when disableAutoRebuild is true. * Handles concurrent queries safely - multiple calls wait for same rebuild. * * Performance: * - First query: Triggers rebuild (~50-200ms for 1K-10K entities) * - Concurrent queries: Wait for same rebuild (no duplicate work) * - Subsequent queries: Instant (0ms check, indexes already loaded) * * Production scale: * - 1K entities: ~50ms * - 10K entities: ~200ms * - 100K entities: ~2s (streaming pagination) * - 1M+ entities: Uses chunked lazy loading (per-type on demand) */ private async ensureIndexesLoaded(): Promise { // Fast path: If rebuild already completed, return immediately (0ms) if (this.lazyRebuildCompleted) { return } // If indexes already populated AND honestly serving, mark complete and skip. // Honest gate: when the provider exposes isReady(), that REPLACES the size()>0 // proxy (a native index can report a non-zero size while its serving structure // is not loaded — the silent-empty cold-load class). A not-ready provider falls // through so the rebuild path can load it; verifyVectorLive() is the query-time // backstop either way. Providers without isReady() keep the size() heuristic // (the JS index's size()>0 genuinely means loaded). const vectorReadiness = assessIndexReadiness(this.index) if (vectorReadiness === 'ready' || (vectorReadiness === 'unknown' && this.index.size() > 0)) { this.lazyRebuildCompleted = true return } // Migration LOCK (#18) deference: while the vector provider runs its one-time // 7.x → 8.0 rebuild-from-canonical, a first query must NOT trigger brainy's // force-rebuild — the provider owns that index. Normally unreachable here: the // data-plane lock (awaitMigrationLock) makes the caller wait upstream, so a // query only reaches this point once the migration has cleared. Defensive // (no `lazyRebuildCompleted` latch) so the check re-runs: once the provider // clears the lock, `index.size() > 0` above ends the lazy path normally. if (this.providerIsMigrating(this.index)) { return } // Concurrency control: If rebuild is in progress, wait for it if (this.lazyRebuildInProgress && this.lazyRebuildPromise) { await this.lazyRebuildPromise return } // Check if lazy rebuild is needed // Only needed if: disableAutoRebuild=true AND indexes are empty AND storage has data if (!this.config.disableAutoRebuild) { // Auto-rebuild is enabled, indexes should already be loaded return } // Check if storage has data (fast check with limit=1) const entities = await this.storage.getNouns({ pagination: { limit: 1 } }) const hasData = (entities.totalCount && entities.totalCount > 0) || entities.items.length > 0 if (!hasData) { // Storage is empty, no rebuild needed this.lazyRebuildCompleted = true return } // Start lazy rebuild (with mutex to prevent concurrent rebuilds) this.lazyRebuildInProgress = true this.lazyRebuildPromise = this.rebuildIndexesIfNeeded(true) .then(() => { this.lazyRebuildCompleted = true }) .finally(() => { this.lazyRebuildInProgress = false this.lazyRebuildPromise = null }) await this.lazyRebuildPromise } /** * @description Wire the shared UUID ↔ int resolver (the metadata index's * idMapper) into the JS graph index and the storage layer (8.0 u64 * contract). Called after the graph index is resolved on init so every * consumer of the BigInt boundary shares one int * universe. Native graph-index providers carry their own mapper and don't * expose `setEntityIdMapper` — the storage-level wiring still applies so * its verb read paths can convert UUIDs before provider calls. */ private wireGraphIdResolver(): void { const resolver = this.metadataIndex.getIdMapper() this.storage.setGraphEntityIdResolver(resolver) const jsGraphIndex = this.graphIndex as Partial> if (typeof jsGraphIndex.setEntityIdMapper === 'function') { jsGraphIndex.setEntityIdMapper(resolver) } } /** * @description Wire the HNSW connections codec (2.4.0 #3). Activates when * BOTH (a) the `graph:compression` provider is registered (cor registers * `{ encode: encodeConnections, decode: decodeConnections }`), and (b) the * metadata index exposes a stable idMapper. Failures are non-fatal: HNSW * keeps working via the legacy JSON-array path. * * This layer does NOT require a real local path — the binary-blob primitive * saves the compressed bytes through the storage adapter directly, so the * codec works on the memory adapter as well as filesystem. (A native vector * provider, by contrast, persists its own single `.dkann` file and has no * per-node connection lists, so it is skipped — see the feature-detect below.) * Format convergence is lazy: pre-2.4.0 nodes load via the legacy path, * then the next dirty save writes the compressed form (and the legacy * field of saveVectorIndexData becomes empty), so all reads converge over time * without an explicit migration step. */ private wireConnectionsCodec(): void { const provider = this.pluginRegistry.getProvider('graph:compression') if (!provider) return const idMapper = this.metadataIndex.getIdMapper?.() if (!idMapper) return // Feature-detect: only the JS HNSW path uses per-node connection lists. // Native vector-index providers (DiskANN-style) persist the graph as a // single mmap'd file and have no analogue, so skip the wiring there. // Pre-8.0 the wire was unconditional and relied on the native wrapper // exposing a no-op setConnectionsCodec(); 8.0 makes it feature-detected. if (typeof (this.index as { setConnectionsCodec?: unknown }).setConnectionsCodec !== 'function') return const codec = new ConnectionsCodec(provider, idMapper) this.index.setConnectionsCodec(codec) if (!this.config.silent) { console.log('[brainy] graph link compression wired (delta-varint connections via graph:compression provider)') } } /** * Rebuild indexes from persisted data if needed (LAZY LOADING) * * FIXES FOR CRITICAL BUGS: * - Bug #1: GraphAdjacencyIndex rebuild never called ✅ FIXED * - Bug #2: Early return blocks recovery when count=0 ✅ FIXED * - Bug #4: HNSW index has no rebuild mechanism ✅ FIXED * - Bug #5: disableAutoRebuild leaves indexes empty forever ✅ FIXED * * Production-grade rebuild with: * - Handles BILLIONS of entities via streaming pagination * - Smart threshold-based decisions (auto-rebuild < 1000 items) * - Lazy loading on first query (when disableAutoRebuild: true) * - Progress reporting for large datasets * - Parallel index rebuilds for performance * - Robust error recovery (continues on partial failures) * - Concurrency-safe (multiple queries wait for same rebuild) * * @param force - Force rebuild even if disableAutoRebuild is true (for lazy loading) */ private async rebuildIndexesIfNeeded(force = false): Promise { try { // Check if auto-rebuild is explicitly disabled (ONLY during init, not for lazy loading) // force=true means this is a lazy rebuild triggered by first query if (this.config.disableAutoRebuild === true && !force) { if (!this.config.silent) { console.log('⚡ Auto-rebuild explicitly disabled via config') console.log('💡 Indexes will build automatically on first query (lazy loading)') } return } // No instant fast-path here: the honest per-leg readiness checks below // are all O(1) (one bounded storage sample + each provider's size()/ // isReady()), and this method runs exactly once per open (init calls it; // the lazy path passes force=true). The removed shortcut keyed off // `this.index.size() > 0`, a dishonest proxy — it skipped the metadata // and graph checks whenever the vector happened to be warm, and it never // fired on a real cold process (the JS vector size is 0 until it loads). // BUG #2 FIX: Don't trust counts - check actual storage instead // Counts can be lost/corrupted in container restarts const entities = await this.storage.getNouns({ pagination: { limit: 1 } }) const totalCount = entities.totalCount || 0 // If storage is truly empty, no rebuild needed if (totalCount === 0 && entities.items.length === 0) { if (force && !this.config.silent) { console.log('✅ Storage empty - no rebuild needed') } // A fresh / empty brain's (empty) derived indexes are trivially current // for this build's epoch — stamp the version marker so the next open // recognises it as current and skips the drift rebuild. await this.stampBrainFormatIfNeeded() return } // Intelligent decision: Auto-rebuild based on dataset size // Production scale: Handles billions via streaming pagination const AUTO_REBUILD_THRESHOLD = 10000 // Auto-rebuild if < 10K items (increased from 1K) // Check if indexes need rebuilding const metadataStats = await this.metadataIndex.getStats() const hnswIndexSize = this.index.size() // Readiness contract: when a provider exposes isReady(), that honest // signal REPLACES the size/count heuristic below — an mmap/disk-native // index legitimately reports 0 resident entries while fully durable on // disk, and rebuilding it from canonical re-reads every entity file on // every boot (the 48-seconds-per-restart class a production deployment // hit). The signal is honest in BOTH directions: a provider whose // durable state failed to load returns false and gets its rebuild even // when size() > 0 (the silent-empty cold-load failure). Providers // without isReady() keep the exact prior empty-heuristics. const providerReady = (leg: unknown): boolean | undefined => { const candidate = leg as { isReady?: () => boolean } return typeof candidate.isReady === 'function' ? candidate.isReady() : undefined } const metadataReady = providerReady(this.metadataIndex) const vectorReady = providerReady(this.index) const graphReady = providerReady(this.graphIndex) // Epoch-drift trigger: a format-version change makes EVERY derived index // suspect even when each is non-empty, so it forces a rebuild of all // three from the canonical records (not just the empty ones below). const epochStale = this._indexEpochStale // Migration LOCK (#18) deference: when a native provider holds the lock for // its one-time 7.x → 8.0 rebuild-from-canonical, brainy must NOT rebuild // that index — the provider owns it in place and clears the lock only after // it verifies and calls brain.stampBrainFormat(). Reads and writes are held // by awaitMigrationLock meanwhile (nothing serves from a half-built index). // Gated per-index, so a non-migrating sibling still rebuilds when it needs // to; a migrating provider is skipped even under epoch-drift or size()===0. const metadataMigrating = this.providerIsMigrating(this.metadataIndex) const vectorMigrating = this.providerIsMigrating(this.index) const graphMigrating = this.providerIsMigrating(this.graphIndex) const anyMigrating = metadataMigrating || vectorMigrating || graphMigrating // Per-leg decision, in precedence order: a migrating provider owns its // index (skip) → epoch drift forces a rebuild → an exposed isReady() // decides → otherwise a per-leg fallback. The fallbacks differ by leg // because "empty" means different things: // - METADATA: past the empty-store early-return, entities exist, so the // id-mapper SHOULD have loaded entries — totalEntries===0 is a real // load-failure signal, so rebuild (self-heal from canonical). // - VECTOR: the JS vector has no passive cold-load — rebuild() IS its // load path — so size()===0 correctly triggers the load. // - GRAPH: entities do NOT imply edges, so size()===0 is a VALID empty // state, not a load failure. The JS graph cold-loads (and self-heals // against canonical) inside storage.getGraphIndex() BEFORE this gate, // so it is already authoritative here; re-deriving would be spurious // (a full O(E) verb scan on every open of an edgeless brain). It // therefore rebuilds only on epoch drift or a native !isReady(). // (verifyGraphAdjacencyLive is the query-time backstop.) const shouldRebuildMetadata = !metadataMigrating && (epochStale || (metadataReady !== undefined ? !metadataReady : metadataStats.totalEntries === 0)) const shouldRebuildVector = !vectorMigrating && (epochStale || (vectorReady !== undefined ? !vectorReady : hnswIndexSize === 0)) const shouldRebuildGraph = !graphMigrating && (epochStale || (graphReady !== undefined ? !graphReady : false)) const needsRebuild = shouldRebuildMetadata || shouldRebuildVector || shouldRebuildGraph if (!needsRebuild && !force) { // All indexes report current — durably loaded (isReady/size), or owned // by a background migration. No rebuild needed. return } // Determine rebuild strategy const isLazyRebuild = force && this.config.disableAutoRebuild === true const isSmallDataset = totalCount < AUTO_REBUILD_THRESHOLD const shouldRebuild = isLazyRebuild || isSmallDataset || this.config.disableAutoRebuild === false if (!shouldRebuild) { // Large dataset with auto-rebuild disabled: Wait for lazy loading if (!this.config.silent) { console.log(`⚡ Large dataset (${totalCount.toLocaleString()} items) - using lazy loading for optimal startup`) console.log('💡 Indexes will build automatically on first query') } return } // REBUILD: Either small dataset, forced rebuild, or explicit enable const rebuildReason = isLazyRebuild ? '🔄 Lazy loading triggered by first query' : isSmallDataset ? `🔄 Small dataset (${totalCount.toLocaleString()} items)` : '🔄 Auto-rebuild explicitly enabled' if (!this.config.silent) { // Name exactly which legs rebuild — "all indexes" was a lie whenever // the durable legs were skipped (e.g. only the JS vector index loads // here on a warm reopen), and it misread as a whole-brain rebuild in // consumer boot logs. const rebuildingLegs = [ shouldRebuildMetadata && 'metadata', shouldRebuildVector && 'vector', shouldRebuildGraph && 'graph' ] .filter(Boolean) .join(' + ') console.log(`${rebuildReason} - loading/rebuilding ${rebuildingLegs || 'no'} index(es) from persisted data...`) } // Before the graph rebuild, hydrate the entity id-mapper from the persisted // snapshot. A native int-keyed adjacency resolves every verb endpoint through // this mapper, so it must reflect the persisted int assignments BEFORE // graphIndex.rebuild() runs — otherwise edges resolve to stale/missing ints // and are silently dropped (CTX-BR-RESTORE-REBUILD). Mirrors restore()'s // ordering; a no-op for the JS mapper (re-derived by metadataIndex.rebuild()). if (shouldRebuildGraph) { await this.hydrateIdMapperForGraphRebuild() } // Rebuild the needed indexes in parallel for performance. Indexes load // their data from storage (no recomputation). An epoch drift (`epochStale`) // rebuilds every NON-migrating index regardless of its current size — the // on-disk format changed, so each is rebuilt from the canonical records. A // provider running its own background migration is skipped here (it owns // its index until it verifies-and-swaps). const rebuildStartTime = Date.now() await Promise.all([ shouldRebuildMetadata ? this.metadataIndex.rebuild() : Promise.resolve(), shouldRebuildVector ? this.index.rebuild() : Promise.resolve(), shouldRebuildGraph ? this.graphIndex.rebuild() : Promise.resolve() ]) const rebuildDuration = Date.now() - rebuildStartTime const metadataCountAfter = (await this.metadataIndex.getStats()).totalEntries if (!this.config.silent) { console.log( `All indexes rebuilt in ${rebuildDuration}ms:\n` + ` - Metadata: ${metadataCountAfter} entries\n` + ` - HNSW Vector: ${this.index.size()} nodes\n` + ` - Graph Adjacency: ${await this.graphIndex.size()} relationships` ) } // Consistency verification: metadata index must match storage entity count. // If mismatch, the rebuild missed entities — force a second attempt. Skipped // when the metadata provider holds the migration lock: a 0 count there // reflects its in-place rebuild in progress, not a missed rebuild, so // forcing a second rebuild would collide with the provider's own. if (metadataCountAfter === 0 && totalCount > 0 && !metadataMigrating) { console.error( `[Brainy] CRITICAL: Metadata index has 0 entries but storage has ${totalCount} entities. ` + `Forcing second rebuild.` ) await this.metadataIndex.rebuild() const secondAttempt = (await this.metadataIndex.getStats()).totalEntries console.log(`[Brainy] Second rebuild result: ${secondAttempt} entries`) } // 8.0 ⇄ native-provider handshake (NON-DESTRUCTIVE): the derived indexes // have now rebuilt and verified, so they match this build's epoch — // re-stamp the marker LAST, only here. A crash anywhere above leaves the // old / absent marker, so the next open re-detects the drift and re-runs // the (idempotent) rebuild; the marker is never advanced ahead of the // indexes it certifies. A no-op when the epoch was already current. // // EXCEPT while a provider holds the migration lock: brainy deferred that // index's rebuild, so it is NOT yet at this epoch. The marker (which // certifies ALL derived indexes) must not advance ahead of the index the // provider is still rebuilding in place — the provider calls // brain.stampBrainFormat() itself once it has verified parity. if (!anyMigrating) { await this.stampBrainFormatIfNeeded() } } catch (error) { // A storage READ failure here is surfaced by getNouns/getVerbs as a named // BrainyError — it means we could not even read the store to decide whether // a rebuild is needed (or to perform it). That is a hard init failure, not a // "start anyway" condition: booting a brain whose data we couldn't read would // serve empty/incomplete results with no signal (the silent-failure class the // 8.0 contract forbids). Re-throw it loud. if (error instanceof BrainyError) throw error // Any other rebuild hiccup is non-fatal — the brain may start on partially // rebuilt indexes — but it is recorded as a queryable degraded state // (surfaced via checkHealth) and escalated to error-level logging, rather // than only emitting a console.warn that is trivially lost. this._indexRebuildFailed = error instanceof Error ? error : new Error(String(error)) console.error('[Brainy] Index rebuild failed; starting in a DEGRADED state (queries may be incomplete):', error) } } /** * @description Re-stamp the 8.0 ⇄ native-provider version-handshake marker * (`_system/brain-format.json`) to this build's {@link CURRENT_DATA_FORMAT} / * {@link EXPECTED_INDEX_EPOCH} — but ONLY when the on-disk marker was absent * or carried a drifted epoch ({@link _indexEpochStale}). Called at the * verified-completion points of {@link rebuildIndexesIfNeeded} (after the * derived-index rebuild, or on the empty-storage fast path), so the marker is * never advanced ahead of the indexes it certifies — a crash before this * point re-triggers the idempotent rebuild on the next open. Readers never * write, so this is a no-op in reader mode. */ private async stampBrainFormatIfNeeded(): Promise { if (this.isReadOnly) return if (!this._indexEpochStale) return await writeBrainFormat(this.storage) this._brainFormat = { dataFormat: CURRENT_DATA_FORMAT, indexEpoch: EXPECTED_INDEX_EPOCH } this._indexEpochStale = false // The JS inline rebuild verified + stamped — drop the pre-upgrade backup. await this.removeMigrationBackupSafe() } /** * @description Whether an index provider holds the coordinated migration LOCK * (#18). A native provider's `init()` flips an OPTIONAL `isMigrating(): boolean` * to `true` the moment it detects a large 7.x epoch-drift and keeps it `true` * while it rebuilds all derived indexes IN PLACE from the canonical records, * clearing it only after it has verified parity and stamped the marker. While * `true`, brainy BLOCKS/QUEUES data-plane reads AND writes on the lock (see * {@link awaitMigrationLock}) so no operation touches a half-built index — * the "unknown/halfway state" is closed by waiting for a known-good one. * Feature-detected: a provider that omits `isMigrating` (every JS index) is * treated as not migrating, so behavior is unchanged. * @param provider - A registered index provider (metadata / vector / graph). * @returns `true` only when the provider exposes `isMigrating()` and it reports * an in-flight migration. */ private providerIsMigrating(provider: unknown): boolean { return ( provider != null && typeof (provider as { isMigrating?: () => boolean }).isMigrating === 'function' && (provider as { isMigrating: () => boolean }).isMigrating() === true ) } /** * @description `true` when ANY index provider (metadata / vector / graph) holds * the migration LOCK. The single predicate the data-plane gate and the status * surface consult; a brain with no migrating provider evaluates three cheap * feature-detects and returns `false`. */ private anyProviderMigrating(): boolean { return ( this.providerIsMigrating(this.metadataIndex) || this.providerIsMigrating(this.index) || this.providerIsMigrating(this.graphIndex) ) } /** * @description The provider instance backing a given index {@link IndexFamily}. * The single place the family label maps to the concrete provider, so the * scoped migration gate and any future family-aware routing agree. */ private providerForFamily(family: IndexFamily): unknown { switch (family) { case 'vector': return this.index case 'metadata': return this.metadataIndex case 'graph': return this.graphIndex } } /** * @description Whether the migration LOCK should hold for an operation that * depends on the given index families: * - `needs === undefined` → WHOLE-BRAIN: any migrating provider counts (the * conservative default for writes and unclassified reads — a write touches * every index). * - `needs === []` → NEVER: a canonical-storage read consults no derived index, * so a migration elsewhere is irrelevant; it serves immediately. * - `needs = [families]` → SCOPED: only those families' providers count, so a * read served from a healthy family is not blocked by an unrelated family's * one-time migration. */ private neededFamiliesMigrating(needs?: IndexFamily[]): boolean { if (needs === undefined) return this.anyProviderMigrating() for (const family of needs) { if (this.providerIsMigrating(this.providerForFamily(family))) return true } return false } /** * @description The index families a {@link find} query consults, derived from * its params — the read-side input to the family-scoped migration gate. A * vector / near / semantic query needs `vector`; a `where` / type / subtype / * service filter or an aggregate needs `metadata`; a `connected` traversal * needs `graph`. A query combining shapes needs the union. Mirrors find()'s * own criteria split so the gate and the executor never disagree. */ private queryIndexFamilies(params: FindParams): IndexFamily[] { const needs: IndexFamily[] = [] if ((params.query && params.query.trim() !== '') || params.vector || params.near) needs.push('vector') if (params.where || params.type || params.subtype || params.service || params.aggregate) needs.push('metadata') if (params.connected) needs.push('graph') return needs } /** * @description Rich migration progress relayed verbatim from whichever provider * exposes the OPTIONAL `migrationStatus(): { phase?, index?, percent?, … } | null`. * The provider owns the rebuild, so it owns the truth of the percentage; brainy * surfaces it through {@link getIndexStatus} and {@link health}. Returns `null` * when no provider reports structured progress (brainy then shows only * `migrating: true` + `elapsedMs`). Feature-detected and best-effort. */ private providerMigrationStatus(): MigrationProgress | null { for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { const fn = (provider as { migrationStatus?: () => unknown }).migrationStatus if (typeof fn === 'function') { try { const status = fn.call(provider) if (status && typeof status === 'object') { return status as MigrationProgress } } catch { // best-effort: a provider status hiccup never breaks the gate/probe } } } return null } /** * @description Block-and-queue on the coordinated migration LOCK (#18). Called * at the {@link ensureInitialized} choke point (and once inside `init()` before * VFS bootstrap), so a data-plane operation waits here while a native provider * runs its one-time 7.x → 8.0 rebuild-from-canonical. The caller gets the * correct answer, never a partial read of a half-built index and never a lost * write. A brain that is not migrating pays a single boolean check and returns * immediately. * * FAMILY-SCOPED: `needs` names the index families the caller depends on, so the * wait holds ONLY for a migration of one of those families. A read served * entirely from canonical storage (`needs: []`) or from a healthy family never * blocks on an unrelated family's migration. `needs` omitted = the conservative * whole-brain wait (writes touch every index; startup wants all of them ready). * @param needs - Index families this operation consults, or `undefined` for the * whole-brain wait. * * IMPORTANT: this timeout bounds THE CALLER'S WAIT, not the migration. The * migration itself is unbounded — the native provider rebuilds a billion-scale * brain for as long as it needs; brainy cannot and does not abort it. The lock * is polled every {@link Brainy.MIGRATION_POLL_INTERVAL_MS}; a small/medium * upgrade clears well within `migrationWaitTimeoutMs` (default 30 s) and the * caller resumes transparently. If the wait outlives the budget, a retryable * {@link MigrationInProgressError} is thrown — the upgrade keeps running in the * background, so the caller retries, watches `getIndexStatus().migration` * (never gated — the primary large-upgrade signal for a readiness probe), or, * for a very large brain, runs the offline migrator. The block/release lines * log once per window; the flags reset on release so a later upgrade re-logs. */ private async awaitMigrationLock(needs?: IndexFamily[]): Promise { if (!this.neededFamiliesMigrating(needs)) return // fast path — the overwhelming common case const startedAt = Date.now() const timeoutMs = this.config.migrationWaitTimeoutMs ?? 30_000 if (!this._migrationBlockLogged) { this._migrationBlockLogged = true prodLog.info( '[Brainy] Auto-upgrade in progress (7.x → 8.0): reads and writes are blocked ' + 'until the rebuild completes. This is automatic and one-time; data is safe.' ) } while (this.neededFamiliesMigrating(needs)) { const remaining = timeoutMs - (Date.now() - startedAt) if (remaining <= 0) { const status = this.providerMigrationStatus() throw new MigrationInProgressError( `Brain is still upgrading (7.x → 8.0)` + (status?.percent !== undefined ? `, ${Math.round(status.percent)}% complete` : '') + ` after ${Math.round((Date.now() - startedAt) / 1000)}s. The wait timed out; the upgrade ` + `continues in the background. Retry shortly, watch brain.getIndexStatus().migration for ` + `progress, raise config.migrationWaitTimeoutMs to wait longer, or for a very large brain ` + `run the offline migrator. Nothing is lost.`, Date.now() - startedAt, status?.percent ) } await new Promise((r) => setTimeout(r, Math.min(Brainy.MIGRATION_POLL_INTERVAL_MS, remaining))) } if (this._migrationBlockLogged && !this._migrationReleaseLogged) { this._migrationReleaseLogged = true prodLog.info( `[Brainy] Auto-upgrade complete in ${Math.round((Date.now() - startedAt) / 1000)}s — ` + 'reads and writes resumed.' ) } // Reset the once-per-window guards so a subsequent upgrade in this same // instance re-logs cleanly and re-clocks its elapsed (a fresh observed-at). this._migrationBlockLogged = false this._migrationReleaseLogged = false this._migrationObservedAt = null } /** * @description The lock-exempt observability snapshot consumed by * {@link getIndexStatus} and {@link health}: whether the brain is upgrading and, * if so, the provider's structured {@link MigrationProgress} enriched with a * `startedAt` / `elapsedMs` that brainy tracks itself (so a provider that omits * timing still yields a live elapsed). Reads the migration flag without waiting, * so a readiness probe can always report `migrating` → HTTP 503 + Retry-After * rather than routing traffic into a half-built index. Lazily stamps the * observed-at clock on first sight and clears it once the lock releases. */ private migrationSnapshot(): { migrating: boolean; migration?: MigrationProgress } { if (!this.anyProviderMigrating()) { this._migrationObservedAt = null return { migrating: false } } if (this._migrationObservedAt === null) { this._migrationObservedAt = Date.now() } const provided = this.providerMigrationStatus() return { migrating: true, migration: { ...provided, startedAt: provided?.startedAt ?? this._migrationObservedAt, elapsedMs: provided?.elapsedMs ?? Date.now() - this._migrationObservedAt } } } /** * @description Take the pre-upgrade backup (default-on) before a native provider * rebuilds anything for a 7.x → 8.0 migration. Called at open, once the on-disk * format is known stale, BEFORE the providers init. Feature-detected + best-effort: * a backend without `createMigrationBackup` (memory) or a store with no data is a * no-op, and a backup failure NEVER blocks the upgrade (the migration is itself * safe — it only reads canonical, and self-heals on re-open). The snapshot is * removed once the upgrade verifies ({@link removeMigrationBackupSafe}) and * retained on failure for rollback. */ private async createMigrationBackupIfNeeded(): Promise { const storage = this.storage as StorageAdapter & { createMigrationBackup?: () => Promise } if (typeof storage.createMigrationBackup !== 'function') return try { const backupPath = await storage.createMigrationBackup() if (backupPath) { this._migrationBackupPath = backupPath prodLog.info( `[Brainy] Pre-upgrade backup created at ${backupPath} (hard-link snapshot; ~zero ` + `cost/space). Removed automatically once the 7.x→8.0 upgrade verifies; retained ` + `for rollback if it fails. Opt out with { migrationBackup: false }.` ) } } catch (error) { prodLog.warn( `[Brainy] Pre-upgrade backup could not be created (continuing — the upgrade is safe ` + `and self-heals): ${error instanceof Error ? error.message : String(error)}` ) } } /** * @description Remove the pre-upgrade backup once the migration has verified and * stamped the format marker. Called from both stamp paths (the JS inline rebuild * and the native provider's `stampBrainFormat()`), gated on a backup having been * taken this open. Best-effort: a removal failure leaves a harmless backup dir. */ private async removeMigrationBackupSafe(): Promise { if (!this._migrationBackupPath) return // Backup-parity gate: the index rebuild "success" that calls this does NOT // certify the VFS. If the on-open blob adoption could not fully bridge the // 7.x `_cow/` blobs, the upgrade is not verifiably complete for VFS content // — retain the backup so the operator can recover from it, and log why. if (this._vfsBlobAdoptionIncomplete) { if (!this.config.silent) { console.warn( `[brainy] Retaining the pre-upgrade backup at ${this._migrationBackupPath}: ` + `some VFS content blobs could not be adopted from _cow/ (see the earlier warning). ` + `Resolve those before discarding the backup.` ) } return } const backupPath = this._migrationBackupPath this._migrationBackupPath = null const storage = this.storage as StorageAdapter & { removeMigrationBackup?: (location: string) => Promise } if (typeof storage.removeMigrationBackup !== 'function') return try { await storage.removeMigrationBackup(backupPath) prodLog.info(`[Brainy] Upgrade verified — pre-upgrade backup at ${backupPath} removed.`) } catch { // best-effort — a leftover backup dir is harmless } } /** * Check health of metadata indexes * * Returns validation result indicating whether indexes are healthy * or corrupted (e.g., from the update() field asymmetry bug). * * This check was previously run on every init(), causing significant * overhead on cloud storage (90+ sequential reads for 30-field datasets). * Now available as an on-demand diagnostic method. */ async checkHealth(): Promise<{ healthy: boolean avgEntriesPerEntity: number entityCount: number indexEntryCount: number recommendation: string | null }> { // Lock-exempt: diagnostics must answer while the brain upgrades. await this.ensureInitialized({ bypassMigrationLock: true }) const result = await this.metadataIndex.validateConsistency() // Fold in a non-fatal index-rebuild failure recorded at init() so the degraded // state is observable through the same health surface (not just the console). if (this._indexRebuildFailed) { return { ...result, healthy: false, recommendation: `Index rebuild failed at init() (degraded — queries may be incomplete): ` + `${this._indexRebuildFailed.message}. Rebuild the indexes and re-open.` + (result.recommendation ? ` Also: ${result.recommendation}` : '') } } if (this._indexDegradedIds.size > 0) { return { ...result, healthy: false, recommendation: `${this._indexDegradedIds.size} record(s) committed via adopt-forward ` + `failed-rollback recovery — the derived index may be incomplete for them. ` + `Run repairIndex() to reconcile.` + (result.recommendation ? ` Also: ${result.recommendation}` : '') } } return result } /** * Run the optional metadata cold-open consistency probe at most once per brain. * When the active provider exposes `probeConsistency()` (the native cross-bucket * O(1) sampler), a `false` result triggers `detectAndRepairCorruption()` so an * already-poisoned index self-heals on first read — the metadata counterpart of * the 7.33.2 graph cold-load guard. Best-effort: a probe failure never breaks the * read (the guard is reset so a transient failure retries). No-op for the JS index * (it exposes no probe), and the full-scan `validateConsistency` stays the explicit * deep diagnostic via `validateIndexConsistency()`. */ private async ensureMetadataConsistencyProbed(): Promise { if (this._metadataConsistencyProbed) return // Defer while the metadata provider runs its one-time in-place migration: // probing (and self-healing via rebuild) an index the provider is mid-rebuild // would collide with the provider that owns it. Mirrors the vector deference // in ensureIndexesLoaded. Do NOT latch — once the migration clears, the next // read runs the probe. (The family-scoped find() gate waits on the metadata // family separately before any actual filter read.) if (this.providerIsMigrating(this.metadataIndex)) return this._metadataConsistencyProbed = true const provider = this.metadataIndex as { probeConsistency?: () => Promise detectAndRepairCorruption?: () => Promise } if (typeof provider.probeConsistency !== 'function') return try { const healthy = await provider.probeConsistency() if (!healthy && typeof provider.detectAndRepairCorruption === 'function') { if (!this.config.silent) { console.warn('[Brainy] metadata index failed the cold-open consistency probe — self-healing via rebuild.') } await provider.detectAndRepairCorruption() } } catch (error) { // The self-heal is best-effort and must never break a read. Reset the guard // so a transient probe failure is retried on the next read. this._metadataConsistencyProbed = false if (!this.config.silent) { console.warn('[Brainy] metadata cold-open consistency probe failed (continuing):', error) } } } /** * Detect and repair corrupted metadata indexes. * * Runs corruption detection and auto-rebuilds if corruption is found. This is * the equivalent of the old init()-time corruption check, now available as an * explicit operation. * * It is ALSO the recovery path for a write-quarantine: when a transaction's * rollback fails and leaves the store inconsistent ({@link StoreInconsistentError}), * writes are refused until this method reconciles the derived indexes against * canonical storage (a forced rebuild — orphaned/lost records reflected * consistently) and lifts the quarantine. Canonical is the source of truth: a * genuinely lost record cannot be resurrected here (restore from a snapshot for * that), but the store is made internally consistent and writes re-enabled. */ /** * Emit ONE loud warning per degraded window when a read is served while the * derived index is known-incomplete — either a non-fatal init rebuild failure * ({@link _indexRebuildFailed}) or an adopt-forward degraded commit * ({@link _indexDegradedIds}). Reads still return (canonical is the source of * truth), but the caller is told the result may be partial and how to heal it. * No-op when healthy or when the brain is configured `silent`. * @param op - The read method name for the message (e.g. `'find'`, `'get'`). */ private warnIfReadsDegraded(op: string): void { const degraded = this._indexRebuildFailed !== null || this._indexDegradedIds.size > 0 if (!degraded) { this._degradedReadWarned = false return } if (this._degradedReadWarned || this.config.silent) return this._degradedReadWarned = true const reason = this._indexRebuildFailed ? `index rebuild failed at init() (${this._indexRebuildFailed.message})` : `${this._indexDegradedIds.size} record(s) committed in a degraded state` prodLog.warn( `[Brainy] ${op}() is serving reads while the derived index is INCOMPLETE ` + `(${reason}). Results may be missing entries — run repairIndex() to ` + `reconcile the derived indexes against canonical storage.` ) } /** * Read-only graph-truth audit — proves (or disproves) that relationship * reads return canonical truth on THIS brain, and classifies every * discrepancy into its failure family: * * - `missingFromReads` — canonical verb records the read path omits * (PRESENT BUT INVISIBLE: adjacency/membership staleness) * - `danglingEndpoints` — verbs whose endpoint entity is gone (SCAR class) * - `readOnlyVerbIds` — read-path edges with no canonical record (GHOSTS) * * Design-hidden edges (internal/system visibility) are counted separately — * the audit reads with every visibility tier included, so intentional * hiding is never misclassified as index loss. Mutates nothing; safe on a * live brain (cost: one canonical walk + one indexed read per source). * Run it after any engine upgrade, restore, or migration; a `coherent` * report is the verified statement that `related()`/`readdir` can be * trusted. If it reports discrepancies, `repairIndex()` is the sanctioned * heal — re-run the audit afterwards to prove the repair. * * @param options.maxExamples - Cap per example list (counts stay exact). Default 100. * @returns The full audit report; also narrated via logs (loud on incoherence). * @since 8.6.0 */ async auditGraph(options: { maxExamples?: number } = {}): Promise { await this.ensureInitialized({ needs: ['graph'] }) const PAGE = 1000 return runGraphAudit( { eachNounId: async (consume) => { let cursor: string | undefined let offset = 0 for (;;) { const page = await (this.storage as unknown as { getNounIdsWithPagination(o: { limit: number offset?: number cursor?: string }): Promise<{ ids: string[]; hasMore: boolean; nextCursor?: string }> }).getNounIdsWithPagination( cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } ) for (const id of page.ids) consume(id) if (!page.hasMore || page.ids.length === 0) break if (page.nextCursor) cursor = page.nextCursor else offset += page.ids.length } }, eachVerb: async (consume) => { let cursor: string | undefined let offset = 0 for (;;) { const page = await this.storage.getVerbs({ pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } }) for (const verb of page.items) { const v = verb as unknown as Record consume({ id: String(v.id), type: String(v.verb ?? 'unknown'), sourceId: String(v.sourceId), targetId: String(v.targetId), visibility: typeof v.visibility === 'string' ? v.visibility : undefined }) } if (!page.hasMore || page.items.length === 0) break if (page.nextCursor) cursor = page.nextCursor else offset += page.items.length } }, readRelationsFrom: async (sourceId) => this.related({ from: sourceId, includeInternal: true, includeSystem: true, limit: 100000 }) }, options ) } async repairIndex(): Promise { await this.ensureInitialized() // Prune orphaned canonical containers left by the pre-8.3.1 partial-delete // defect: a delete that removed the metadata (content) leg but left the // vector leg + the entity directory (a "ghost"), or left an empty directory // (a "scar"). These are not live entities (getNoun needs the content leg) // yet inflate enumerated counts and confuse locator resolution. Feature- // detected (filesystem-only — key/prefix stores have no orphan containers); // recompute counts afterward so the totals stop counting the ghosts. const pruner = this.storage as { pruneOrphanedEntities?: () => Promise<{ nouns: string[]; verbs: string[] }> rebuildTypeCounts?: () => Promise rebuildSubtypeCounts?: () => Promise } if (typeof pruner.pruneOrphanedEntities === 'function') { const orphans = await pruner.pruneOrphanedEntities() if (orphans.nouns.length + orphans.verbs.length > 0) { prodLog.warn( `[Brainy] repairIndex() pruned ${orphans.nouns.length} orphaned noun + ` + `${orphans.verbs.length} orphaned verb container(s) left by a pre-8.3.1 ` + `partial delete.` ) } } // SANCTIONED RECOUNT — unconditional, not gated on orphans found: the // persisted counters can be inflated over perfectly clean shelves (deletes // whose decrement was skipped by the removed-record re-read), and // Math.max(totalNounCount, scanned) means an inflated scalar can never // correct itself. rebuildTypeCounts() recomputes EVERY counter rollup // (scalar totals + per-type maps + type-statistics arrays) from one // canonical walk and persists them. await pruner.rebuildTypeCounts?.() await pruner.rebuildSubtypeCounts?.() // The recount changed the rollup truth — re-stamp the entity tree so the // stamp's invariants match the healed counters (repair leaves a coherent // stamp, not a stale one that warns on the next open). await this.stampEntityTree() // VFS containment reconciliation: heal "cosmetic ghost" edges left by // pre-fix renames (an entity Contains-linked from BOTH its old and new // directory — readdir listed it in two places) and duplicate edges from // concurrent writers. Canonical metadata.path is the truth; only VFS // containment edges are touched. Loud per repair. if (this._vfsInitialized && this._vfs) { const containment = await this._vfs.repairContainment() if (containment.removed + containment.restored > 0) { prodLog.warn( `[Brainy] repairIndex() reconciled VFS containment: removed ${containment.removed} ` + `stale/duplicate edge(s), restored ${containment.restored} missing edge(s).` ) } } await this.metadataIndex.detectAndRepairCorruption() // Lift a failed-rollback write-quarantine: force a full rebuild so the // derived indexes are provably reconciled with canonical, then clear the // flag so writes resume. if (this.storeInconsistency) { await this.rebuildIndexesIfNeeded(true) const cleared = this.storeInconsistency this.storeInconsistency = null prodLog.warn( `[Brainy] repairIndex() reconciled the store and LIFTED the write-quarantine ` + `set by a failed transaction rollback (${cleared.records.length} record(s) affected). ` + `Writes are re-enabled.` ) } // Cross-layer repair: repairIndex must reconcile NATIVE derived // state from canonical, not just the JS metadata index. Consult each provider's // own validateInvariants() and rebuild any whose failing invariant asks for it // (heal: 'rebuild') — the native counterpart of detectAndRepairCorruption(). for (const provider of [this.metadataIndex, this.index, this.graphIndex]) { const p = provider as { validateInvariants?: () => Promise rebuild?: () => Promise } | null if (!p || typeof p.validateInvariants !== 'function' || typeof p.rebuild !== 'function') continue let report: ProviderInvariantReport try { report = await p.validateInvariants() } catch { continue // a throwing validateInvariants is surfaced by validateIndexConsistency; skip repair here } if (report.healthy) continue if (report.invariants.some((i) => !i.holds && i.heal === 'rebuild')) { prodLog.warn( `[Brainy] repairIndex(): provider '${report.provider}' has a failing invariant ` + `requiring a rebuild — reconciling its derived state from canonical.` ) await p.rebuild() } } // detectAndRepairCorruption() above rebuilt the derived indexes from // canonical, so any adopt-forward degraded ids and a non-fatal init // rebuild failure are now reconciled — clear the queryable degraded state // and re-arm the read-path warning. if (this._indexDegradedIds.size > 0 || this._indexRebuildFailed) { this._indexDegradedIds.clear() this._indexRebuildFailed = null this._degradedReadWarned = false } } /** * Register a plugin manually. * * Must be called BEFORE init(). Plugins registered after init() * will not be activated. */ use(plugin: BrainyPlugin): this { this.pluginRegistry.register(plugin) return this } /** * Get list of active plugin names. */ getActivePlugins(): string[] { return this.pluginRegistry.getActivePlugins() } /** * Auto-detect and activate plugins. * Called internally during init(). */ /** * Import a plugin package by name. Isolated as a seam so tests can simulate * the three auto-detect outcomes (not installed / broken install / valid) * without the package being present. The specifier is a variable, so * bundlers cannot statically resolve — and cannot force-include — the * optional accelerator. */ private async importPluginPackage(pkg: string): Promise { return import(pkg) } /** * True when a dynamic-import failure means "the package itself is not * installed" (the silent free path), as opposed to "the package is present * but broken" (which must fail loud). Node and Bun both name the missing * package in the resolution error; a failure naming anything else — an * internal file, a dependency of the plugin, a syntax error — is a broken * install, never a not-installed. */ private static isPackageNotInstalledError(error: unknown, pkg: string): boolean { const code = (error as { code?: string })?.code const message = error instanceof Error ? error.message : String(error) const namesPackage = message.includes(`'${pkg}'`) || message.includes(`"${pkg}"`) || message.includes(` ${pkg}`) const isResolutionFailure = code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' || /cannot find (package|module)/i.test(message) || /failed to resolve/i.test(message) // Bun's resolver phrasing return isResolutionFailure && namesPackage } private async loadPlugins(): Promise { // plugins config: // undefined (default) → guarded auto-detection of the first-party // accelerator: installing @soulcraft/cor IS the // opt-in. Not installed → plain brainy, silently. // Installed → it loads and announces itself; if it // is installed but broken, init() THROWS — an // installed accelerator never silently vanishes. // false / [] → no plugins, no detection (explicit opt-out) // ['@soulcraft/cor'] → load only these explicitly listed packages // Note: plugins registered via brain.use() are always activated regardless of config const pluginConfig = this.config.plugins if (Array.isArray(pluginConfig) && pluginConfig.length > 0) { // Explicit list: import and register the specified packages. A package // listed here is REQUIRED — a missing or invalid plugin must fail LOUD, // never silently fall back to the default engine (the cross-repo drift // this whole guard exists to prevent). for (const pkg of pluginConfig) { let mod: any try { mod = await this.importPluginPackage(pkg) } catch (error) { throw new Error( `[brainy] Plugin "${pkg}" is listed in config.plugins but could not be loaded: ` + `${error instanceof Error ? error.message : String(error)}. ` + `Install it (e.g. npm i ${pkg}) or remove it from config.plugins.` ) } const plugin: BrainyPlugin = mod.default || mod if (plugin && typeof plugin.activate === 'function' && plugin.name) { this.pluginRegistry.register(plugin) } else { throw new Error( `[brainy] Package "${pkg}" (config.plugins) is not a valid Brainy plugin ` + `(missing { name, activate }). Remove it from config.plugins.` ) } } } else if (pluginConfig === undefined) { // Guarded auto-detection (default). Installing the first-party // accelerator is the opt-in — probe for it, and apply the SAME loud // posture as the explicit branch to everything except "not installed": // a present-but-broken accelerator must never silently degrade to JS. for (const pkg of Brainy.AUTO_DETECT_PLUGIN_PACKAGES) { let mod: any try { mod = await this.importPluginPackage(pkg) } catch (error) { if (Brainy.isPackageNotInstalledError(error, pkg)) { continue // the free path: not installed, nothing to load, no noise } throw new Error( `[brainy] The accelerator "${pkg}" is installed but failed to load: ` + `${error instanceof Error ? error.message : String(error)}. ` + `brainy will NOT silently run the default JS engines in its place — ` + `fix the install (npm i ${pkg}) or disable detection with plugins: [].` ) } const plugin: BrainyPlugin = mod.default || mod if (plugin && typeof plugin.activate === 'function' && plugin.name) { this.pluginRegistry.register(plugin) } else { throw new Error( `[brainy] The installed accelerator "${pkg}" is not a valid Brainy plugin ` + `(missing { name, activate }) — a broken or incompatible install. ` + `brainy will NOT silently run the default JS engines in its place — ` + `fix the install (npm i ${pkg}) or disable detection with plugins: [].` ) } } } // Create plugin context const context: BrainyPluginContext = { registerProvider: (key, impl) => this.pluginRegistry.registerProvider(key, impl), version: getBrainyVersion() } // Activate all registered plugins const activated = await this.pluginRegistry.activateAll(context) // Version-coupling guard (the provider-key cliff). A pre-8.0 native plugin // (e.g. @soulcraft/cor < 3.0) registers its vector engine under a LEGACY key // ('hnsw'/'diskann') instead of the 8.0 canonical 'vector'. brainy 8.x reads // only 'vector', so without this check the native engine silently never // engages and every query runs on the default JS index — exactly the // invisible degrade we must prevent. brainy never registers these keys // itself, so their presence can only come from a plugin. const LEGACY_VECTOR_KEYS = ['hnsw', 'diskann', 'hnswIndex'] const legacyVector = LEGACY_VECTOR_KEYS.filter( (k) => this.pluginRegistry.getProvider(k) !== undefined ) if (legacyVector.length > 0 && this.pluginRegistry.getProvider('vector') === undefined) { throw new Error( `[brainy] A native acceleration plugin registered a legacy vector provider ` + `(${legacyVector.join(', ')}) but not the 'vector' provider that brainy 8.x requires. ` + `This is an incompatible (pre-8.0) accelerator — upgrade to @soulcraft/cor >= 3.0. ` + `brainy will NOT silently run the default JS vector engine in its place.` ) } if (activated.length > 0) { // Only log if not in silent mode if (!this.config.silent) { for (const name of activated) { console.log(`[brainy] Plugin activated: ${name}`) } } } } /** * Execute an aggregate query, returning results as Result[] for API consistency. */ private async findAggregate(params: FindParams): Promise[]> { if (!this._aggregationIndex) { throw new Error('No aggregates defined. Call defineAggregate() first.') } // Normalize aggregate params const aggParams: AggregateQueryParams = typeof params.aggregate === 'string' ? { name: params.aggregate } : params.aggregate as AggregateQueryParams // Merge find-level params into aggregate query if (params.where && !aggParams.where) { aggParams.where = params.where as Record } if (params.orderBy && !aggParams.orderBy) { aggParams.orderBy = params.orderBy } if (params.order && !aggParams.order) { aggParams.order = params.order } if (params.limit !== undefined && aggParams.limit === undefined) { aggParams.limit = params.limit } if (params.offset !== undefined && aggParams.offset === undefined) { aggParams.offset = params.offset } // Backfill from already-stored entities if this aggregate was defined over a // populated store (write-time hooks only capture entities added after define). await this.backfillAggregateIfNeeded(aggParams.name) const aggregateResults = this._aggregationIndex.queryAggregate(aggParams) // Convert AggregateResult[] to Result[] for API consistency return aggregateResults.map((agg, index) => { const entity: Entity = { id: agg.entityId || `__agg_${aggParams.name}_${index}`, vector: [], type: NounType.Measurement, data: `${aggParams.name}: ${Object.entries(agg.groupKey).map(([k, v]) => `${k}=${v}`).join(', ')}`, metadata: { ...agg.groupKey, ...agg.metrics, __aggregate: aggParams.name, count: agg.count } as T, createdAt: Date.now(), service: 'brainy:aggregation' } return { id: entity.id, score: 1.0, type: NounType.Measurement, metadata: entity.metadata, data: entity.data, entity, // Surface the documented AggregateResult fields at the top level so consumers can read // groupKey/metrics/count directly. Previously these were only reachable under .metadata, // so callers expecting an AggregateResult saw rows with no groupKey/metrics/count and // interpreted the output as degenerate/empty. groupKey: agg.groupKey, metrics: agg.metrics, count: agg.count } }) } /** * Backfill a named aggregate from entities already in storage. * * Write-time hooks (`onEntityAdded` etc.) only capture entities added *after* an * aggregate is defined, so an aggregate defined over a populated store — the common * case under durable storage, where a brain reopens pre-populated — would otherwise * return `[]`. On first query we clear the aggregate's state and stream every stored * noun back through it. Storage-agnostic: `getNouns()` works for in-memory, the * filesystem adapter, and native (Cor) storage alike. One-time per definition — * the rebuilt state is persisted on flush() and reloaded on the next session. */ private async backfillAggregateIfNeeded(name: string): Promise { const index = this._aggregationIndex if (!index) return // Persisted-state adoption happens inside ready(); after it resolves the // pending-backfill set is authoritative (an unchanged definition with valid // persisted state is NOT listed — no walk at all on a clean reopen). await index.ready() // Single-flight: concurrent queries share ONE walk instead of each wiping // the others' partial state and starting their own (the stampede that kept // a busy store from ever converging). The loop covers the rare case where // the in-flight walk snapshotted its batch before `name` became pending — // the next iteration starts a fresh walk that includes it. while (index.getPendingBackfills().includes(name)) { // Failure latch: a deterministically-failing walk must not be re-run at // the caller's retry rate — that is a silent CPU loop wearing a retry // loop's clothes. Within the cooldown, rethrow the recorded failure // immediately; after it, one fresh attempt is allowed. const failure = this._aggregationBackfillFailure if (failure && Date.now() - failure.at < AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS) { throw new Error( `Aggregation backfill for '${name}' is in failure cooldown (retry in ` + `${Math.ceil((AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS - (Date.now() - failure.at)) / 1000)}s). ` + `Last failure: ${failure.error.message}` ) } if (!this._aggregationBackfillFlight) { this._aggregationBackfillFlight = this.runAggregationBackfillWalk() .finally(() => { this._aggregationBackfillFlight = null }) } await this._aggregationBackfillFlight } } /** * One store walk fills EVERY aggregate currently pending backfill — M pending * aggregates cost one enumeration, not M. Only reached when an aggregate * genuinely needs a rescan (new definition over a populated store, changed * definition, or failed state load); a clean reopen adopts persisted state * and never walks. */ private async runAggregationBackfillWalk(): Promise { const index = this._aggregationIndex! const names = index.getPendingBackfills() if (names.length === 0) return prodLog.info(`[Aggregation] backfill walk starting for: ${names.join(', ')}`) const startedAt = Date.now() for (const n of names) index.beginBackfill(n) let scanned = 0 try { const PAGE = 500 let offset = 0 let cursor: string | undefined for (;;) { const page = await this.storage.getNouns({ pagination: cursor ? { limit: PAGE, cursor } : { limit: PAGE, offset } }) for (const noun of page.items) { const record = noun as unknown as Record for (const n of names) { index.backfillEntity(n, record) } } scanned += page.items.length if (!page.hasMore || page.items.length === 0) break if (page.nextCursor) { if (page.nextCursor === cursor) { // A non-advancing cursor with hasMore=true would loop this walk at // CPU speed forever, silently. That is a storage pagination defect — // fail the waiting queries loudly instead of spinning. throw new Error( `Aggregation backfill aborted: storage pagination returned a non-advancing cursor ` + `after ${scanned} entities with hasMore=true — the storage adapter's getNouns cursor is broken.` ) } cursor = page.nextCursor } else { offset += page.items.length } } } catch (err) { // Non-destructive failure: drop the staging maps (live state keeps // serving), keep the aggregates flagged pending, latch the error so // retries within the cooldown fail fast, and say all of it out loud. for (const n of names) index.abortBackfill(n) this._aggregationBackfillFailure = { at: Date.now(), error: err as Error } prodLog.warn( `[Aggregation] backfill walk FAILED after ${scanned} entities: ${(err as Error).message} — ` + `prior aggregate state preserved; retries suppressed for ${AGGREGATION_BACKFILL_RETRY_COOLDOWN_MS / 1000}s` ) throw err } for (const n of names) index.finishBackfill(n) this._aggregationBackfillFailure = null prodLog.info( `[Aggregation] backfill walk finished: ${scanned} entities → ${names.length} aggregate(s) in ${Date.now() - startedAt}ms` ) } /** * Close and cleanup * * Now flushes HNSW dirty nodes before closing * This ensures deferred persistence mode data is saved */ async close(): Promise { // Change-feed teardown: no events are delivered for or after close(). this._changeFeed.close() // Phase 0a: Persist buffered single-op generation history (async // group-commit) before anything else, so a clean close never drops history // the caller already observed. No-op when nothing is pending or read-only. if (this.generationStore && !this.isReadOnly) { await this.generationStore.flushPendingSingleOps() } // Phase 0b: Auto-compact generational history per config.retention (default // on) BEFORE the generation store closes below. Respects live Db pins and // an explicit autoCompact: false; no-op on read-only instances. await this.autoCompactHistory() // Phase 1: Flush ALL components in parallel to persist buffered data // This is critical when cor native providers buffer data in Rust memory await Promise.all([ // Flush HNSW dirty nodes (deferred persistence mode) (async () => { if (this.index && typeof this.index.flush === 'function') { await this.index.flush() } })(), // Flush metadata index (field indexes + EntityIdMapper) (async () => { if (this.metadataIndex && typeof this.metadataIndex.flush === 'function') { await this.metadataIndex.flush() } })(), // Flush graph adjacency index (LSM trees) (async () => { if (this.graphIndex && typeof this.graphIndex.flush === 'function') { await this.graphIndex.flush() } })(), // Flush storage adapter counts (async () => { if (this.storage && typeof this.storage.flushCounts === 'function') { await this.storage.flushCounts() } })(), // Flush aggregation index state (async () => { if (this._aggregationIndex) { await this._aggregationIndex.flush() } })(), // 8.0 MVCC: detach the generation-bump hook and persist the counter (async () => { if (this.generationStore) { await this.generationStore.close() } })() ]) // Stamp the entity tree at the close boundary (counters + counter now // durable from Phase 1/0a), so a cleanly-closed store reopens COHERENT // instead of benign-behind. Best-effort, never blocks the close. if (this.generationStore && !this.isReadOnly) { await this.stampEntityTree() } // Phase 2: Close components to release resources (timers, file handles) // Data is already safe on disk from Phase 1 await Promise.all([ (async () => { if (this.graphIndex && typeof this.graphIndex.close === 'function') { await this.graphIndex.close() } })(), (async () => { const index = this.index as JsHnswVectorIndex & VectorIndexOptionalHooks if (index && typeof index.close === 'function') { await index.close() } })(), (async () => { const metadataIndex = this.metadataIndex as MetadataIndexManager & MetadataIndexOptionalHooks if (metadataIndex && typeof metadataIndex.close === 'function') { await metadataIndex.close() } })(), (async () => { if (this._materializer) { this._materializer.close() } })() ]) // Deactivate plugins (safe — all data flushed and resources released above) await this.pluginRegistry.deactivateAll() // Restore console methods if silent mode was enabled if (this.config.silent && this.originalConsole) { console.log = this.originalConsole.log as typeof console.log console.info = this.originalConsole.info as typeof console.info console.warn = this.originalConsole.warn as typeof console.warn console.error = this.originalConsole.error as typeof console.error this.originalConsole = undefined } // Drain the metadata write buffer if the storage adapter has one if (this.storage && 'metadataWriteBuffer' in this.storage) { // Boundary: drains BaseStorage's protected `metadataWriteBuffer` member // (cloud adapters initialize it in their init()). No public drain hook // exists on the adapter surface, and close() must not leave a pending // write to land after a successor writer claims the lock. const buffer = (this.storage as unknown as { metadataWriteBuffer?: MetadataWriteBuffer | null }).metadataWriteBuffer if (buffer && typeof buffer.destroy === 'function') { await buffer.destroy() } } // Stop the cross-process flush-request watcher (no-op if never started). if (this.storage && typeof this.storage.stopFlushRequestWatcher === 'function') { this.storage.stopFlushRequestWatcher() } // Release the writer lock (no-op for readers and for backends that don't // hold a lock). Must run after the metadata buffer drain — otherwise a // pending write could land after a successor writer claimed the lock. if (this.storage && typeof this.storage.releaseWriterLock === 'function') { await this.storage.releaseWriterLock() } // Shut down the VFS: stops its background maintenance interval and the // PathResolver's — both are ref'd timers that would keep the process // alive after the last brain closes (consumer-reported hang). if (this._vfs) { await this._vfs.close() } this.initialized = false // close() is terminal: block lazy re-initialization on any subsequent // operation (ensureInitialized() throws once this is set). this.closed = true // Drop this instance from the global registry, and when it was the last // one, deregister the global shutdown hooks — their ref'd signal handles // would otherwise keep the process alive after every brain is closed. const instanceIndex = Brainy.instances.indexOf(this) if (instanceIndex !== -1) { Brainy.instances.splice(instanceIndex, 1) } Brainy.deregisterShutdownHooksIfIdle() } } /** * @description Extract the entity/relationship id from a canonical storage * path of the form `entities/(nouns|verbs)///metadata.json`. * Vector-file and non-canonical paths return `null` — the historical * materializer enumerates each id exactly once, from its metadata file. * @param path - A storage-root-relative object path. * @returns The id, or `null` when the path is not a metadata file. */ function entityIdFromCanonicalPath(path: string): string | null { const match = /^entities[/\\](?:nouns|verbs)[/\\][^/\\]+[/\\]([^/\\]+)[/\\]metadata\.json$/.exec(path) return match ? match[1] : null } /** * @description Narrow `BrainyConfig['storage']` to a pre-constructed storage * adapter instance (vs. a factory config object). Instances are detected by * their `init` method — config objects are plain data and never carry one. * @param value - The configured storage value. * @returns Whether `value` is an adapter instance to use directly. */ function isStorageAdapterInstance( value: BrainyConfig['storage'] ): value is StorageAdapter { return !!value && typeof (value as StorageAdapter).init === 'function' } // Re-export types for convenience export * from './types/brainy.types.js' export { NounType, VerbType } from './types/graphTypes.js'